rails_template_18f 0.2.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f3b689fcfbc4d7ed1a476adc0b415ed0cbbfc24828937431820b5850db77e564
-  data.tar.gz: 5f95740853330bc904b23c67059a578b3f616cea6228c7c99a19d4eee5c26518
+  metadata.gz: ccf6b140f15f85229d716db5c0b51db5b88b41172000f89a613bdc09975c61f7
+  data.tar.gz: 6a3a14efb4cb1373236a53260cf06b161375346d37097563ef65197862e538a8
 SHA512:
-  metadata.gz: 6c350c36aeddf44806c2ddbd36dbacd626f361890a9a648ec6e58e4dbeba5b60928ece499542bae318f7f7f644af52b55c7b9e9f6639291f7dd00c00229f5334
-  data.tar.gz: 75079b3718a90069ae1484b54b86dac37c89be836bd132afc8bd936fb0e1451aa85882a5fafa8a980f60094ce352fb933d7ae0179a557a9b6196d018d2a681c1
+  metadata.gz: 8fb2a0c4a4adf9fa72ea8078a6b1001671f2e562ece7b3d74b64ffa39c113b96c280f64a3eacc0233b67c8bc2b6ec160b2867c78e49363100d1cdc7055352516
+  data.tar.gz: 48e96e771a2fb4dd8ca03719f923ae2749b6b212577c88c8e72a3e2788d71b1a6a6bda768611509aa2001e129a7a29c833ad99a93433bf954c46552671d40c2b

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 ## [Unreleased]
 
+## [0.4.1] - 2022-02-25
+
+- update gem dependencies
+- fix issues when included gem hadn't been previously installed
+
+## [0.4.0] - 2022-02-24
+
+- helper script to run rails app:update
+- cloud.gov configuration helper generator
+- activestorage/clamav generator
+- activejob/sidekiq generator
+- i18n-js generator
+
+## [0.3.0] - 2022-02-17
+
+- i18n generator
+- helper script to run rails new without cloning repo
+
 ## [0.2.0] - 2022-02-16
 
 - terraform generator

data/Gemfile

@@ -7,6 +7,4 @@ gemspec
 
 gem "rake", "~> 13.0"
 
-gem "colorize", "~> 0.8"
-
 gem "byebug"

data/Gemfile.lock

@@ -1,9 +1,11 @@
 PATH
   remote: .
   specs:
-    rails_template_18f (0.2.0)
+    rails_template_18f (0.4.1)
       activesupport (~> 7.0.0)
+      colorize (~> 0.8)
       railties (~> 7.0.0)
+      thor (~> 1.0)
 
 GEM
   remote: https://rubygems.org/
@@ -124,7 +126,6 @@ PLATFORMS
 DEPENDENCIES
   ammeter (~> 1.1)
   byebug
-  colorize (~> 0.8)
   rails_template_18f!
   rake (~> 13.0)
   rspec (~> 3.11)

data/README.md

@@ -7,35 +7,32 @@ See the `rails-6` branch for Rails 6.1.x
 
 ## Use for new Rails Project
 
-1. Clone this repository to your computer
-1. Change directory into the clone
-1. Run `rails new <<PATH_TO_PROJECT>> --rc=<<RC_FILE>>` with the appropriate rc file for your needs. The path should not be a subdirectory of this repository.
+1. `gem install rails_template_18f`
+1. `rails_template_18f help new` for usage instructions
 
-### Choosing which RC file to use
-
-You should run this template with either `railsrc` or `railsrc-hotwire` depending on your development needs.
+### Choosing whether to use `--hotwire`
 
 #### Server Rendered _or_ Single Page Applications
 
-`rails new <<PATH_TO_PROJECT>> --rc=railsrc`
+`rails_template_18f new <<PATH_TO_PROJECT>>` _or_ `rails_template_18f new <<PATH_TO_PROJECT>> --no-hotwire`
 
-The base `railsrc` file creates a Rails application that is appropriate for both server-rendered applications,
+This creates a Rails application that is appropriate for both server-rendered applications,
 as well as a basis for installing a separate Single Page Application (SPA) library such as React.
 
 #### A bit more JavaScript needed
 
-`rails new <<PATH_TO_PROJECT>> --rc=railsrc-hotwire`
+`rails_template_18f new <<PATH_TO_PROJECT>> --hotwire`
 
-The `railsrc-hotwire` file creates a Rails application that includes the [Hotwire](https://hotwired.dev/) JavaScript framework.
+This creates a Rails application that includes the [Hotwire](https://hotwired.dev/) JavaScript framework.
 
 Hotwire can be used to add [a bit of JavaScript](https://engineering.18f.gov/web-architecture/#:~:text=are%20more%20complex-,If%20your%20use%20case%20requires%20a%20bit%20of%20client%2Dside%20interactivity%2C%20use%20the%20above%20options%20with%20a%20bit%20of%20JavaScript.,-You%20might%20use)
 for more interactivity than server-rendered apps, but less than a full SPA.
 
 ### Available Options
 
-The following options can be added after `--rc=<<RC_FILE>>` to change how the template behaves.
+The following options can be added to change how the template behaves.
 
-**Important:** You must not pass `--skip-bundle` or `--skip-javascript` to `rails new` or various aspects of the template will be broken
+**Important:** You must not pass `--skip-bundle` or `--skip-javascript` to `rails_template_18f` or various aspects of the template will be broken
 
 #### `--javascript=esbuild`
 
@@ -46,7 +43,7 @@ maintaining IE11 support with esbuild may be tricky.
 
 Each of the skipped frameworks in `railsrc` can be overridden on the command line. For example: `--no-skip-active-storage` will include support for `ActiveStorage` document uploads
 
-### What `railsrc` does
+### What default use or `--no-hotwire` does
 
 ```
 --skip-active-storage   # don't include ActiveStorage for document upload
@@ -61,9 +58,9 @@ Each of the skipped frameworks in `railsrc` can be overridden on the command lin
 --database=postgresql   # default to PostgreSQL
 ```
 
-### What `railsrc-hotwire` does
+### What `--hotwire` does
 
-`railsrc-hotwire` is identical to `railsrc` except that [Hotwire](https://hotwired.dev/) and [ActionCable](https://guides.rubyonrails.org/action_cable_overview.html) are not skipped.
+Identical to `--no-hotwire` except that [Hotwire](https://hotwired.dev/) and [ActionCable](https://guides.rubyonrails.org/action_cable_overview.html) are not skipped.
 
 ActionCable is included to enable the [Turbo Streams](https://turbo.hotwired.dev/handbook/streams) functionality of Hotwire.
 

data/exe/rails_template_18f

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "thor"
+require_relative "../lib/rails_template18f/version"
+
+class CLI < Thor
+  include Thor::Actions
+
+  desc "new APP_DIRECTORY [options] [rails new arguments]", "Run rails new with 18F flavor"
+  option :hotwire, type: :boolean, default: false, desc: "Enable hotwire JS framework"
+  long_desc <<-LONGDESC
+    Create a new rails application in <APP_DIRECTORY> as customized by
+
+    * railsrc: https://github.com/18F/rails-template/blob/main/railsrc
+
+    * template.rb: https://github.com/18F/rails-template/blob/main/template.rb
+
+    with --hotwire option, includes the Hotwire JS framework
+
+    all other arguments will be passed as-is to `rails new`
+  LONGDESC
+  def new(app_directory, *rails_arguments)
+    gem_path = File.expand_path("..", __dir__)
+    railsrc = options[:hotwire] ? "railsrc-hotwire" : "railsrc"
+    run "rails new #{app_directory} --rc=#{File.join(gem_path, railsrc)} --template=#{File.join(gem_path, "template.rb")} #{rails_arguments.join(" ")}"
+  end
+
+  desc "update", "Run rails app:update with some enhancements"
+  long_desc <<-LONGDESC
+    Run `rails app:update` with frameworks fully defined by what is commented out at the top
+    of config/application.rb
+
+    Example: to enable ActiveStorage
+
+    1) Uncomment `require "active_storage/engine"` in `config/application.rb`
+
+    2) Run `bin/rails active_storage:install`
+
+    3) Run bundle exec rails_template_18f update
+
+    4) Optional: run other rails_template18f generators that may be applicable
+  LONGDESC
+  def update
+    require_relative "../lib/rails_template18f/app_updater"
+    require "rails/command"
+    Rails::Command.invoke "app:update"
+  end
+
+  desc "version", "Output gem version"
+  def version
+    puts RailsTemplate18f::VERSION
+  end
+
+  def self.exit_on_failure?
+    true
+  end
+end
+
+CLI.start(ARGV)

data/lib/generators/rails_template18f/active_storage/active_storage_generator.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module RailsTemplate18f
+  module Generators
+    class ActiveStorageGenerator < ::Rails::Generators::Base
+      include Base
+
+      desc <<~DESC
+        Description:
+          Document use of Clamav as ActiveStorage scanner
+      DESC
+
+      def configure_active_storage
+        generate "rails_template18f:cloud_gov_config", inline: true
+        rails_command "active_storage:install", inline: true
+        comment_lines "config/environments/production.rb", /active_storage\.service/
+        insert_into_file "config/environments/production.rb", "\n  config.active_storage.service = :amazon", after: /active_storage\.service.*$/
+        environment "config.active_storage.service = :local", env: "ci"
+        append_to_file "config/storage.yml", <<~EOYAML
+
+          amazon:
+            service: S3
+            access_key_id: <%= CloudGovConfig.dig(:s3, :credentials, :access_key_id) %>
+            secret_access_key: <%= CloudGovConfig.dig(:s3, :credentials, :secret_access_key) %>
+            region: us-gov-west-1
+            bucket: <%= CloudGovConfig.dig(:s3, :credentials, :bucket) %>
+        EOYAML
+      end
+
+      def install_gems
+        faraday_installed = gem_installed?("faraday")
+        middleware_installed = gem_installed?("faraday-multipart")
+        sdk_installed = gem_installed?("aws-sdk-s3")
+        return if faraday_installed && middleware_installed && sdk_installed
+        gem "faraday", "~> 2.2" unless faraday_installed
+        gem "faraday-multipart", "~> 1.0" unless middleware_installed
+        unless sdk_installed
+          gem_group :production do
+            gem "aws-sdk-s3", "~> 1.112"
+          end
+        end
+        bundle_install
+      end
+
+      def create_scanned_upload_model_and_job
+        generate :migration, "CreateFileUploads", "file:attachment", "record:references{polymorphic}", "scan_status:string", inline: true
+        migration_file = Dir.glob(File.expand_path(File.join("db", "migrate", "[0-9]*_*.rb"), destination_root)).grep(/\d+_create_file_uploads.rb$/).first
+        unless migration_file.nil?
+          gsub_file migration_file, ":scan_status", ":scan_status, null: false, default: \"uploaded\""
+        end
+        directory "app"
+        directory "spec"
+      end
+
+      def configure_local_clamav_runner
+        append_to_file "Procfile.dev", "clamav: docker run --rm -p 9443:9443 ajilaag/clamav-rest:20211229\n"
+      end
+
+      def configure_clamav_env_var
+        append_to_file ".env", <<~EOM
+
+
+          # CLAMAV_API_URL tells FileScanJob where to send files for virus scans
+          CLAMAV_API_URL=https://localhost:9443
+        EOM
+        insert_into_file "manifest.yml", "    CLAMAV_API_URL: \"https://#{app_name}-clamapi-((env)).apps.internal:9443\"\n", before: /^\s+processes:/
+        insert_into_file "manifest.yml", "\n  - #{app_name}-s3-((env))", after: "services:"
+      end
+
+      def update_boundary_diagram
+        boundary_filename = "doc/compliance/apps/application.boundary.md"
+
+        insert_into_file boundary_filename, indent(<<~EOB, 16), after: /ContainerDb\(app_db.*$\n/
+          Container(clamav, "File Scanning API", "ClamAV", "Internal application for scanning user uploads")
+          ContainerDb(app_s3, "File Storage", "AWS S3", "User-uploaded file storage")
+        EOB
+        insert_into_file boundary_filename, <<~EOB, before: "@enduml"
+          Rel(app, app_s3, "reads/writes file data", "https (443)")
+        EOB
+        if has_active_job?
+          insert_into_file boundary_filename, <<~EOB, before: "@enduml"
+            Rel(worker, app_s3, "reads/writes file data", "https (443)")
+            Rel(worker, clamav, "scans files", "https (9443)")
+          EOB
+        end
+      end
+
+      def update_data_model_uml
+        insert_into_file "doc/compliance/apps/data.logical.md", data_model_uml, before: "@enduml"
+      end
+
+      def generate_adr
+        adr_dir = File.expand_path(File.join("doc", "adr"), destination_root)
+        if Dir.exist? adr_dir
+          @next_adr_id = `ls #{adr_dir} | tail -n 1 | awk -F '-' '{print $1}'`.strip.to_i + 1
+          template "doc/adr/clamav.md", "doc/adr/#{"%04d" % @next_adr_id}-clamav-file-scanning.md"
+        end
+      end
+
+      no_tasks do
+        def data_model_uml
+          <<~UML
+            class file_uploads {
+              * id : bigint <<generated>>
+              * scan_status : string
+              * record_id : bigint
+              * record_type : string
+            }
+            class active_storage_attachments {
+              * id : bigint <<generated>>
+              * name : string
+              * record_type : string
+              * record_id : bigint
+              * blob_id : bigint
+              * created_at : timestamp without time zone
+            }
+            class active_storage_blobs {
+              * id : bigint <<generated>>
+              * key : string
+              * filename : string
+              content_type : string
+              metadata : text
+              * service_name : string
+              * byte_size : bigint
+              checksum : string
+              * created_at : timestamp without time zone
+            }
+            class active_storage_variant_records {
+              * id : bigint <<generated>>
+              * variation_digest : string
+            }
+            file_uploads ||--|| active_storage_attachments
+            active_storage_attachments ||--|{ active_storage_blobs
+            active_storage_variant_records ||--|{ active_storage_blobs
+          UML
+        end
+      end
+    end
+  end
+end

data/lib/generators/rails_template18f/active_storage/templates/app/jobs/file_scan_job.rb

@@ -0,0 +1,33 @@
+class FileScanJob < ApplicationJob
+  queue_as :default
+
+  def perform(file_upload)
+    return if file_upload.nil? || file_upload.clean?
+    file_upload.open do |file|
+      payload = {file: Faraday::Multipart::FilePart.new(
+        file,
+        file_upload.content_type,
+        file_upload.filename
+      )}
+      response = connection.post("/scan", payload)
+      if response.success?
+        file_upload.update_columns scan_status: "scanned", updated_at: Time.now
+      else
+        logger.error "File Scan for #{file_upload.id} failed: #{response.body}"
+        file_upload.update_columns scan_status: "quarantined", updated_at: Time.now
+      end
+    end
+  rescue => ex
+    file_upload&.update_columns scan_status: "scan_failed", updated_at: Time.now
+    raise ex
+  end
+
+  def connection
+    @connection ||= Faraday.new(
+      url: ENV["CLAMAV_API_URL"],
+      ssl: {verify: false}
+    ) do |f|
+      f.request :multipart
+    end
+  end
+end

data/lib/generators/rails_template18f/active_storage/templates/app/models/file_upload.rb

@@ -0,0 +1,25 @@
+class FileUpload < ApplicationRecord
+  belongs_to :record, polymorphic: true
+  has_one_attached :file
+
+  delegate :open, :content_type, to: :file
+
+  validates_presence_of :file
+  validates_inclusion_of :scan_status, in: %w[uploaded scan_failed scanned quarantined]
+
+  after_commit :scan
+
+  def clean?
+    scan_status == "scanned"
+  end
+
+  def filename
+    file.filename.to_s
+  end
+
+  private
+
+  def scan
+    FileScanJob.perform_later self
+  end
+end

data/lib/generators/rails_template18f/active_storage/templates/doc/adr/clamav.md.tt

@@ -0,0 +1,30 @@
+# <%= @next_adr_id %>. ClamAV File Scanning
+
+Date: <%= Date.today.iso8601 %>
+
+## Status
+
+Accepted
+
+## Context
+
+In order to satisfy the [RA-5](https://nvd.nist.gov/800-53/Rev4/control/RA-5)
+control around vulnerability scanning, we wish to scan all user-uploaded files
+with a malware detection service. These files are user-supplied, and thus
+cannot fit into our other security controls built into the CI/CD pipeline.
+
+## Decision
+
+We will run a ClamAV daemon, fronted by a REST API that we will pass all user-uploaded
+files through as part of the upload process.
+<% if terraform_dir_exists? %>
+The ClamAV app is deployed along with other infrastructure by terraform.
+<% else %>
+The ClamAV app is based on a [separate government-controlled repository](https://github.com/18f/clamav-api-cg-app)
+<% end %>
+
+## Consequences
+
+While our user-supplied files will now have vulnerability protection, this architecture
+does not provide scanning of the application itself. Therefore we must find other solutions
+to addressing SI-3 or RA-5 in the context of the application.

data/lib/generators/rails_template18f/active_storage/templates/spec/jobs/file_scan_job_spec.rb

@@ -0,0 +1,35 @@
+require "rails_helper"
+
+RSpec.describe FileScanJob, type: :job do
+  subject { described_class.new }
+  let(:scanned_file) { double(clean?: true) }
+  let(:unscanned_file) { double(id: 1, clean?: false, content_type: "text/plain", filename: "test.txt") }
+  let(:success_response) { double(success?: true) }
+  let(:error_response) { double(success?: false, body: "Error response body") }
+
+  it "deals with a nil argument" do
+    expect { subject.perform nil }.to_not raise_error
+  end
+
+  it "returns quickly if the file is already scanned" do
+    expect { subject.perform scanned_file }.to_not raise_error
+  end
+
+  it "updates the scan_status after scanning the file" do
+    now = Time.now
+    allow(Time).to receive(:now).and_return now
+    allow(unscanned_file).to receive(:open).and_yield __FILE__
+    expect(unscanned_file).to receive(:update_columns).with scan_status: "scanned", updated_at: Time.now
+    allow(subject).to receive(:connection).and_return double(post: success_response)
+    subject.perform unscanned_file
+  end
+
+  it "marks the file as quarantined when dirty" do
+    now = Time.now
+    allow(Time).to receive(:now).and_return now
+    allow(unscanned_file).to receive(:open).and_yield __FILE__
+    expect(unscanned_file).to receive(:update_columns).with scan_status: "quarantined", updated_at: Time.now
+    allow(subject).to receive(:connection).and_return double(post: error_response)
+    subject.perform unscanned_file
+  end
+end

data/lib/generators/rails_template18f/active_storage/templates/spec/models/file_upload_spec.rb

@@ -0,0 +1,38 @@
+require "rails_helper"
+
+RSpec.describe FileUpload, type: :model do
+  subject { described_class.new }
+
+  describe "validations" do
+    before do
+      subject.file.attach(io: File.open(__FILE__), filename: "file_upload_spec.rb")
+    end
+
+    %w[uploaded scan_failed scanned quarantined].each do |valid_status|
+      it "allows scan_status=#{valid_status}" do
+        pending "#{described_class.name} cannot be valid without a record to belong_to"
+        subject.scan_status = valid_status
+        expect(subject).to be_valid
+      end
+    end
+
+    it "is invalid with a bad scan_status" do
+      subject.scan_status = "invalid"
+      expect(subject).to_not be_valid
+    end
+  end
+
+  describe "#clean?" do
+    it "returns true when scan_status is scanned" do
+      subject.scan_status = "scanned"
+      expect(subject).to be_clean
+    end
+
+    it "returns false when scan_status is not scanned" do
+      subject.scan_status = "uploaded"
+      expect(subject).to_not be_clean
+      subject.scan_status = "quarantined"
+      expect(subject).to_not be_clean
+    end
+  end
+end

data/lib/generators/rails_template18f/circleci/circleci_generator.rb

@@ -14,7 +14,10 @@ module RailsTemplate18f
       DESC
 
       def install_needed_gems
-        gem "rspec_junit_formatter", "~> 0.5", group: :test
+        gem_name = "rspec_junit_formatter"
+        return if gem_installed? gem_name
+        gem gem_name, "~> 0.5", group: :test
+        bundle_install
       end
 
       def install_pipeline

data/lib/generators/rails_template18f/cloud_gov_config/cloud_gov_config_generator.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module RailsTemplate18f
+  module Generators
+    class CloudGovConfigGenerator < ::Rails::Generators::Base
+      include Base
+
+      desc <<~DESC
+        Description:
+          Install a helper class to retrieve configuration from ENV["VCAP_SERVICES"]
+      DESC
+
+      def install_climate_control
+        return if gem_installed?("climate_control")
+        gem_group :test do
+          gem "climate_control", "~> 1.0"
+        end
+        bundle_install
+      end
+
+      def install_model_and_test
+        copy_file "app/models/cloud_gov_config.rb"
+        copy_file "spec/models/cloud_gov_config_spec.rb"
+      end
+    end
+  end
+end

data/lib/generators/rails_template18f/cloud_gov_config/templates/app/models/cloud_gov_config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CloudGovConfig
+  ENV_VARIABLE = "VCAP_SERVICES"
+
+  def self.dig(*path)
+    return nil if ENV[ENV_VARIABLE].blank?
+    first, *rest = path
+    vcap_services[first]&.first&.dig(*rest)
+  end
+
+  def self.vcap_services
+    @vcap_services ||= JSON.parse(ENV[ENV_VARIABLE]).with_indifferent_access
+  end
+end

data/lib/generators/rails_template18f/cloud_gov_config/templates/spec/models/cloud_gov_config_spec.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe CloudGovConfig, type: :model do
+  subject { described_class }
+
+  describe ".dig" do
+    context "VCAP_SERVICES is blank" do
+      it "returns nil" do
+        expect(subject.dig(:s3, :credentials, :bucket)).to be_nil
+      end
+    end
+
+    context "VCAP_SERVICES is set" do
+      let(:bucket_name) { "bucket-name" }
+      let(:vcap) {
+        {
+          s3: [
+            {
+              credentials: {
+                bucket: bucket_name
+              }
+            }
+          ]
+        }
+      }
+
+      around do |example|
+        ClimateControl.modify VCAP_SERVICES: vcap.to_json do
+          example.run
+        end
+      end
+
+      it "can find a path" do
+        expect(subject.dig(:s3, :credentials, :bucket)).to eq bucket_name
+      end
+
+      it "returns nil for a missing path" do
+        expect(subject.dig(:s3, :credentials, :other)).to be_nil
+      end
+    end
+  end
+end