rails_template_18f 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c732c75ebc9f84d2f730c43042eb922392954f18e8a2500722e4f2e070f329a
-  data.tar.gz: 0fca28e40b443de8032d7a76f38f889449238c62ffeb15444b78a73d9482eaf0
+  metadata.gz: c9452d60916a767ee1b00bb0a76e6582d4d1ed760b3bf0c4d68a07c8225ddfd4
+  data.tar.gz: 481c2ee7693849f2c21d382d78edd08a59370b8c2b440bef8502d22a039a2243
 SHA512:
-  metadata.gz: c3900e08a7ae9227bad8e4f341b867556ddbaae1614a526ab64be758cd0c78ad0b09e4be1587da1038623151a045f05aa7a7fe3ea82676479f29d54d4cc4d94f
-  data.tar.gz: 7da9792ec9280f443a5bfa2bfc6ffca23a60cf984cb64c0773019dd27ea130b02fcd331647493b81ffcac434fbc8bb5cdf064c1d3e5a5714617a4bcac5335c90
+  metadata.gz: 8a5b1be47db25412b0ba04dd19a395401f86b6d5ee14b907c2de54e95bfae159a6407851bfcd83b07ac20ca9423de568580c5cfdec0c5d49bfd5e2f7f62e59df
+  data.tar.gz: 75ff52ec4661f95c4da617217d7eb7dc993192610d9209ab3f9a438504111281a8c779db3074c1d6506073b126d437c6aaebe328b11add1363797938f70d006e

data/CHANGELOG.md

@@ -1,5 +1,24 @@
 ## [Unreleased]
 
+## [0.5.0] - 2022-03-04
+
+- use Brewfile for installing homebrew-based dependencies
+- move test site banner to the _usa_banner.html.erb partial
+- use dockerize within bin/with-server to wait for rails to start
+
+## [0.4.1] - 2022-02-25
+
+- update gem dependencies
+- fix issues when included gem hadn't been previously installed
+
+## [0.4.0] - 2022-02-24
+
+- helper script to run rails app:update
+- cloud.gov configuration helper generator
+- activestorage/clamav generator
+- activejob/sidekiq generator
+- i18n-js generator
+
 ## [0.3.0] - 2022-02-17
 
 - i18n generator

data/Gemfile

@@ -7,6 +7,4 @@ gemspec
 
 gem "rake", "~> 13.0"
 
-gem "colorize", "~> 0.8"
-
 gem "byebug"

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    rails_template_18f (0.3.0)
+    rails_template_18f (0.5.0)
       activesupport (~> 7.0.0)
+      colorize (~> 0.8)
       railties (~> 7.0.0)
       thor (~> 1.0)
 
@@ -46,9 +47,9 @@ GEM
       nokogiri (>= 1.5.9)
     method_source (1.0.0)
     minitest (5.15.0)
-    nokogiri (1.13.1-x86_64-darwin)
+    nokogiri (1.13.3-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.13.1-x86_64-linux)
+    nokogiri (1.13.3-x86_64-linux)
       racc (~> 1.4)
     parallel (1.21.0)
     parser (3.1.0.0)
@@ -125,7 +126,6 @@ PLATFORMS
 DEPENDENCIES
   ammeter (~> 1.1)
   byebug
-  colorize (~> 0.8)
   rails_template_18f!
   rake (~> 13.0)
   rspec (~> 3.11)

data/exe/rails_template_18f

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require "thor"
+require_relative "../lib/rails_template18f/version"
 
 class CLI < Thor
   include Thor::Actions
@@ -24,6 +25,36 @@ class CLI < Thor
     railsrc = options[:hotwire] ? "railsrc-hotwire" : "railsrc"
     run "rails new #{app_directory} --rc=#{File.join(gem_path, railsrc)} --template=#{File.join(gem_path, "template.rb")} #{rails_arguments.join(" ")}"
   end
+
+  desc "update", "Run rails app:update with some enhancements"
+  long_desc <<-LONGDESC
+    Run `rails app:update` with frameworks fully defined by what is commented out at the top
+    of config/application.rb
+
+    Example: to enable ActiveStorage
+
+    1) Uncomment `require "active_storage/engine"` in `config/application.rb`
+
+    2) Run `bin/rails active_storage:install`
+
+    3) Run bundle exec rails_template_18f update
+
+    4) Optional: run other rails_template18f generators that may be applicable
+  LONGDESC
+  def update
+    require_relative "../lib/rails_template18f/app_updater"
+    require "rails/command"
+    Rails::Command.invoke "app:update"
+  end
+
+  desc "version", "Output gem version"
+  def version
+    puts RailsTemplate18f::VERSION
+  end
+
+  def self.exit_on_failure?
+    true
+  end
 end
 
 CLI.start(ARGV)

data/lib/generators/rails_template18f/active_storage/active_storage_generator.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module RailsTemplate18f
+  module Generators
+    class ActiveStorageGenerator < ::Rails::Generators::Base
+      include Base
+
+      desc <<~DESC
+        Description:
+          Document use of Clamav as ActiveStorage scanner
+      DESC
+
+      def configure_active_storage
+        generate "rails_template18f:cloud_gov_config", inline: true
+        rails_command "active_storage:install", inline: true
+        comment_lines "config/environments/production.rb", /active_storage\.service/
+        insert_into_file "config/environments/production.rb", "\n  config.active_storage.service = :amazon", after: /active_storage\.service.*$/
+        environment "config.active_storage.service = :local", env: "ci"
+        append_to_file "config/storage.yml", <<~EOYAML
+
+          amazon:
+            service: S3
+            access_key_id: <%= CloudGovConfig.dig(:s3, :credentials, :access_key_id) %>
+            secret_access_key: <%= CloudGovConfig.dig(:s3, :credentials, :secret_access_key) %>
+            region: us-gov-west-1
+            bucket: <%= CloudGovConfig.dig(:s3, :credentials, :bucket) %>
+        EOYAML
+      end
+
+      def install_gems
+        faraday_installed = gem_installed?("faraday")
+        middleware_installed = gem_installed?("faraday-multipart")
+        sdk_installed = gem_installed?("aws-sdk-s3")
+        return if faraday_installed && middleware_installed && sdk_installed
+        gem "faraday", "~> 2.2" unless faraday_installed
+        gem "faraday-multipart", "~> 1.0" unless middleware_installed
+        unless sdk_installed
+          gem_group :production do
+            gem "aws-sdk-s3", "~> 1.112"
+          end
+        end
+        bundle_install
+      end
+
+      def create_scanned_upload_model_and_job
+        generate :migration, "CreateFileUploads", "file:attachment", "record:references{polymorphic}", "scan_status:string", inline: true
+        migration_file = Dir.glob(File.expand_path(File.join("db", "migrate", "[0-9]*_*.rb"), destination_root)).grep(/\d+_create_file_uploads.rb$/).first
+        unless migration_file.nil?
+          gsub_file migration_file, ":scan_status", ":scan_status, null: false, default: \"uploaded\""
+        end
+        directory "app"
+        directory "spec"
+      end
+
+      def configure_local_clamav_runner
+        append_to_file "Procfile.dev", "clamav: docker run --rm -p 9443:9443 ajilaag/clamav-rest:20211229\n"
+      end
+
+      def configure_clamav_env_var
+        append_to_file ".env", <<~EOM
+
+          # CLAMAV_API_URL tells FileScanJob where to send files for virus scans
+          CLAMAV_API_URL=https://localhost:9443
+        EOM
+        insert_into_file "manifest.yml", "    CLAMAV_API_URL: \"https://#{app_name}-clamapi-((env)).apps.internal:9443\"\n", before: /^\s+processes:/
+        insert_into_file "manifest.yml", "\n  - #{app_name}-s3-((env))", after: "services:"
+      end
+
+      def update_boundary_diagram
+        boundary_filename = "doc/compliance/apps/application.boundary.md"
+
+        insert_into_file boundary_filename, indent(<<~EOB, 16), after: /ContainerDb\(app_db.*$\n/
+          Container(clamav, "File Scanning API", "ClamAV", "Internal application for scanning user uploads")
+          ContainerDb(app_s3, "File Storage", "AWS S3", "User-uploaded file storage")
+        EOB
+        insert_into_file boundary_filename, <<~EOB, before: "@enduml"
+          Rel(app, app_s3, "reads/writes file data", "https (443)")
+        EOB
+        if has_active_job?
+          insert_into_file boundary_filename, <<~EOB, before: "@enduml"
+            Rel(worker, app_s3, "reads/writes file data", "https (443)")
+            Rel(worker, clamav, "scans files", "https (9443)")
+          EOB
+        end
+      end
+
+      def update_data_model_uml
+        insert_into_file "doc/compliance/apps/data.logical.md", data_model_uml, before: "@enduml"
+      end
+
+      def generate_adr
+        adr_dir = File.expand_path(File.join("doc", "adr"), destination_root)
+        if Dir.exist? adr_dir
+          @next_adr_id = `ls #{adr_dir} | tail -n 1 | awk -F '-' '{print $1}'`.strip.to_i + 1
+          template "doc/adr/clamav.md", "doc/adr/#{"%04d" % @next_adr_id}-clamav-file-scanning.md"
+        end
+      end
+
+      no_tasks do
+        def data_model_uml
+          <<~UML
+            class file_uploads {
+              * id : bigint <<generated>>
+              * scan_status : string
+              * record_id : bigint
+              * record_type : string
+            }
+            class active_storage_attachments {
+              * id : bigint <<generated>>
+              * name : string
+              * record_type : string
+              * record_id : bigint
+              * blob_id : bigint
+              * created_at : timestamp without time zone
+            }
+            class active_storage_blobs {
+              * id : bigint <<generated>>
+              * key : string
+              * filename : string
+              content_type : string
+              metadata : text
+              * service_name : string
+              * byte_size : bigint
+              checksum : string
+              * created_at : timestamp without time zone
+            }
+            class active_storage_variant_records {
+              * id : bigint <<generated>>
+              * variation_digest : string
+            }
+            file_uploads ||--|| active_storage_attachments
+            active_storage_attachments ||--|{ active_storage_blobs
+            active_storage_variant_records ||--|{ active_storage_blobs
+          UML
+        end
+      end
+    end
+  end
+end

data/lib/generators/rails_template18f/active_storage/templates/app/jobs/file_scan_job.rb

@@ -0,0 +1,33 @@
+class FileScanJob < ApplicationJob
+  queue_as :default
+
+  def perform(file_upload)
+    return if file_upload.nil? || file_upload.clean?
+    file_upload.open do |file|
+      payload = {file: Faraday::Multipart::FilePart.new(
+        file,
+        file_upload.content_type,
+        file_upload.filename
+      )}
+      response = connection.post("/scan", payload)
+      if response.success?
+        file_upload.update_columns scan_status: "scanned", updated_at: Time.now
+      else
+        logger.error "File Scan for #{file_upload.id} failed: #{response.body}"
+        file_upload.update_columns scan_status: "quarantined", updated_at: Time.now
+      end
+    end
+  rescue => ex
+    file_upload&.update_columns scan_status: "scan_failed", updated_at: Time.now
+    raise ex
+  end
+
+  def connection
+    @connection ||= Faraday.new(
+      url: ENV["CLAMAV_API_URL"],
+      ssl: {verify: false}
+    ) do |f|
+      f.request :multipart
+    end
+  end
+end

data/lib/generators/rails_template18f/active_storage/templates/app/models/file_upload.rb

@@ -0,0 +1,25 @@
+class FileUpload < ApplicationRecord
+  belongs_to :record, polymorphic: true
+  has_one_attached :file
+
+  delegate :open, :content_type, to: :file
+
+  validates_presence_of :file
+  validates_inclusion_of :scan_status, in: %w[uploaded scan_failed scanned quarantined]
+
+  after_commit :scan
+
+  def clean?
+    scan_status == "scanned"
+  end
+
+  def filename
+    file.filename.to_s
+  end
+
+  private
+
+  def scan
+    FileScanJob.perform_later self
+  end
+end

data/lib/generators/rails_template18f/active_storage/templates/doc/adr/clamav.md.tt

@@ -0,0 +1,30 @@
+# <%= @next_adr_id %>. ClamAV File Scanning
+
+Date: <%= Date.today.iso8601 %>
+
+## Status
+
+Accepted
+
+## Context
+
+In order to satisfy the [RA-5](https://nvd.nist.gov/800-53/Rev4/control/RA-5)
+control around vulnerability scanning, we wish to scan all user-uploaded files
+with a malware detection service. These files are user-supplied, and thus
+cannot fit into our other security controls built into the CI/CD pipeline.
+
+## Decision
+
+We will run a ClamAV daemon, fronted by a REST API that we will pass all user-uploaded
+files through as part of the upload process.
+<% if terraform_dir_exists? %>
+The ClamAV app is deployed along with other infrastructure by terraform.
+<% else %>
+The ClamAV app is based on a [separate government-controlled repository](https://github.com/18f/clamav-api-cg-app)
+<% end %>
+
+## Consequences
+
+While our user-supplied files will now have vulnerability protection, this architecture
+does not provide scanning of the application itself. Therefore we must find other solutions
+to addressing SI-3 or RA-5 in the context of the application.

data/lib/generators/rails_template18f/active_storage/templates/spec/jobs/file_scan_job_spec.rb

@@ -0,0 +1,35 @@
+require "rails_helper"
+
+RSpec.describe FileScanJob, type: :job do
+  subject { described_class.new }
+  let(:scanned_file) { double(clean?: true) }
+  let(:unscanned_file) { double(id: 1, clean?: false, content_type: "text/plain", filename: "test.txt") }
+  let(:success_response) { double(success?: true) }
+  let(:error_response) { double(success?: false, body: "Error response body") }
+
+  it "deals with a nil argument" do
+    expect { subject.perform nil }.to_not raise_error
+  end
+
+  it "returns quickly if the file is already scanned" do
+    expect { subject.perform scanned_file }.to_not raise_error
+  end
+
+  it "updates the scan_status after scanning the file" do
+    now = Time.now
+    allow(Time).to receive(:now).and_return now
+    allow(unscanned_file).to receive(:open).and_yield __FILE__
+    expect(unscanned_file).to receive(:update_columns).with scan_status: "scanned", updated_at: Time.now
+    allow(subject).to receive(:connection).and_return double(post: success_response)
+    subject.perform unscanned_file
+  end
+
+  it "marks the file as quarantined when dirty" do
+    now = Time.now
+    allow(Time).to receive(:now).and_return now
+    allow(unscanned_file).to receive(:open).and_yield __FILE__
+    expect(unscanned_file).to receive(:update_columns).with scan_status: "quarantined", updated_at: Time.now
+    allow(subject).to receive(:connection).and_return double(post: error_response)
+    subject.perform unscanned_file
+  end
+end

data/lib/generators/rails_template18f/active_storage/templates/spec/models/file_upload_spec.rb

@@ -0,0 +1,38 @@
+require "rails_helper"
+
+RSpec.describe FileUpload, type: :model do
+  subject { described_class.new }
+
+  describe "validations" do
+    before do
+      subject.file.attach(io: File.open(__FILE__), filename: "file_upload_spec.rb")
+    end
+
+    %w[uploaded scan_failed scanned quarantined].each do |valid_status|
+      it "allows scan_status=#{valid_status}" do
+        pending "#{described_class.name} cannot be valid without a record to belong_to"
+        subject.scan_status = valid_status
+        expect(subject).to be_valid
+      end
+    end
+
+    it "is invalid with a bad scan_status" do
+      subject.scan_status = "invalid"
+      expect(subject).to_not be_valid
+    end
+  end
+
+  describe "#clean?" do
+    it "returns true when scan_status is scanned" do
+      subject.scan_status = "scanned"
+      expect(subject).to be_clean
+    end
+
+    it "returns false when scan_status is not scanned" do
+      subject.scan_status = "uploaded"
+      expect(subject).to_not be_clean
+      subject.scan_status = "quarantined"
+      expect(subject).to_not be_clean
+    end
+  end
+end

data/lib/generators/rails_template18f/circleci/circleci_generator.rb

@@ -14,7 +14,10 @@ module RailsTemplate18f
       DESC
 
       def install_needed_gems
-        gem "rspec_junit_formatter", "~> 0.5", group: :test
+        gem_name = "rspec_junit_formatter"
+        return if gem_installed? gem_name
+        gem gem_name, "~> 0.5", group: :test
+        bundle_install
       end
 
       def install_pipeline

data/lib/generators/rails_template18f/cloud_gov_config/cloud_gov_config_generator.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module RailsTemplate18f
+  module Generators
+    class CloudGovConfigGenerator < ::Rails::Generators::Base
+      include Base
+
+      desc <<~DESC
+        Description:
+          Install a helper class to retrieve configuration from ENV["VCAP_SERVICES"]
+      DESC
+
+      def install_climate_control
+        return if gem_installed?("climate_control")
+        gem_group :test do
+          gem "climate_control", "~> 1.0"
+        end
+        bundle_install
+      end
+
+      def install_model_and_test
+        copy_file "app/models/cloud_gov_config.rb"
+        copy_file "spec/models/cloud_gov_config_spec.rb"
+      end
+    end
+  end
+end

data/lib/generators/rails_template18f/cloud_gov_config/templates/app/models/cloud_gov_config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CloudGovConfig
+  ENV_VARIABLE = "VCAP_SERVICES"
+
+  def self.dig(*path)
+    return nil if ENV[ENV_VARIABLE].blank?
+    first, *rest = path
+    vcap_services[first]&.first&.dig(*rest)
+  end
+
+  def self.vcap_services
+    @vcap_services ||= JSON.parse(ENV[ENV_VARIABLE]).with_indifferent_access
+  end
+end

data/lib/generators/rails_template18f/cloud_gov_config/templates/spec/models/cloud_gov_config_spec.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe CloudGovConfig, type: :model do
+  subject { described_class }
+
+  describe ".dig" do
+    context "VCAP_SERVICES is blank" do
+      it "returns nil" do
+        expect(subject.dig(:s3, :credentials, :bucket)).to be_nil
+      end
+    end
+
+    context "VCAP_SERVICES is set" do
+      let(:bucket_name) { "bucket-name" }
+      let(:vcap) {
+        {
+          s3: [
+            {
+              credentials: {
+                bucket: bucket_name
+              }
+            }
+          ]
+        }
+      }
+
+      around do |example|
+        ClimateControl.modify VCAP_SERVICES: vcap.to_json do
+          example.run
+        end
+      end
+
+      it "can find a path" do
+        expect(subject.dig(:s3, :credentials, :bucket)).to eq bucket_name
+      end
+
+      it "returns nil for a missing path" do
+        expect(subject.dig(:s3, :credentials, :other)).to be_nil
+      end
+    end
+  end
+end

data/lib/generators/rails_template18f/i18n/i18n_generator.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "rails/generators"
-require "bundler"
 
 module RailsTemplate18f
   module Generators
@@ -16,17 +15,17 @@ module RailsTemplate18f
           Always installs configuration for English
       DESC
 
-      def install_helper_gem_and_tasks
-        return if file_content("Gemfile").match?(/gem "i18n-tasks"/)
+      def install_gem
+        return if gem_installed?("i18n-tasks")
         gem_group :development, :test do
           gem "i18n-tasks", "~> 0.9"
         end
-        Bundler.with_original_env do
-          in_root do
-            run "bundle install"
-            run "cp $(i18n-tasks gem-path)/templates/config/i18n-tasks.yml config/"
-            run "cp $(i18n-tasks gem-path)/templates/rspec/i18n_spec.rb spec/"
-          end
+      end
+
+      def install_helper_tasks
+        bundle_install do
+          run "cp $(i18n-tasks gem-path)/templates/config/i18n-tasks.yml config/"
+          run "cp $(i18n-tasks gem-path)/templates/rspec/i18n_spec.rb spec/"
         end
         insert_into_file "config/i18n-tasks.yml", "\n#{indent("- app/assets/builds", 4)}", after: "exclude:"
         uncomment_lines "config/i18n-tasks.yml", "ignore_missing:"

data/lib/generators/rails_template18f/i18n_js/i18n_js_generator.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module RailsTemplate18f
+  module Generators
+    class I18nJsGenerator < ::Rails::Generators::Base
+      include Base
+
+      desc <<~DESC
+        Description:
+          Install and configure i18n-js gem to provide translations to JS code.
+
+          By default, will only export translations with keys that match `*.js.*`
+      DESC
+
+      def install_gem_and_tasks
+        return if gem_installed?("i18n-js")
+        gem "i18n-js", "~> 3.9"
+        bundle_install do
+          run "yarn add i18n-js"
+          generate "i18n:js:config"
+        end
+      end
+
+      def configure_translation_yaml
+        append_to_file "config/i18n-js.yml", <<~EOYAML
+          # remove `only` to include all translations
+          translations:
+            - file: "app/assets/builds/translations.js"
+              only: "*.js.*"
+        EOYAML
+      end
+
+      def configure_asset_pipeline
+        copy_file "lib/tasks/i18n.rake"
+        environment "config.middleware.use I18n::JS::Middleware", env: :development
+        insert_into_file "app/views/layouts/application.html.erb", indent(<<~EOHTML, 4), after: /<%= stylesheet_link_tag "application".*$\n/
+          <%= javascript_include_tag "i18n", "data-turbo-track": "reload" %>
+          <%= javascript_include_tag "translations", "data-turbo-track": "reload" %>
+        EOHTML
+        append_to_file "app/assets/config/manifest.js", <<~EOJS
+          //= link i18n.js
+          //= link translations.js
+        EOJS
+      end
+
+      def ignore_generated_file
+        unless skip_git?
+          append_to_file ".gitignore", <<~EOM
+
+            # Generated by i18n-js
+            /public/javascripts/i18n.js
+          EOM
+        end
+      end
+    end
+  end
+end

data/lib/generators/rails_template18f/i18n_js/templates/lib/tasks/i18n.rake

@@ -0,0 +1,9 @@
+# export translations as part of asset precompile
+
+Rake::Task["assets:precompile"].enhance(["i18n:js:export"])
+
+if Rake::Task.task_defined?("test:prepare")
+  Rake::Task["test:prepare"].enhance(["i18n:js:export"])
+elsif Rake::Task.task_defined?("db:test:prepare")
+  Rake::Task["db:test:prepare"].enhance(["i18n:js:export"])
+end

data/lib/generators/rails_template18f/newrelic/newrelic_generator.rb

@@ -23,7 +23,9 @@ module RailsTemplate18f
       end
 
       def install_gem
+        return if gem_installed?("newrelic_rpm")
         gem "newrelic_rpm", "~> 8.4"
+        bundle_install
       end
 
       def install_config