rails_template_18f 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c732c75ebc9f84d2f730c43042eb922392954f18e8a2500722e4f2e070f329a
-  data.tar.gz: 0fca28e40b443de8032d7a76f38f889449238c62ffeb15444b78a73d9482eaf0
+  metadata.gz: c87267d368d3e90a500c0fc1bf23312a3f6edf437ba991a9a4c7b994fff4282d
+  data.tar.gz: ec24cf97ec587cf730bb2f63ff4cf0fd5b13ae64364b175041b0b0afd42a4321
 SHA512:
-  metadata.gz: c3900e08a7ae9227bad8e4f341b867556ddbaae1614a526ab64be758cd0c78ad0b09e4be1587da1038623151a045f05aa7a7fe3ea82676479f29d54d4cc4d94f
-  data.tar.gz: 7da9792ec9280f443a5bfa2bfc6ffca23a60cf984cb64c0773019dd27ea130b02fcd331647493b81ffcac434fbc8bb5cdf064c1d3e5a5714617a4bcac5335c90
+  metadata.gz: c2c05519936ff836d7c99dd16a5363755a6e86731c5d41454adbc0f6a7ada083fd1b636d0f5da3db446225866f095b42c2278100c85d3c10a3a2182100fcf550
+  data.tar.gz: b2e6da4f298a5c0054997c2c44244e4956044af83af81792937adf7d52716a68b3f8ebf8332f598943b9bad9a37bebf3c6dd4da0eb196ac119cc32455db00e57

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 ## [Unreleased]
 
+## [0.4.0] - 2022-02-24
+
+- helper script to run rails app:update
+- cloud.gov configuration helper generator
+- activestorage/clamav generator
+- activejob/sidekiq generator
+- i18n-js generator
+
 ## [0.3.0] - 2022-02-17
 
 - i18n generator

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rails_template_18f (0.3.0)
+    rails_template_18f (0.4.0)
       activesupport (~> 7.0.0)
       railties (~> 7.0.0)
       thor (~> 1.0)

data/exe/rails_template_18f

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require "thor"
+require_relative "../lib/rails_template18f/version"
 
 class CLI < Thor
   include Thor::Actions
@@ -24,6 +25,36 @@ class CLI < Thor
     railsrc = options[:hotwire] ? "railsrc-hotwire" : "railsrc"
     run "rails new #{app_directory} --rc=#{File.join(gem_path, railsrc)} --template=#{File.join(gem_path, "template.rb")} #{rails_arguments.join(" ")}"
   end
+
+  desc "update", "Run rails app:update with some enhancements"
+  long_desc <<-LONGDESC
+    Run `rails app:update` with frameworks fully defined by what is commented out at the top
+    of config/application.rb
+
+    Example: to enable ActiveStorage
+
+    1) Uncomment `require "active_storage/engine"` in `config/application.rb`
+
+    2) Run `bin/rails active_storage:install`
+
+    3) Run bundle exec rails_template_18f update
+
+    4) Optional: run other rails_template18f generators that may be applicable
+  LONGDESC
+  def update
+    require_relative "../lib/rails_template18f/app_updater"
+    require "rails/command"
+    Rails::Command.invoke "app:update"
+  end
+
+  desc "version", "Output gem version"
+  def version
+    puts RailsTemplate18f::VERSION
+  end
+
+  def self.exit_on_failure?
+    true
+  end
 end
 
 CLI.start(ARGV)

data/lib/generators/rails_template18f/active_storage/active_storage_generator.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module RailsTemplate18f
+  module Generators
+    class ActiveStorageGenerator < ::Rails::Generators::Base
+      include Base
+
+      desc <<~DESC
+        Description:
+          Document use of Clamav as ActiveStorage scanner
+      DESC
+
+      def configure_active_storage
+        generate "rails_template18f:cloud_gov_config"
+        rails_command "active_storage:install"
+        comment_lines "config/environments/production.rb", /active_storage\.service/
+        insert_into_file "config/environments/production.rb", "\n  config.active_storage.service = :amazon", after: /active_storage\.service.*$/
+        environment "config.active_storage.service = :local", env: "ci"
+        append_to_file "config/storage.yml", <<~EOYAML
+
+          amazon:
+            service: S3
+            access_key_id: <%= CloudGovConfig.dig(:s3, :credentials, :access_key_id) %>
+            secret_access_key: <%= CloudGovConfig.dig(:s3, :credentials, :secret_access_key) %>
+            region: us-gov-west-1
+            bucket: <%= CloudGovConfig.dig(:s3, :credentials, :bucket) %>
+        EOYAML
+      end
+
+      def install_gems
+        gem "faraday", "~> 2.2"
+        gem "faraday-multipart", "~> 1.0"
+        gem_group :production do
+          gem "aws-sdk-s3", "~> 1.112"
+        end
+      end
+
+      def create_scanned_upload_model_and_job
+        generate :migration, "CreateFileUploads", "file:attachment", "record:references{polymorphic}", "scan_status:string"
+        migration_file = Dir.glob(File.expand_path(File.join("db", "migrate", "[0-9]*_*.rb"), destination_root)).grep(/\d+_create_file_uploads.rb$/).first
+        unless migration_file.nil?
+          gsub_file migration_file, ":scan_status", ":scan_status, null: false, default: \"uploaded\""
+        end
+        directory "app"
+        directory "spec"
+      end
+
+      def configure_local_clamav_runner
+        append_to_file "Procfile.dev", "clamav: docker run --rm -p 9443:9443 ajilaag/clamav-rest:20211229\n"
+      end
+
+      def configure_clamav_env_var
+        append_to_file ".env", <<~EOM
+
+
+          # CLAMAV_API_URL tells FileScanJob where to send files for virus scans
+          CLAMAV_API_URL=https://localhost:9443
+        EOM
+        insert_into_file "manifest.yml", "    CLAMAV_API_URL: \"https://#{app_name}-clamapi-((env)).apps.internal:9443\"\n", before: /^\s+processes:/
+        insert_into_file "manifest.yml", "\n  - #{app_name}-s3-((env))", after: "services:"
+      end
+
+      def update_boundary_diagram
+        boundary_filename = "doc/compliance/apps/application.boundary.md"
+
+        insert_into_file boundary_filename, indent(<<~EOB, 16), after: /ContainerDb\(app_db.*$\n/
+          Container(clamav, "File Scanning API", "ClamAV", "Internal application for scanning user uploads")
+          ContainerDb(app_s3, "File Storage", "AWS S3", "User-uploaded file storage")
+        EOB
+        insert_into_file boundary_filename, <<~EOB, before: "@enduml"
+          Rel(app, app_s3, "reads/writes file data", "https (443)")
+        EOB
+        if has_active_job?
+          insert_into_file boundary_filename, <<~EOB, before: "@enduml"
+            Rel(worker, app_s3, "reads/writes file data", "https (443)")
+            Rel(worker, clamav, "scans files", "https (9443)")
+          EOB
+        end
+      end
+
+      def update_data_model_uml
+        insert_into_file "doc/compliance/apps/data.logical.md", data_model_uml, before: "@enduml"
+      end
+
+      def generate_adr
+        adr_dir = File.expand_path(File.join("doc", "adr"), destination_root)
+        if Dir.exist? adr_dir
+          @next_adr_id = `ls #{adr_dir} | tail -n 1 | awk -F '-' '{print $1}'`.strip.to_i + 1
+          template "doc/adr/clamav.md", "doc/adr/#{"%04d" % @next_adr_id}-clamav-file-scanning.md"
+        end
+      end
+
+      no_tasks do
+        def data_model_uml
+          <<~UML
+            class file_uploads {
+              * id : bigint <<generated>>
+              * scan_status : string
+              * record_id : bigint
+              * record_type : string
+            }
+            class active_storage_attachments {
+              * id : bigint <<generated>>
+              * name : string
+              * record_type : string
+              * record_id : bigint
+              * blob_id : bigint
+              * created_at : timestamp without time zone
+            }
+            class active_storage_blobs {
+              * id : bigint <<generated>>
+              * key : string
+              * filename : string
+              content_type : string
+              metadata : text
+              * service_name : string
+              * byte_size : bigint
+              checksum : string
+              * created_at : timestamp without time zone
+            }
+            class active_storage_variant_records {
+              * id : bigint <<generated>>
+              * variation_digest : string
+            }
+            file_uploads ||--|| active_storage_attachments
+            active_storage_attachments ||--|{ active_storage_blobs
+            active_storage_variant_records ||--|{ active_storage_blobs
+          UML
+        end
+      end
+    end
+  end
+end

data/lib/generators/rails_template18f/active_storage/templates/app/jobs/file_scan_job.rb

@@ -0,0 +1,33 @@
+class FileScanJob < ApplicationJob
+  queue_as :default
+
+  def perform(file_upload)
+    return if file_upload.nil? || file_upload.clean?
+    file_upload.open do |file|
+      payload = {file: Faraday::Multipart::FilePart.new(
+        file,
+        file_upload.content_type,
+        file_upload.filename
+      )}
+      response = connection.post("/scan", payload)
+      if response.success?
+        file_upload.update_columns scan_status: "scanned", updated_at: Time.now
+      else
+        logger.error "File Scan for #{file_upload.id} failed: #{response.body}"
+        file_upload.update_columns scan_status: "quarantined", updated_at: Time.now
+      end
+    end
+  rescue => ex
+    file_upload&.update_columns scan_status: "scan_failed", updated_at: Time.now
+    raise ex
+  end
+
+  def connection
+    @connection ||= Faraday.new(
+      url: ENV["CLAMAV_API_URL"],
+      ssl: {verify: false}
+    ) do |f|
+      f.request :multipart
+    end
+  end
+end

data/lib/generators/rails_template18f/active_storage/templates/app/models/file_upload.rb

@@ -0,0 +1,25 @@
+class FileUpload < ApplicationRecord
+  belongs_to :record, polymorphic: true
+  has_one_attached :file
+
+  delegate :open, :content_type, to: :file
+
+  validates_presence_of :file
+  validates_inclusion_of :scan_status, in: %w[uploaded scan_failed scanned quarantined]
+
+  after_commit :scan
+
+  def clean?
+    scan_status == "scanned"
+  end
+
+  def filename
+    file.filename.to_s
+  end
+
+  private
+
+  def scan
+    FileScanJob.perform_later self
+  end
+end

data/lib/generators/rails_template18f/active_storage/templates/doc/adr/clamav.md.tt

@@ -0,0 +1,30 @@
+# <%= @next_adr_id %>. ClamAV File Scanning
+
+Date: <%= Date.today.iso8601 %>
+
+## Status
+
+Accepted
+
+## Context
+
+In order to satisfy the [RA-5](https://nvd.nist.gov/800-53/Rev4/control/RA-5)
+control around vulnerability scanning, we wish to scan all user-uploaded files
+with a malware detection service. These files are user-supplied, and thus
+cannot fit into our other security controls built into the CI/CD pipeline.
+
+## Decision
+
+We will run a ClamAV daemon, fronted by a REST API that we will pass all user-uploaded
+files through as part of the upload process.
+<% if terraform_dir_exists? %>
+The ClamAV app is deployed along with other infrastructure by terraform.
+<% else %>
+The ClamAV app is based on a [separate government-controlled repository](https://github.com/18f/clamav-api-cg-app)
+<% end %>
+
+## Consequences
+
+While our user-supplied files will now have vulnerability protection, this architecture
+does not provide scanning of the application itself. Therefore we must find other solutions
+to addressing SI-3 or RA-5 in the context of the application.

data/lib/generators/rails_template18f/active_storage/templates/spec/jobs/file_scan_job_spec.rb

@@ -0,0 +1,35 @@
+require "rails_helper"
+
+RSpec.describe FileScanJob, type: :job do
+  subject { described_class.new }
+  let(:scanned_file) { double(clean?: true) }
+  let(:unscanned_file) { double(id: 1, clean?: false, content_type: "text/plain", filename: "test.txt") }
+  let(:success_response) { double(success?: true) }
+  let(:error_response) { double(success?: false, body: "Error response body") }
+
+  it "deals with a nil argument" do
+    expect { subject.perform nil }.to_not raise_error
+  end
+
+  it "returns quickly if the file is already scanned" do
+    expect { subject.perform scanned_file }.to_not raise_error
+  end
+
+  it "updates the scan_status after scanning the file" do
+    now = Time.now
+    allow(Time).to receive(:now).and_return now
+    allow(unscanned_file).to receive(:open).and_yield __FILE__
+    expect(unscanned_file).to receive(:update_columns).with scan_status: "scanned", updated_at: Time.now
+    allow(subject).to receive(:connection).and_return double(post: success_response)
+    subject.perform unscanned_file
+  end
+
+  it "marks the file as quarantined when dirty" do
+    now = Time.now
+    allow(Time).to receive(:now).and_return now
+    allow(unscanned_file).to receive(:open).and_yield __FILE__
+    expect(unscanned_file).to receive(:update_columns).with scan_status: "quarantined", updated_at: Time.now
+    allow(subject).to receive(:connection).and_return double(post: error_response)
+    subject.perform unscanned_file
+  end
+end

data/lib/generators/rails_template18f/active_storage/templates/spec/models/file_upload_spec.rb

@@ -0,0 +1,38 @@
+require "rails_helper"
+
+RSpec.describe FileUpload, type: :model do
+  subject { described_class.new }
+
+  describe "validations" do
+    before do
+      subject.file.attach(io: File.open(__FILE__), filename: "file_upload_spec.rb")
+    end
+
+    %w[uploaded scan_failed scanned quarantined].each do |valid_status|
+      it "allows scan_status=#{valid_status}" do
+        pending "#{described_class.name} cannot be valid without a record to belong_to"
+        subject.scan_status = valid_status
+        expect(subject).to be_valid
+      end
+    end
+
+    it "is invalid with a bad scan_status" do
+      subject.scan_status = "invalid"
+      expect(subject).to_not be_valid
+    end
+  end
+
+  describe "#clean?" do
+    it "returns true when scan_status is scanned" do
+      subject.scan_status = "scanned"
+      expect(subject).to be_clean
+    end
+
+    it "returns false when scan_status is not scanned" do
+      subject.scan_status = "uploaded"
+      expect(subject).to_not be_clean
+      subject.scan_status = "quarantined"
+      expect(subject).to_not be_clean
+    end
+  end
+end

data/lib/generators/rails_template18f/cloud_gov_config/cloud_gov_config_generator.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module RailsTemplate18f
+  module Generators
+    class CloudGovConfigGenerator < ::Rails::Generators::Base
+      include Base
+
+      desc <<~DESC
+        Description:
+          Install a helper class to retrieve configuration from ENV["VCAP_SERVICES"]
+      DESC
+
+      def install_climate_control
+        return if file_content("Gemfile").match?(/gem "climate_control"/)
+        gem_group :test do
+          gem "climate_control", "~> 1.0"
+        end
+      end
+
+      def install_model_and_test
+        copy_file "app/models/cloud_gov_config.rb"
+        copy_file "spec/models/cloud_gov_config_spec.rb"
+      end
+    end
+  end
+end

data/lib/generators/rails_template18f/cloud_gov_config/templates/app/models/cloud_gov_config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CloudGovConfig
+  ENV_VARIABLE = "VCAP_SERVICES"
+
+  def self.dig(*path)
+    return nil if ENV[ENV_VARIABLE].blank?
+    first, *rest = path
+    vcap_services[first]&.first&.dig(*rest)
+  end
+
+  def self.vcap_services
+    @vcap_services ||= JSON.parse(ENV[ENV_VARIABLE]).with_indifferent_access
+  end
+end

data/lib/generators/rails_template18f/cloud_gov_config/templates/spec/models/cloud_gov_config_spec.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe CloudGovConfig, type: :model do
+  subject { described_class }
+
+  describe ".dig" do
+    context "VCAP_SERVICES is blank" do
+      it "returns nil" do
+        expect(subject.dig(:s3, :credentials, :bucket)).to be_nil
+      end
+    end
+
+    context "VCAP_SERVICES is set" do
+      let(:bucket_name) { "bucket-name" }
+      let(:vcap) {
+        {
+          s3: [
+            {
+              credentials: {
+                bucket: bucket_name
+              }
+            }
+          ]
+        }
+      }
+
+      around do |example|
+        ClimateControl.modify VCAP_SERVICES: vcap.to_json do
+          example.run
+        end
+      end
+
+      it "can find a path" do
+        expect(subject.dig(:s3, :credentials, :bucket)).to eq bucket_name
+      end
+
+      it "returns nil for a missing path" do
+        expect(subject.dig(:s3, :credentials, :other)).to be_nil
+      end
+    end
+  end
+end

data/lib/generators/rails_template18f/i18n_js/i18n_js_generator.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "bundler"
+
+module RailsTemplate18f
+  module Generators
+    class I18nJsGenerator < ::Rails::Generators::Base
+      include Base
+
+      desc <<~DESC
+        Description:
+          Install and configure i18n-js gem to provide translations to JS code.
+
+          By default, will only export translations with keys that match `*.js.*`
+      DESC
+
+      def install_gem_and_tasks
+        return if file_content("Gemfile").match?(/gem "i18n-js"/)
+        gem "i18n-js", "~> 3.9"
+        Bundler.with_original_env do
+          in_root do
+            run "bundle install"
+            run "yarn add i18n-js"
+            generate "i18n:js:config"
+          end
+        end
+        append_to_file "config/i18n-js.yml", <<~EOYAML
+          # remove `only` to include all translations
+          translations:
+            - file: "app/assets/builds/translations.js"
+              only: "*.js.*"
+        EOYAML
+      end
+
+      def configure_asset_pipeline
+        copy_file "lib/tasks/i18n.rake"
+        environment "config.middleware.use I18n::JS::Middleware", env: :development
+        insert_into_file "app/views/layouts/application.html.erb", indent(<<~EOHTML, 4), after: /<%= stylesheet_link_tag "application".*$\n/
+          <%= javascript_include_tag "i18n", "data-turbo-track": "reload" %>
+          <%= javascript_include_tag "translations", "data-turbo-track": "reload" %>
+        EOHTML
+        append_to_file "app/assets/config/manifest.js", <<~EOJS
+          //= link i18n.js
+          //= link translations.js
+        EOJS
+      end
+
+      def ignore_generated_file
+        unless skip_git?
+          append_to_file ".gitignore", <<~EOM
+
+            # Generated by i18n-js
+            /public/javascripts/i18n.js
+          EOM
+        end
+      end
+    end
+  end
+end

data/lib/generators/rails_template18f/i18n_js/templates/lib/tasks/i18n.rake

@@ -0,0 +1,9 @@
+# export translations as part of asset precompile
+
+Rake::Task["assets:precompile"].enhance(["i18n:js:export"])
+
+if Rake::Task.task_defined?("test:prepare")
+  Rake::Task["test:prepare"].enhance(["i18n:js:export"])
+elsif Rake::Task.task_defined?("db:test:prepare")
+  Rake::Task["db:test:prepare"].enhance(["i18n:js:export"])
+end

data/lib/generators/rails_template18f/sidekiq/sidekiq_generator.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module RailsTemplate18f
+  module Generators
+    class SidekiqGenerator < ::Rails::Generators::Base
+      include Base
+
+      desc <<~DESC
+        Description:
+          Install Sidekiq and configure it as the ActiveJob backend
+      DESC
+
+      def install_gem
+        gem "sidekiq", "~> 6.4"
+      end
+
+      def configure_server_runner
+        append_to_file "Procfile.dev", "worker: bundle exec sidekiq\n"
+        insert_into_file "manifest.yml", indent(<<~EOYAML), after: /processes:$\n/
+          - type: worker
+            instances: ((worker_instances))
+            memory: ((worker_memory))
+            command: bundle exec sidekiq
+        EOYAML
+        insert_into_file "manifest.yml", "\n  - #{app_name}-redis-((env))", after: "services:"
+        inside "config/deployment" do
+          append_to_file "staging.yml", <<~EOYAML
+            worker_instances: 1
+            worker_memory: 256M
+          EOYAML
+          append_to_file "production.yml", <<~EOYAML
+            worker_instances: 1
+            worker_memory: 512M
+          EOYAML
+        end
+      end
+
+      def configure_active_job
+        generate "rails_template18f:cloud_gov_config"
+        copy_file "config/initializers/redis.rb"
+        application "config.active_job.queue_adapter = :sidekiq"
+      end
+
+      def configure_sidekiq_ui
+        prepend_to_file "config/routes.rb", "require \"sidekiq/web\"\n\n"
+        route <<~EOR
+          if Rails.env.development?
+            mount Sidekiq::Web => "/sidekiq"
+          end
+        EOR
+      end
+
+      def update_boundary_diagram
+        boundary_filename = "doc/compliance/apps/application.boundary.md"
+
+        insert_into_file boundary_filename, indent(<<~EOB, 16), after: /ContainerDb\(app_db.*$\n/
+          Container(worker, "<&layers> Sidekiq workers", "Ruby #{ruby_version}, Sidekiq", "Perform background work and data processing")
+          ContainerDb(redis, "Redis Database", "AWS ElastiCache (Redis)", "Background job queue")
+        EOB
+        insert_into_file boundary_filename, <<~EOB, before: "@enduml"
+          Rel(app, redis, "enqueue job parameters", "redis")
+          Rel(worker, redis, "dequeues job parameters", "redis")
+          Rel(worker, app_db, "reads/writes primary data", "psql (5432)")
+        EOB
+      end
+    end
+  end
+end

data/lib/generators/rails_template18f/sidekiq/templates/config/initializers/redis.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+Rails.application.config.to_prepare do
+  redis_url = CloudGovConfig.dig "aws-elasticache-redis", "credentials", "uri"
+  if redis_url.present?
+    Sidekiq.configure_server do |config|
+      config.redis = {url: redis_url, ssl: true}
+    end
+
+    Sidekiq.configure_client do |config|
+      config.redis = {url: redis_url, ssl: true}
+    end
+  end
+end

data/lib/generators/rails_template18f/terraform/templates/terraform/production/main.tf.tt

@@ -16,17 +16,49 @@ module "database" {
   recursive_delete = local.recursive_delete
   rds_plan_name    = "TKTK-production-rds-plan"
 }
+<% if has_active_job? %>
+module "redis" {
+  source = "../shared/redis"
 
+  cf_user          = var.cf_user
+  cf_password      = var.cf_password
+  cf_org_name      = local.cf_org_name
+  cf_space_name    = local.cf_space_name
+  env              = local.env
+  recursive_delete = local.recursive_delete
+  redis_plan_name  = "TKTK-production-redis-plan"
+}
+<% end %>
 <% if has_active_storage? %>
 module "s3" {
   source = "../shared/s3"
 
-  cf_user         = var.cf_user
-  cf_password     = var.cf_password
-  cf_org_name     = local.cf_org_name
-  cf_space_name   = local.cf_space_name
-  s3_service_name = "<%= app_name %>-s3-${local.env}"
+  cf_user          = var.cf_user
+  cf_password      = var.cf_password
+  cf_org_name      = local.cf_org_name
+  cf_space_name    = local.cf_space_name
+  recursive_delete = local.recursive_delete
+  s3_service_name  = "<%= app_name %>-s3-${local.env}"<% if cloud_gov_organization == "sandbox-gsa" %>
+  s3_plan_name     = "basic-sandbox"<% end %>
 }
+
+###########################################################################
+# The following lines need to be commented out for the initial `terraform apply`
+# It can be re-enabled after:
+# 1) the app has first been deployed
+# 2) Your organization has sufficient memory. Each clamav app requires 3GB
+###########################################################################
+# module "clamav" {
+#   source = "../shared/clamav"
+#
+#   cf_user       = var.cf_user
+#   cf_password   = var.cf_password
+#   cf_org_name   = local.cf_org_name
+#   cf_space_name = local.cf_space_name
+#   env           = local.env
+#   clamav_image  = "ajilaag/clamav-rest:20211229"
+#   max_file_size = "30M"
+# }
 <% end %>
 
 ###########################################################################

data/lib/generators/rails_template18f/terraform/templates/terraform/shared/clamav/main.tf.tt

@@ -0,0 +1,50 @@
+###
+# Target space/org
+###
+
+data "cloudfoundry_space" "space" {
+  org_name = var.cf_org_name
+  name     = var.cf_space_name
+}
+
+data "cloudfoundry_domain" "internal" {
+  name = "apps.internal"
+}
+
+data "cloudfoundry_app" "app" {
+  name_or_id = "<%= app_name %>-${var.env}"
+  space = data.cloudfoundry_space.space.id
+}
+
+###
+# ClamAV API app
+###
+
+resource "cloudfoundry_route" "clamav_route" {
+  space    = data.cloudfoundry_space.space.id
+  domain   = data.cloudfoundry_domain.internal.id
+  hostname = "<%= app_name %>-clamapi-${var.env}"
+}
+
+resource "cloudfoundry_app" "clamav_api" {
+  name         = "<%= app_name %>-clamav-api-${var.env}"
+  space        = data.cloudfoundry_space.space.id
+  memory       = var.clamav_memory
+  disk_quota   = 2048
+  timeout      = 600
+  docker_image = var.clamav_image
+  routes {
+    route = cloudfoundry_route.clamav_route.id
+  }
+  environment = {
+    MAX_FILE_SIZE = var.max_file_size
+  }
+}
+
+resource "cloudfoundry_network_policy" "clamav_routing" {
+  policy {
+    source_app      = data.cloudfoundry_app.app.id
+    destination_app = cloudfoundry_app.clamav_api.id
+    port            = "9443"
+  }
+}

data/lib/generators/rails_template18f/terraform/templates/terraform/shared/clamav/providers.tf

@@ -0,0 +1,16 @@
+terraform {
+  required_version = "~> 1.0"
+  required_providers {
+    cloudfoundry = {
+      source  = "cloudfoundry-community/cloudfoundry"
+      version = "0.15.0"
+    }
+  }
+}
+
+provider "cloudfoundry" {
+  api_url      = var.cf_api_url
+  user         = var.cf_user
+  password     = var.cf_password
+  app_logs_max = 30
+}

data/lib/generators/rails_template18f/terraform/templates/terraform/shared/clamav/variables.tf

@@ -0,0 +1,47 @@
+variable "cf_api_url" {
+  type        = string
+  description = "cloud.gov api url"
+  default     = "https://api.fr.cloud.gov"
+}
+
+variable "cf_user" {
+  type        = string
+  description = "cloud.gov deployer account user"
+}
+
+variable "cf_password" {
+  type        = string
+  description = "secret; cloud.gov deployer account password"
+  sensitive   = true
+}
+
+variable "cf_org_name" {
+  type        = string
+  description = "cloud.gov organization name"
+}
+
+variable "cf_space_name" {
+  type        = string
+  description = "cloud.gov space name (staging or prod)"
+}
+
+variable "env" {
+  type        = string
+  description = "deployment environment (staging, production)"
+}
+
+variable "clamav_image" {
+  type        = string
+  description = "Docker image to deploy the clamav api app"
+}
+
+variable "clamav_memory" {
+  type        = number
+  description = "Memory in MB to allocate to clamav app"
+  default     = 3072
+}
+
+variable "max_file_size" {
+  type        = string
+  description = "Maximum file size the API will accept for scanning"
+}

data/lib/generators/rails_template18f/terraform/templates/terraform/shared/redis/main.tf.tt

@@ -0,0 +1,23 @@
+###
+# Target space/org
+###
+
+data "cloudfoundry_space" "space" {
+  org_name = var.cf_org_name
+  name     = var.cf_space_name
+}
+
+###
+# RDS instance
+###
+
+data "cloudfoundry_service" "redis" {
+  name = "aws-elasticache-redis"
+}
+
+resource "cloudfoundry_service_instance" "redis" {
+  name             = "<%= app_name %>-redis-${var.env}"
+  space            = data.cloudfoundry_space.space.id
+  service_plan     = data.cloudfoundry_service.redis.service_plans[var.redis_plan_name]
+  recursive_delete = var.recursive_delete
+}

data/lib/generators/rails_template18f/terraform/templates/terraform/shared/redis/providers.tf

@@ -0,0 +1,16 @@
+terraform {
+  required_version = "~> 1.0"
+  required_providers {
+    cloudfoundry = {
+      source  = "cloudfoundry-community/cloudfoundry"
+      version = "0.15.0"
+    }
+  }
+}
+
+provider "cloudfoundry" {
+  api_url      = var.cf_api_url
+  user         = var.cf_user
+  password     = var.cf_password
+  app_logs_max = 30
+}

data/lib/generators/rails_template18f/terraform/templates/terraform/shared/redis/variables.tf

@@ -0,0 +1,42 @@
+variable "cf_api_url" {
+  type        = string
+  description = "cloud.gov api url"
+  default     = "https://api.fr.cloud.gov"
+}
+
+variable "cf_user" {
+  type        = string
+  description = "cloud.gov deployer account user"
+}
+
+variable "cf_password" {
+  type        = string
+  description = "secret; cloud.gov deployer account password"
+  sensitive   = true
+}
+
+variable "cf_org_name" {
+  type        = string
+  description = "cloud.gov organization name"
+}
+
+variable "cf_space_name" {
+  type        = string
+  description = "cloud.gov space name (staging or prod)"
+}
+
+variable "env" {
+  type        = string
+  description = "deployment environment (staging, production)"
+}
+
+variable "recursive_delete" {
+  type        = bool
+  description = "when true, deletes service bindings attached to the resource (not recommended for production)"
+  default     = false
+}
+
+variable "redis_plan_name" {
+  type        = string
+  description = "name of the service plan name to create"
+}

data/lib/generators/rails_template18f/terraform/templates/terraform/staging/main.tf.tt

@@ -16,15 +16,47 @@ module "database" {
   recursive_delete = local.recursive_delete
   rds_plan_name    = "micro-psql"
 }
+<% if has_active_job? %>
+module "redis" {
+  source = "../shared/redis"
 
+  cf_user          = var.cf_user
+  cf_password      = var.cf_password
+  cf_org_name      = local.cf_org_name
+  cf_space_name    = local.cf_space_name
+  env              = local.env
+  recursive_delete = local.recursive_delete
+  redis_plan_name  = "redis-dev"
+}
+<% end %>
 <% if has_active_storage? %>
 module "s3" {
   source = "../shared/s3"
 
-  cf_user         = var.cf_user
-  cf_password     = var.cf_password
-  cf_org_name     = local.cf_org_name
-  cf_space_name   = local.cf_space_name
-  s3_service_name = "<%= app_name %>-s3-${local.env}"
+  cf_user          = var.cf_user
+  cf_password      = var.cf_password
+  cf_org_name      = local.cf_org_name
+  cf_space_name    = local.cf_space_name
+  recursive_delete = local.recursive_delete
+  s3_service_name  = "<%= app_name %>-s3-${local.env}"<% if cloud_gov_organization == "sandbox-gsa" %>
+  s3_plan_name     = "basic-sandbox"<% end %>
 }
+
+###########################################################################
+# The following lines need to be commented out for the initial `terraform apply`
+# It can be re-enabled after:
+# 1) the app has first been deployed
+# 2) Your organization has sufficient memory. Each clamav app requires 3GB
+###########################################################################
+# module "clamav" {
+#   source = "../shared/clamav"
+#
+#   cf_user       = var.cf_user
+#   cf_password   = var.cf_password
+#   cf_org_name   = local.cf_org_name
+#   cf_space_name = local.cf_space_name
+#   env           = local.env
+#   clamav_image  = "ajilaag/clamav-rest:20211229"
+#   max_file_size = "30M"
+# }
 <% end %>

data/lib/generators/rails_template18f/terraform/terraform_generator.rb

@@ -79,17 +79,6 @@ module RailsTemplate18f
           EOM
         end
       end
-
-      private
-
-      def terraform_dir_exists?
-        # prevents cloud_gov_* helpers from trying to read non-existant .tf files
-        false
-      end
-
-      def has_active_storage?
-        defined?(::ActiveStorage)
-      end
     end
   end
 end

data/lib/rails_template18f/app_updater.rb

@@ -0,0 +1,19 @@
+require "rails/app_updater"
+
+module AppUpdaterOptions
+  extend ActiveSupport::Concern
+
+  class_methods do
+    def generator_options
+      options = super
+      # These options all end up hardcoded to true in the default `rails app:update`
+      options[:skip_active_job] = !defined?(ActiveJob::Railtie)
+      options[:skip_action_mailbox] = !defined?(ActionMailbox::Engine)
+      options[:skip_action_text] = !defined?(ActionText::Engine)
+      options[:skip_test] = !defined?(Rails::TestUnitRailtie)
+      options
+    end
+  end
+end
+
+Rails::AppUpdater.prepend(AppUpdaterOptions)

data/lib/rails_template18f/generators/base.rb

@@ -6,10 +6,6 @@ module RailsTemplate18f
       extend ActiveSupport::Concern
       include ::Rails::Generators::AppName
 
-      included do
-        self.source_path = RailsTemplate18f::Generators.const_source_location(name).first
-      end
-
       class_methods do
         attr_accessor :source_path
 
@@ -18,19 +14,40 @@ module RailsTemplate18f
         end
       end
 
+      included do
+        self.source_path = RailsTemplate18f::Generators.const_source_location(name).first
+      end
+
       private
 
       def file_content(filename)
-        File.read(File.expand_path(filename, destination_root))
+        file_path = File.expand_path(filename, destination_root)
+        if File.exist? file_path
+          File.read(file_path)
+        else
+          ""
+        end
       end
 
       def ruby_version
         RUBY_VERSION
       end
 
+      def terraform_dir_exists?
+        Dir.exist? File.expand_path("terraform", destination_root)
+      end
+
       def skip_git?
         !Dir.exist?(File.expand_path(".git", destination_root))
       end
+
+      def has_active_job?
+        defined?(::ActiveJob)
+      end
+
+      def has_active_storage?
+        defined?(::ActiveStorage)
+      end
     end
   end
 end

data/lib/rails_template18f/generators/cloud_gov_options.rb

@@ -48,10 +48,6 @@ module RailsTemplate18f
         end
         "prod"
       end
-
-      def terraform_dir_exists?
-        Dir.exist? File.expand_path("terraform", destination_root)
-      end
     end
   end
 end

data/lib/rails_template18f/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsTemplate18f
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/template.rb

@@ -10,6 +10,10 @@ def skip_git?
   !!options[:skip_git]
 end
 
+def skip_active_job?
+  !!options[:skip_active_job]
+end
+
 def webpack?
   adjusted_javascript_option == "webpack"
 end
@@ -339,6 +343,18 @@ if terraform
   register_announcement("Terraform", "Run the bootstrap script and update the appropriate CI/CD environment variables defined in the Deployment section of the README")
 end
 
+if !skip_active_job?
+  after_bundle do
+    generate "rails_template18f:sidekiq"
+  end
+end
+
+if !skip_active_storage?
+  after_bundle do
+    generate "rails_template18f:active_storage"
+  end
+end
+
 if @github_actions
   after_bundle do
     generator_arguments = [

data/templates/config/deployment/staging.yml

@@ -1,3 +1,3 @@
 env: staging
 web_instances: 1
-web_memory: 512M
+web_memory: 256M

data/templates/config/environments/ci.rb

@@ -1,7 +1,6 @@
 require_relative "./production"
 
 Rails.application.configure do
-  config.assets.compile = true
   config.public_file_server.enabled = true
 
   logger = ActiveSupport::Logger.new($stdout)

data/templates/doc/compliance/apps/application.boundary.md.tt

@@ -30,15 +30,11 @@ Boundary(aws, "AWS GovCloud") {
             System_Boundary(inventory, "Application") {
                 Container(app, "<&layers> <%= app_name.titleize %>", "Ruby <%= @ruby_version %>, Rails <%= Rails.version %>", "TKTK Application Description")
                 ContainerDb(app_db, "Application DB", "AWS RDS (PostgreSQL)", "Primary data storage")
-                <% if !skip_active_storage? %>
-                ContainerDb(app_s3, "File Storage", "AWS S3", "User-uploaded file storage")
-                <% end %>
             }
         }
     }
 }
 
-
 Boundary(gsa_saas, "GSA-authorized SaaS") {
 }
 
@@ -49,9 +45,6 @@ Rel(browser, aws_alb, "request info, submit requests", "https GET/POST (443)")
 Rel(aws_alb, cloudgov_router, "proxies requests", "https GET/POST (443)")
 Rel(cloudgov_router, app, "proxies requests", "https GET/POST (443)")
 Rel(app, app_db, "reads/writes primary data", "psql (5432)")
-<% if !skip_active_storage? %>
-Rel(app, app_s3, "reads/writes file data", "https (443)")
-<% end %>
 @enduml
 ```
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_template_18f
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Ryan Ahearn
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-02-17 00:00:00.000000000 Z
+date: 2022-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties
@@ -114,11 +114,20 @@ files:
 - bin/console
 - bin/setup
 - exe/rails_template_18f
+- lib/generators/rails_template18f/active_storage/active_storage_generator.rb
+- lib/generators/rails_template18f/active_storage/templates/app/jobs/file_scan_job.rb
+- lib/generators/rails_template18f/active_storage/templates/app/models/file_upload.rb
+- lib/generators/rails_template18f/active_storage/templates/doc/adr/clamav.md.tt
+- lib/generators/rails_template18f/active_storage/templates/spec/jobs/file_scan_job_spec.rb
+- lib/generators/rails_template18f/active_storage/templates/spec/models/file_upload_spec.rb
 - lib/generators/rails_template18f/circleci/circleci_generator.rb
 - lib/generators/rails_template18f/circleci/templates/Dockerfile.tt
 - lib/generators/rails_template18f/circleci/templates/bin/ci-server-start
 - lib/generators/rails_template18f/circleci/templates/circleci/config.yml.tt
 - lib/generators/rails_template18f/circleci/templates/docker-compose.ci.yml
+- lib/generators/rails_template18f/cloud_gov_config/cloud_gov_config_generator.rb
+- lib/generators/rails_template18f/cloud_gov_config/templates/app/models/cloud_gov_config.rb
+- lib/generators/rails_template18f/cloud_gov_config/templates/spec/models/cloud_gov_config_spec.rb
 - lib/generators/rails_template18f/dap/dap_generator.rb
 - lib/generators/rails_template18f/github_actions/github_actions_generator.rb
 - lib/generators/rails_template18f/github_actions/templates/github/actions/run-server/action.yml
@@ -139,8 +148,12 @@ files:
 - lib/generators/rails_template18f/i18n/templates/config/locales/es.yml
 - lib/generators/rails_template18f/i18n/templates/config/locales/fr.yml
 - lib/generators/rails_template18f/i18n/templates/config/locales/zh.yml
+- lib/generators/rails_template18f/i18n_js/i18n_js_generator.rb
+- lib/generators/rails_template18f/i18n_js/templates/lib/tasks/i18n.rake
 - lib/generators/rails_template18f/newrelic/newrelic_generator.rb
 - lib/generators/rails_template18f/newrelic/templates/config/newrelic.yml.tt
+- lib/generators/rails_template18f/sidekiq/sidekiq_generator.rb
+- lib/generators/rails_template18f/sidekiq/templates/config/initializers/redis.rb
 - lib/generators/rails_template18f/terraform/templates/terraform/README.md.tt
 - lib/generators/rails_template18f/terraform/templates/terraform/bootstrap/import.sh
 - lib/generators/rails_template18f/terraform/templates/terraform/bootstrap/main.tf.tt
@@ -153,12 +166,18 @@ files:
 - lib/generators/rails_template18f/terraform/templates/terraform/production/main.tf.tt
 - lib/generators/rails_template18f/terraform/templates/terraform/production/providers.tf.tt
 - lib/generators/rails_template18f/terraform/templates/terraform/production/variables.tf
+- lib/generators/rails_template18f/terraform/templates/terraform/shared/clamav/main.tf.tt
+- lib/generators/rails_template18f/terraform/templates/terraform/shared/clamav/providers.tf
+- lib/generators/rails_template18f/terraform/templates/terraform/shared/clamav/variables.tf
 - lib/generators/rails_template18f/terraform/templates/terraform/shared/database/main.tf.tt
 - lib/generators/rails_template18f/terraform/templates/terraform/shared/database/providers.tf
 - lib/generators/rails_template18f/terraform/templates/terraform/shared/database/variables.tf
 - lib/generators/rails_template18f/terraform/templates/terraform/shared/domain/main.tf.tt
 - lib/generators/rails_template18f/terraform/templates/terraform/shared/domain/providers.tf
 - lib/generators/rails_template18f/terraform/templates/terraform/shared/domain/variables.tf
+- lib/generators/rails_template18f/terraform/templates/terraform/shared/redis/main.tf.tt
+- lib/generators/rails_template18f/terraform/templates/terraform/shared/redis/providers.tf
+- lib/generators/rails_template18f/terraform/templates/terraform/shared/redis/variables.tf
 - lib/generators/rails_template18f/terraform/templates/terraform/shared/s3/main.tf
 - lib/generators/rails_template18f/terraform/templates/terraform/shared/s3/providers.tf
 - lib/generators/rails_template18f/terraform/templates/terraform/shared/s3/variables.tf
@@ -166,6 +185,7 @@ files:
 - lib/generators/rails_template18f/terraform/templates/terraform/staging/providers.tf.tt
 - lib/generators/rails_template18f/terraform/templates/terraform/staging/variables.tf
 - lib/generators/rails_template18f/terraform/terraform_generator.rb
+- lib/rails_template18f/app_updater.rb
 - lib/rails_template18f/generators.rb
 - lib/rails_template18f/generators/base.rb
 - lib/rails_template18f/generators/cloud_gov_options.rb