rails_template_18f 0.3.0 → 0.5.0

data/lib/generators/rails_template18f/sidekiq/sidekiq_generator.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module RailsTemplate18f
+  module Generators
+    class SidekiqGenerator < ::Rails::Generators::Base
+      include Base
+
+      desc <<~DESC
+        Description:
+          Install Sidekiq and configure it as the ActiveJob backend
+      DESC
+
+      def install_gem
+        return if gem_installed?("sidekiq")
+        gem "sidekiq", "~> 6.4"
+        bundle_install
+      end
+
+      def install_redis
+        append_to_file "Brewfile", <<~EOB
+
+          # queue for sidekiq jobs
+          brew "redis"
+        EOB
+        insert_into_file "README.md", indent("* [redis]()\n"), after: /\* Install homebrew dependencies: `brew bundle`\n/
+      end
+
+      def configure_server_runner
+        append_to_file "Procfile.dev", "worker: bundle exec sidekiq\n"
+        insert_into_file "manifest.yml", indent(<<~EOYAML), after: /processes:$\n/
+          - type: worker
+            instances: ((worker_instances))
+            memory: ((worker_memory))
+            command: bundle exec sidekiq
+        EOYAML
+        insert_into_file "manifest.yml", "\n  - #{app_name}-redis-((env))", after: "services:"
+        inside "config/deployment" do
+          append_to_file "staging.yml", <<~EOYAML
+            worker_instances: 1
+            worker_memory: 256M
+          EOYAML
+          append_to_file "production.yml", <<~EOYAML
+            worker_instances: 1
+            worker_memory: 512M
+          EOYAML
+        end
+      end
+
+      def configure_active_job
+        generate "rails_template18f:cloud_gov_config", inline: true
+        copy_file "config/initializers/redis.rb"
+        application "config.active_job.queue_adapter = :sidekiq"
+      end
+
+      def configure_sidekiq_ui
+        prepend_to_file "config/routes.rb", "require \"sidekiq/web\"\n\n"
+        route <<~EOR
+          if Rails.env.development?
+            mount Sidekiq::Web => "/sidekiq"
+          end
+        EOR
+      end
+
+      def update_boundary_diagram
+        boundary_filename = "doc/compliance/apps/application.boundary.md"
+
+        insert_into_file boundary_filename, indent(<<~EOB, 16), after: /ContainerDb\(app_db.*$\n/
+          Container(worker, "<&layers> Sidekiq workers", "Ruby #{ruby_version}, Sidekiq", "Perform background work and data processing")
+          ContainerDb(redis, "Redis Database", "AWS ElastiCache (Redis)", "Background job queue")
+        EOB
+        insert_into_file boundary_filename, <<~EOB, before: "@enduml"
+          Rel(app, redis, "enqueue job parameters", "redis")
+          Rel(worker, redis, "dequeues job parameters", "redis")
+          Rel(worker, app_db, "reads/writes primary data", "psql (5432)")
+        EOB
+      end
+    end
+  end
+end

data/lib/generators/rails_template18f/sidekiq/templates/config/initializers/redis.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+Rails.application.config.to_prepare do
+  redis_url = CloudGovConfig.dig "aws-elasticache-redis", "credentials", "uri"
+  if redis_url.present?
+    Sidekiq.configure_server do |config|
+      config.redis = {url: redis_url, ssl: true}
+    end
+
+    Sidekiq.configure_client do |config|
+      config.redis = {url: redis_url, ssl: true}
+    end
+  end
+end

data/lib/generators/rails_template18f/terraform/templates/terraform/README.md.tt

@@ -2,7 +2,7 @@
 
 This directory holds the terraform modules for maintaining your complete persistent infrastructure.
 
-Prerequisite: install the `jq` JSON processor: `brew install jq`
+Prerequisite: install the `jq` JSON processor: `brew bundle` or `brew install jq`
 
 ## Initial setup
 

data/lib/generators/rails_template18f/terraform/templates/terraform/production/main.tf.tt

@@ -16,17 +16,49 @@ module "database" {
   recursive_delete = local.recursive_delete
   rds_plan_name    = "TKTK-production-rds-plan"
 }
+<% if has_active_job? %>
+module "redis" {
+  source = "../shared/redis"
 
+  cf_user          = var.cf_user
+  cf_password      = var.cf_password
+  cf_org_name      = local.cf_org_name
+  cf_space_name    = local.cf_space_name
+  env              = local.env
+  recursive_delete = local.recursive_delete
+  redis_plan_name  = "TKTK-production-redis-plan"
+}
+<% end %>
 <% if has_active_storage? %>
 module "s3" {
   source = "../shared/s3"
 
-  cf_user         = var.cf_user
-  cf_password     = var.cf_password
-  cf_org_name     = local.cf_org_name
-  cf_space_name   = local.cf_space_name
-  s3_service_name = "<%= app_name %>-s3-${local.env}"
+  cf_user          = var.cf_user
+  cf_password      = var.cf_password
+  cf_org_name      = local.cf_org_name
+  cf_space_name    = local.cf_space_name
+  recursive_delete = local.recursive_delete
+  s3_service_name  = "<%= app_name %>-s3-${local.env}"<% if cloud_gov_organization == "sandbox-gsa" %>
+  s3_plan_name     = "basic-sandbox"<% end %>
 }
+
+###########################################################################
+# The following lines need to be commented out for the initial `terraform apply`
+# It can be re-enabled after:
+# 1) the app has first been deployed
+# 2) Your organization has sufficient memory. Each clamav app requires 3GB
+###########################################################################
+# module "clamav" {
+#   source = "../shared/clamav"
+#
+#   cf_user       = var.cf_user
+#   cf_password   = var.cf_password
+#   cf_org_name   = local.cf_org_name
+#   cf_space_name = local.cf_space_name
+#   env           = local.env
+#   clamav_image  = "ajilaag/clamav-rest:20211229"
+#   max_file_size = "30M"
+# }
 <% end %>
 
 ###########################################################################

data/lib/generators/rails_template18f/terraform/templates/terraform/shared/clamav/main.tf.tt

@@ -0,0 +1,50 @@
+###
+# Target space/org
+###
+
+data "cloudfoundry_space" "space" {
+  org_name = var.cf_org_name
+  name     = var.cf_space_name
+}
+
+data "cloudfoundry_domain" "internal" {
+  name = "apps.internal"
+}
+
+data "cloudfoundry_app" "app" {
+  name_or_id = "<%= app_name %>-${var.env}"
+  space = data.cloudfoundry_space.space.id
+}
+
+###
+# ClamAV API app
+###
+
+resource "cloudfoundry_route" "clamav_route" {
+  space    = data.cloudfoundry_space.space.id
+  domain   = data.cloudfoundry_domain.internal.id
+  hostname = "<%= app_name %>-clamapi-${var.env}"
+}
+
+resource "cloudfoundry_app" "clamav_api" {
+  name         = "<%= app_name %>-clamav-api-${var.env}"
+  space        = data.cloudfoundry_space.space.id
+  memory       = var.clamav_memory
+  disk_quota   = 2048
+  timeout      = 600
+  docker_image = var.clamav_image
+  routes {
+    route = cloudfoundry_route.clamav_route.id
+  }
+  environment = {
+    MAX_FILE_SIZE = var.max_file_size
+  }
+}
+
+resource "cloudfoundry_network_policy" "clamav_routing" {
+  policy {
+    source_app      = data.cloudfoundry_app.app.id
+    destination_app = cloudfoundry_app.clamav_api.id
+    port            = "9443"
+  }
+}

data/lib/generators/rails_template18f/terraform/templates/terraform/shared/clamav/providers.tf

@@ -0,0 +1,16 @@
+terraform {
+  required_version = "~> 1.0"
+  required_providers {
+    cloudfoundry = {
+      source  = "cloudfoundry-community/cloudfoundry"
+      version = "0.15.0"
+    }
+  }
+}
+
+provider "cloudfoundry" {
+  api_url      = var.cf_api_url
+  user         = var.cf_user
+  password     = var.cf_password
+  app_logs_max = 30
+}

data/lib/generators/rails_template18f/terraform/templates/terraform/shared/clamav/variables.tf

@@ -0,0 +1,47 @@
+variable "cf_api_url" {
+  type        = string
+  description = "cloud.gov api url"
+  default     = "https://api.fr.cloud.gov"
+}
+
+variable "cf_user" {
+  type        = string
+  description = "cloud.gov deployer account user"
+}
+
+variable "cf_password" {
+  type        = string
+  description = "secret; cloud.gov deployer account password"
+  sensitive   = true
+}
+
+variable "cf_org_name" {
+  type        = string
+  description = "cloud.gov organization name"
+}
+
+variable "cf_space_name" {
+  type        = string
+  description = "cloud.gov space name (staging or prod)"
+}
+
+variable "env" {
+  type        = string
+  description = "deployment environment (staging, production)"
+}
+
+variable "clamav_image" {
+  type        = string
+  description = "Docker image to deploy the clamav api app"
+}
+
+variable "clamav_memory" {
+  type        = number
+  description = "Memory in MB to allocate to clamav app"
+  default     = 3072
+}
+
+variable "max_file_size" {
+  type        = string
+  description = "Maximum file size the API will accept for scanning"
+}

data/lib/generators/rails_template18f/terraform/templates/terraform/shared/redis/main.tf.tt

@@ -0,0 +1,23 @@
+###
+# Target space/org
+###
+
+data "cloudfoundry_space" "space" {
+  org_name = var.cf_org_name
+  name     = var.cf_space_name
+}
+
+###
+# RDS instance
+###
+
+data "cloudfoundry_service" "redis" {
+  name = "aws-elasticache-redis"
+}
+
+resource "cloudfoundry_service_instance" "redis" {
+  name             = "<%= app_name %>-redis-${var.env}"
+  space            = data.cloudfoundry_space.space.id
+  service_plan     = data.cloudfoundry_service.redis.service_plans[var.redis_plan_name]
+  recursive_delete = var.recursive_delete
+}

data/lib/generators/rails_template18f/terraform/templates/terraform/shared/redis/providers.tf

@@ -0,0 +1,16 @@
+terraform {
+  required_version = "~> 1.0"
+  required_providers {
+    cloudfoundry = {
+      source  = "cloudfoundry-community/cloudfoundry"
+      version = "0.15.0"
+    }
+  }
+}
+
+provider "cloudfoundry" {
+  api_url      = var.cf_api_url
+  user         = var.cf_user
+  password     = var.cf_password
+  app_logs_max = 30
+}

data/lib/generators/rails_template18f/terraform/templates/terraform/shared/redis/variables.tf

@@ -0,0 +1,42 @@
+variable "cf_api_url" {
+  type        = string
+  description = "cloud.gov api url"
+  default     = "https://api.fr.cloud.gov"
+}
+
+variable "cf_user" {
+  type        = string
+  description = "cloud.gov deployer account user"
+}
+
+variable "cf_password" {
+  type        = string
+  description = "secret; cloud.gov deployer account password"
+  sensitive   = true
+}
+
+variable "cf_org_name" {
+  type        = string
+  description = "cloud.gov organization name"
+}
+
+variable "cf_space_name" {
+  type        = string
+  description = "cloud.gov space name (staging or prod)"
+}
+
+variable "env" {
+  type        = string
+  description = "deployment environment (staging, production)"
+}
+
+variable "recursive_delete" {
+  type        = bool
+  description = "when true, deletes service bindings attached to the resource (not recommended for production)"
+  default     = false
+}
+
+variable "redis_plan_name" {
+  type        = string
+  description = "name of the service plan name to create"
+}

data/lib/generators/rails_template18f/terraform/templates/terraform/staging/main.tf.tt

@@ -16,15 +16,47 @@ module "database" {
   recursive_delete = local.recursive_delete
   rds_plan_name    = "micro-psql"
 }
+<% if has_active_job? %>
+module "redis" {
+  source = "../shared/redis"
 
+  cf_user          = var.cf_user
+  cf_password      = var.cf_password
+  cf_org_name      = local.cf_org_name
+  cf_space_name    = local.cf_space_name
+  env              = local.env
+  recursive_delete = local.recursive_delete
+  redis_plan_name  = "redis-dev"
+}
+<% end %>
 <% if has_active_storage? %>
 module "s3" {
   source = "../shared/s3"
 
-  cf_user         = var.cf_user
-  cf_password     = var.cf_password
-  cf_org_name     = local.cf_org_name
-  cf_space_name   = local.cf_space_name
-  s3_service_name = "<%= app_name %>-s3-${local.env}"
+  cf_user          = var.cf_user
+  cf_password      = var.cf_password
+  cf_org_name      = local.cf_org_name
+  cf_space_name    = local.cf_space_name
+  recursive_delete = local.recursive_delete
+  s3_service_name  = "<%= app_name %>-s3-${local.env}"<% if cloud_gov_organization == "sandbox-gsa" %>
+  s3_plan_name     = "basic-sandbox"<% end %>
 }
+
+###########################################################################
+# The following lines need to be commented out for the initial `terraform apply`
+# It can be re-enabled after:
+# 1) the app has first been deployed
+# 2) Your organization has sufficient memory. Each clamav app requires 3GB
+###########################################################################
+# module "clamav" {
+#   source = "../shared/clamav"
+#
+#   cf_user       = var.cf_user
+#   cf_password   = var.cf_password
+#   cf_org_name   = local.cf_org_name
+#   cf_space_name = local.cf_space_name
+#   env           = local.env
+#   clamav_image  = "ajilaag/clamav-rest:20211229"
+#   max_file_size = "30M"
+# }
 <% end %>

data/lib/generators/rails_template18f/terraform/terraform_generator.rb

@@ -19,6 +19,15 @@ module RailsTemplate18f
         chmod "terraform/bootstrap/teardown_creds.sh", 0o755
       end
 
+      def install_jq
+        append_to_file "Brewfile", <<~EOB
+
+          # used in terraform/create_space_deployer.sh
+          brew "jq"
+        EOB
+        insert_into_file "README.md", indent("* [jq](https://stedolan.github.io/jq/)\n"), after: /\* Install homebrew dependencies: `brew bundle`\n/
+      end
+
       def ignore_files
         unless skip_git?
           append_to_file ".gitignore", <<~EOM
@@ -34,7 +43,7 @@ module RailsTemplate18f
       end
 
       def update_readme
-        gsub_file "README.md", /^(### Automatic linting)/, '\1 and terraform formatting'
+        gsub_file "README.md", /^(### Automatic linting)\s*$/, '\1 and terraform formatting'
         gsub_file "README.md", /(ruby linting) (on every)/, '\1 and terraform formatting \2'
         gsub_file "README.md", /^Before the first deploy only.*$/, "Follow the instructions in `terraform/README.md` to create the supporting services."
       end
@@ -79,17 +88,6 @@ module RailsTemplate18f
           EOM
         end
       end
-
-      private
-
-      def terraform_dir_exists?
-        # prevents cloud_gov_* helpers from trying to read non-existant .tf files
-        false
-      end
-
-      def has_active_storage?
-        defined?(::ActiveStorage)
-      end
     end
   end
 end

data/lib/rails_template18f/app_updater.rb

@@ -0,0 +1,19 @@
+require "rails/app_updater"
+
+module AppUpdaterOptions
+  extend ActiveSupport::Concern
+
+  class_methods do
+    def generator_options
+      options = super
+      # These options all end up hardcoded to true in the default `rails app:update`
+      options[:skip_active_job] = !defined?(ActiveJob::Railtie)
+      options[:skip_action_mailbox] = !defined?(ActionMailbox::Engine)
+      options[:skip_action_text] = !defined?(ActionText::Engine)
+      options[:skip_test] = !defined?(Rails::TestUnitRailtie)
+      options
+    end
+  end
+end
+
+Rails::AppUpdater.prepend(AppUpdaterOptions)

data/lib/rails_template18f/generators/base.rb

@@ -1,15 +1,13 @@
 # frozen_string_literal: true
 
+require "bundler"
+
 module RailsTemplate18f
   module Generators
     module Base
       extend ActiveSupport::Concern
       include ::Rails::Generators::AppName
 
-      included do
-        self.source_path = RailsTemplate18f::Generators.const_source_location(name).first
-      end
-
       class_methods do
         attr_accessor :source_path
 
@@ -18,19 +16,53 @@ module RailsTemplate18f
         end
       end
 
+      included do
+        self.source_path = RailsTemplate18f::Generators.const_source_location(name).first
+      end
+
       private
 
+      def bundle_install
+        Bundler.with_original_env do
+          in_root do
+            run "bundle install"
+            yield if block_given?
+          end
+        end
+      end
+
+      def gem_installed?(gem_name)
+        file_content("Gemfile").match?(/gem "#{gem_name}"/)
+      end
+
       def file_content(filename)
-        File.read(File.expand_path(filename, destination_root))
+        file_path = File.expand_path(filename, destination_root)
+        if File.exist? file_path
+          File.read(file_path)
+        else
+          ""
+        end
       end
 
       def ruby_version
         RUBY_VERSION
       end
 
+      def terraform_dir_exists?
+        Dir.exist? File.expand_path("terraform", destination_root)
+      end
+
       def skip_git?
         !Dir.exist?(File.expand_path(".git", destination_root))
       end
+
+      def has_active_job?
+        defined?(::ActiveJob)
+      end
+
+      def has_active_storage?
+        defined?(::ActiveStorage)
+      end
     end
   end
 end

data/lib/rails_template18f/generators/cloud_gov_options.rb

@@ -48,10 +48,6 @@ module RailsTemplate18f
         end
         "prod"
       end
-
-      def terraform_dir_exists?
-        Dir.exist? File.expand_path("terraform", destination_root)
-      end
     end
   end
 end

data/lib/rails_template18f/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsTemplate18f
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end

data/rails-template-18f.gemspec

@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "railties", "~> 7.0.0"
   spec.add_dependency "activesupport", "~> 7.0.0"
   spec.add_dependency "thor", "~> 1.0"
+  spec.add_dependency "colorize", "~> 0.8"
 
   spec.add_development_dependency "rspec", "~> 3.11"
   spec.add_development_dependency "ammeter", "~> 1.1"

data/template.rb

@@ -10,6 +10,10 @@ def skip_git?
   !!options[:skip_git]
 end
 
+def skip_active_job?
+  !!options[:skip_active_job]
+end
+
 def webpack?
   adjusted_javascript_option == "webpack"
 end
@@ -83,6 +87,13 @@ register_announcement("Documentation", <<~EOM)
   * Review any TBD sections of the README and update where appropriate.
 EOM
 
+# ensure dependencies are installed
+copy_file "Brewfile"
+insert_into_file "bin/setup", <<EOSETUP, after: /Add necessary setup steps to this file.\n/
+  puts "== Installing homebrew dependencies =="
+  system("brew bundle --no-lock")
+EOSETUP
+
 # setup nvmrc
 file ".nvmrc", @node_version
 
@@ -187,9 +198,14 @@ copy_file "env", ".env"
 copy_file "githooks/pre-commit", ".githooks/pre-commit", mode: :preserve
 
 unless skip_git?
-  rails_command "credentials:diff --enroll"
+  after_bundle do
+    rails_command "credentials:diff --enroll"
+  end
   append_to_file ".gitignore", <<~EOM
 
+    # Ignore Brewfile debug info
+    Brewfile.lock.json
+
     # Ignore local dotenv overrides
     .env*.local
 
@@ -339,6 +355,18 @@ if terraform
   register_announcement("Terraform", "Run the bootstrap script and update the appropriate CI/CD environment variables defined in the Deployment section of the README")
 end
 
+if !skip_active_job?
+  after_bundle do
+    generate "rails_template18f:sidekiq"
+  end
+end
+
+if !skip_active_storage?
+  after_bundle do
+    generate "rails_template18f:active_storage"
+  end
+end
+
 if @github_actions
   after_bundle do
     generator_arguments = [

data/templates/Brewfile

@@ -0,0 +1,14 @@
+# Brewfile
+# add any dependencies that must be installed from homebrew here
+
+# cloud.gov RDS is on postgres 12
+brew "postgresql@12", link: true
+
+# used in bin/with-server script
+brew "dockerize"
+
+# helper scripts for creating new ADRs
+brew "adr-tools"
+
+# chromedriver for integration tests
+cask "chromedriver"