hecks 3.6.0 → 3.9.0

data/lib/hecks/hecks/adapters/site_toolchain/prober.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "openssl"
+require_relative "../../../projections/site/probe"
+
+module Hecks
+  module Adapters
+    class SiteToolchain
+      # Sends a route table's probe requests to a running site and reports each answer.
+      #
+      # Every request is anonymous and follows no redirect, so a probe changes nothing on the site.
+      class Prober
+        # Seconds to wait to connect to the site, and again for its answer.
+        TIMEOUT = 10
+
+        # The failures of the network that fail one check rather than the whole run.
+        NETWORK = [SystemCallError, Timeout::Error, SocketError, OpenSSL::SSL::SSLError].freeze
+
+        # @param rows [Array<Projections::Site::Table::Row>] the checked rows of the route table
+        # @param run [String] the run's name
+        # @param base [String] the site's scheme, host and port
+        def initialize(rows, run:, base:)
+          @probe = Projections::Site::Probe.new(rows, run: run)
+          @base = base
+        end
+
+        # @return [Array<(String, Integer)>] the report, one line per check ending in the count,
+        #   and how many checks failed
+        def call
+          lines = @probe.checks.map { |check| judged(check) }
+          failed = lines.count { |_, wrong| wrong }
+          [lines.map(&:first).append("#{lines.size} checks, #{failed} failed").join("\n"), failed]
+        end
+
+        private
+
+        # One check's line of the report: its label and `ok`, or the reason it failed.
+        def judged(check)
+          why = @probe.verdict(check, answer_to(check))
+          ["  #{check.label}... #{why ? "FAILED: #{why}" : "ok"}", !why.nil?]
+        rescue *NETWORK => e
+          ["  #{check.label}... FAILED: #{e.class}: #{e.message}", true]
+        end
+
+        def answer_to(check)
+          uri = URI.join(@base, check.path)
+          request = Net::HTTP.const_get(check.verb.capitalize).new(uri)
+          request["Origin"] = @base
+          response = connection(uri).request(request)
+          Projections::Site::Probe::Response.new(status: response.code.to_i, body: response.body,
+                                                 headers: response.to_hash.transform_values(&:first))
+        end
+
+        def connection(uri)
+          Net::HTTP.new(uri.host, uri.port).tap do |http|
+            http.use_ssl = uri.scheme == "https"
+            http.open_timeout = http.read_timeout = TIMEOUT
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hecks/hecks/adapters/site_toolchain.rb

@@ -8,6 +8,7 @@ require_relative "codebase/ruby_child"
 require "hecks/tools"
 require "hecks/tools/site_routes"
 require "hecks/projections/site/live_edge"
+require_relative "site_toolchain/prober"
 
 module Hecks
   module Adapters
@@ -66,8 +67,35 @@ module Hecks
         { output: { value: comparison.to_s } }
       end
 
+      # Asks a running site what its route table says it must answer, and judges each answer. It
+      # sends anonymous requests only, follows no redirect, and changes nothing on the site: an
+      # admin route is asked without a session and must refuse.
+      #
+      # @param held [Hash] the `SiteProjection` record: `domain`, `run` and `url`, the site's base
+      #   address
+      # @return [Hash{Symbol => Hash}] `output:` one line per check, ending in the count
+      # @raise [ConsoleCapture::Failure] when the address is not http(s) or an answer is wrong; the
+      #   message is the report
+      def probe(**held)
+        rows = table_of(located(:domain, held[:domain])).rows
+        report, failed = Prober.new(rows, run: plain(held[:run]).to_s, base: base_url(plain(held[:url]))).call
+        raise ConsoleCapture::Failure, report if failed.positive?
+
+        { output: { value: report } }
+      end
+
       private
 
+      # The scheme, host and port of an http(s) address.
+      def base_url(url)
+        uri = URI.parse(url.to_s)
+        raise URI::InvalidURIError unless uri.is_a?(URI::HTTP) && uri.host
+
+        "#{uri.scheme}://#{uri.host}#{":#{uri.port}" unless uri.port == uri.default_port}"
+      rescue URI::InvalidURIError
+        raise ConsoleCapture::Failure, "url must be an http(s) address, got #{url.inspect}"
+      end
+
       # The flags `project_site` is run with: each the record names, and `--check` when asked.
       def project_flags(held)
         flags = FLAGS.filter_map { |flag, key| "--#{flag}=#{located(key, held[key])}" unless plain(held[key]).nil? }
@@ -84,15 +112,23 @@ module Hecks
 
       # The project's checked edge, read the way `project_site` reads it.
       def edge_of(root, template = nil)
+        registry, chapter, table = read_table(root)
+        site = Projections::Site
+        site::Edge.read(chapter, table: table, template: template, vocabulary: site::Table.vocabulary(registry)) ||
+          raise(ConsoleCapture::Failure, "#{root} declares no Edge rows")
+      end
+
+      # The project's checked route table.
+      def table_of(root) = read_table(root).last
+
+      # The registry, chapter and checked table of the project at `root`.
+      def read_table(root)
         registry = nil
         outcome = ConsoleCapture.capture { registry = Tools::SiteRoutes.registry_for(root) }
         raise ConsoleCapture::Failure, outcome.output.strip unless outcome.ok?
 
-        site = Projections::Site
-        chapter = site::Table.chapter(registry)
-        table = site::Table.read(chapter, registry: registry)
-        site::Edge.read(chapter, table: table, template: template, vocabulary: site::Table.vocabulary(registry)) ||
-          raise(ConsoleCapture::Failure, "#{root} declares no Edge rows")
+        chapter = Projections::Site::Table.chapter(registry)
+        [registry, chapter, Projections::Site::Table.read(chapter, registry: registry)]
       rescue Projections::Site::Table::Invalid => e
         raise ConsoleCapture::Failure, e.message
       end

data/lib/hecks/hecks/custodian.behaviors

@@ -147,6 +147,12 @@ Hecks.behaviors "Custodian" do
     expect refused: "a count is positive"
   end
 
+  test "Mutate refuses a budget that is not positive" do
+    tests "Mutate", on: "FuzzRun"
+    input run: { value: "mutants" }, domain: { value: "examples/pizzas" }, budget: { value: 0 }
+    expect refused: "a count is positive"
+  end
+
   test "Fuzz refuses a persistence it does not know" do
     tests "Fuzz", on: "FuzzRun"
     input run: { value: "sweep" }, adapter: { value: "cassandra" }

data/lib/hecks/hecks/custodian.bluebook

@@ -1963,7 +1963,9 @@ policy "RecordTheEra" do
     attribute :seeds,      Count,       optional: true
     attribute :steps,      Count,       optional: true
     attribute :workers,    Count,       optional: true
+    attribute :budget,     Count,       optional: true
     attribute :adapter,    Persistence, optional: true
+    attribute :persist_regressions, RegressionSwitch, optional: true
     attribute :domains,    NameList,    optional: true
     attribute :targets,    NameList,    optional: true
     attribute :iterations, Count,       optional: true
@@ -2006,6 +2008,11 @@ policy "RecordTheEra" do
       attribute :value, String, pattern: '^(memory|sqlite|postgres)$'
     end
 
+    # Whether each distinct finding's minimized repro is also kept under `spec/corpus/regressions/`.
+    value_object "RegressionSwitch" do
+      attribute :value, Boolean
+    end
+
     # Names, comma separated: benchmark domains, or the targets to measure.
     value_object "NameList" do
       attribute :value, String, pattern: '^[^ \t\n\r,]+(,[^ \t\n\r,]+)*$'
@@ -2047,6 +2054,7 @@ policy "RecordTheEra" do
       attribute :steps,   Count,       optional: true
       attribute :workers, Count,       optional: true
       attribute :adapter, Persistence, optional: true
+      attribute :persist_regressions, RegressionSwitch, optional: true
 
       sets :domain
       sets :run
@@ -2054,10 +2062,34 @@ policy "RecordTheEra" do
       sets :steps
       sets :workers
       sets :adapter
+      sets :persist_regressions
 
       emits FuzzRequested
     end
 
+    # Mutation-tests a domain's own checks: changes its bluebooks in small ways and keeps which
+    # changes the fuzz properties, the corpus script and the domain's behaviors tests let through.
+    # A survivor is a hole in those checks. `budget` caps how many mutants are tried. The run is a
+    # child process started through the ProcessPool adapter, as a sweep is.
+    command "Mutate" do
+      role "Operator"
+      goal "Mutate a domain's rules and keep whether its checks noticed each change"
+
+      attribute :domain, DomainPath
+      attribute :run,    FuzzKey
+      attribute :seeds,  Count, optional: true
+      attribute :steps,  Count, optional: true
+      attribute :budget, Count, optional: true
+
+      sets :domain
+      sets :run
+      sets :seeds
+      sets :steps
+      sets :budget
+
+      emits MutateRequested
+    end
+
     # Measures each persistence adapter, and the Rust binary when there is one to build or use.
     # The report is kept in this record, which the Hecks domain binds to Memory: keeping a
     # domain's benchmark history beside it, in Sqlite under the project's `.hecks/`, is decided
@@ -2148,6 +2180,11 @@ policy "RecordTheEra" do
     trigger FuzzRun::ProcessPool::Sweep, with: { run: :run }
   end
 
+  policy "MutateWhenRequested" do
+    on      "FuzzRun.MutateRequested"
+    trigger FuzzRun::ProcessPool::Probe, with: { run: :run }
+  end
+
   policy "MeasureWhenRequested" do
     on      "FuzzRun.BenchRequested"
     trigger FuzzRun::RustToolchain::Measure, with: { run: :run }

data/lib/hecks/hecks/hecks.hecksagon

@@ -188,6 +188,12 @@ Hecks.hecksagon "Hecks" do
       answers "FuzzAnswered"
       refuses "FuzzRefused"
     end
+
+    asks "Probe", to: FuzzRun do
+      attribute :run, FuzzKey
+      answers "FuzzAnswered"
+      refuses "FuzzRefused"
+    end
   end
 
   Hecks::FuzzRun.port "RustToolchain" do

data/lib/hecks/language/bluebook/vocabulary.bluebook

@@ -626,7 +626,7 @@ Hecks.bluebook "Bluebook" do
              label: "does this change touch lib/hecks/runtime/**?"
       member name: "postgres_io_relevant_changed", workflow: "ci-postgres-io-parallel.yml", mode: "skips_unless",
              pattern: '^(docs/|editors/|release/|deploy/|\.claude/|\.githooks/|rust/(parser|codegen|host|build|lsp|web|tests)/|\.rubocop\.yml$|\.rubocop_todo\.yml$|\.mcp\.json$|\.rspec-local\.example$|README\.md$|CHANGELOG\.md$|CONTRIBUTING\.md$|SECURITY\.md$|LICENSE$|\.gitignore$)',
-             push: "before_sha",
+             push: "skip",
              label: "does this change touch anything rspec_postgres_io_parallel covers?"
     end
 
@@ -706,6 +706,8 @@ Hecks.bluebook "Bluebook" do
              meaning: "Fraction (0.0..1.0) of seeds that start from a prefix of an earlier seed's new coverage. Off by default: measured, it trades breadth for depth (banking reached 1.41x the deep-state tuples, chess lost 0.12x of all tuples), while favoring rare verbs alone never lost."
       member name: "favor_rare_verbs", type: "Integer",
              meaning: "How many of the least-hit verbs the next seed weights up; 0 turns the lever off."
+      member name: "runtime_coverage_feedback", type: "Boolean",
+             meaning: "Whether guided generation also admits a seed to the corpus for reaching a new line or branch of the Ruby runtime (`Hecks::Fuzzing::RuntimeCoverage`), not only a new (verb, kind, state, mutation, outcome) tuple. Off by default: tracing every line slows each seed."
       member name: "self_consistency_checks", type: "Boolean",
              meaning: "Whether `hecks quality_control query sweep.run` also runs the self-consistency pass (cold rehydration, replay idempotency, value-object JSON round trip) on the same generated sequence."
       member name: "shrink_budget", type: "Integer",
@@ -802,6 +804,7 @@ Hecks.bluebook "Bluebook" do
       member script: "fuzz", section: "Custodian", aggregate: "FuzzRun", verb: "fuzz"
       member script: "generate", section: "Custodian", aggregate: "FuzzRun", verb: "generate_sequence"
       member script: "bench", section: "Custodian", aggregate: "FuzzRun", verb: "bench"
+      member script: "(new)", section: "Custodian", aggregate: "FuzzRun", verb: "mutate"
       member script: "(new)", section: "Custodian", aggregate: "Era", verb: "hold_first"
       member script: "project_deploy", section: "Deploy", aggregate: "Recipe", verb: "project"
       member script: "lint_deploy_recipes", section: "Deploy", aggregate: "MakefileCheck", verb: "lint"

data/lib/hecks/projections/deploy/box/roll_scripts.rb

@@ -44,7 +44,8 @@ module Hecks
             template("deploy-box.sh.tmpl", "STACK" => plan.infra_name, "BOX_STACK" => plan.box_stack,
                                            "RDS_STACK" => plan.rds_stack, "REGION" => region,
                                            "DIR" => "/opt/#{plan.infra_name}", "HEALTH_CHECKS" => health_checks(plan),
-                                           "SMOKE_HEADER" => smoke_header(plan), "USAGE" => deploy_usage(plan))
+                                           "SMOKE_HEADER" => smoke_header(plan), "USAGE" => deploy_usage(plan),
+                                           "DB_SECRET_LINE" => database_secret_line(plan))
           end
 
           # @param plan [Settings::Plan] the resolved settings
@@ -96,9 +97,10 @@ module Hecks
           # @return [String] a Makefile whose targets create the two stacks and roll the box
           def makefile(plan)
             hint = plan.task_definition ? "[TASKDEF=family:revision]" : '[TAGS="web=20260101 worker=20260101"]'
-            base = template("Makefile.tmpl", "STACK" => plan.infra_name, "DEPLOY_HINT" => hint,
-                                             "RDS_STACK" => plan.rds_stack, "BOX_STACK" => plan.box_stack,
-                                             "DEPLOY_RECIPE" => RollRecipe.deploy(plan))
+            base = template("Makefile.tmpl", { "STACK" => plan.infra_name, "DEPLOY_HINT" => hint,
+                                               "RDS_STACK" => plan.rds_stack, "BOX_STACK" => plan.box_stack,
+                                               "DEPLOY_RECIPE" => RollRecipe.deploy(plan) }
+                                               .merge(database_makefile_values(plan)))
             plan.hosting ? "#{base}\ninclude hosting.mk\n" : base
           end
 

data/lib/hecks/projections/deploy/box/settings/patterns.rb

@@ -23,6 +23,10 @@ module Hecks
             S3_BUCKET   = /\A[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]\z/
             S3_SHAPE    = "s3_access: a list of `{ bucket: \"name\", write: true }` hashes (write is optional)".freeze
             FROM_TASKDEF = %i[env secrets repository].freeze
+
+            # The settings that size an RDS instance: with `shared_database` they are the shared
+            # instance's.
+            SHARED_INSTANCE_SETTINGS = %i[database_class storage_gb engine_version backup_days].freeze
             # Default images, each a version tag plus the digest of its multi-architecture index,
             # so a rebuilt box pulls the same bytes.
             TUNNEL_IMAGE = "cloudflare/cloudflared:2026.9.3" \

data/lib/hecks/projections/deploy/box/settings/types.rb

@@ -51,10 +51,14 @@ module Hecks
               :storage_gb, :backup_days, :snapshots_keep, :database_name, :engine_version,
               :containers, :routes, :default_container, :origin_header, :origin_secret,
               :secret_prefixes, :writable_secrets, :origin_env, :tunnel, :tunnel_service, :proxy_image,
-              :task_definition, :migration, :s3_buckets, :hosting, keyword_init: true
+              :task_definition, :migration, :s3_buckets, :hosting, :shared_database, keyword_init: true
             ) do
-              # @return [String] the CloudFormation stack that holds the database
-              def rds_stack = "#{stack_prefix}-#{infra_name}-rds"
+              # @return [String] the CloudFormation stack that holds the database: the shared
+              #   instance's when the world names one, else this site's own
+              def rds_stack = shared_database || "#{stack_prefix}-#{infra_name}-rds"
+
+              # @return [Boolean] whether the database lives on an instance shared with other sites
+              def shared? = !shared_database.nil?
 
               # @return [String] the CloudFormation stack that holds the box
               def box_stack = "#{stack_prefix}-#{infra_name}-box"

data/lib/hecks/projections/deploy/box/settings.rb

@@ -47,6 +47,7 @@ module Hecks
           def extra_fields(settings, task_definition, database_name)
             {
               task_definition: task_definition, migration: read_migration(settings[:migration], database_name),
+              shared_database: read_shared_database(settings),
               s3_buckets: read_s3_access(settings.fetch(:s3_access, [])),
               hosting: HostingSettings.read(settings, task_definition)
             }
@@ -129,6 +130,23 @@ module Hecks
             end.uniq(&:name)
           end
 
+          # The stack of the RDS instance this site shares with others. The instance's size,
+          # storage, engine and backups are the instance's, so a world that sets them is refused.
+          #
+          # @param settings [Hash{Symbol => Object}] the world's `AwsBox` settings
+          # @return [String, nil] the shared stack's name, or nil for a database of its own
+          # @raise [ArgumentError] when the name is malformed or the world also sizes the instance
+          def read_shared_database(settings)
+            return nil unless settings.key?(:shared_database)
+
+            clash = SHARED_INSTANCE_SETTINGS & settings.keys
+            unless clash.empty?
+              raise ArgumentError, "shared_database: #{clash.join(", ")} size the instance, which belongs to the shared " \
+                                   "stack #{settings[:shared_database]}; drop #{clash.size == 1 ? "it" : "them"}"
+            end
+            check(:shared_database, settings[:shared_database], NAME)
+          end
+
           def read_database_name(settings, infra_name)
             check(:database_name, settings.fetch(:database_name, infra_name.gsub(/[^a-zA-Z0-9]/, "")), DB_NAME)
           end

data/lib/hecks/projections/deploy/box/shared_database.rb

@@ -0,0 +1,69 @@
+module Hecks
+  module Projections
+    module Deploy
+      module Box
+        # The parts of a box deploy that change when its database is one of several on a shared RDS
+        # instance, each client in its own database with its own login role (ADR 0092). Mixed into
+        # `Box`. Everything here returns the dedicated-instance text unchanged when the plan has no
+        # `shared_database`, so a site with its own instance generates the same files as before.
+        module SharedDatabase
+          # @param plan [Settings::Plan] the resolved settings
+          # @return [Hash{String => String}] the Makefile markers that depend on where the database
+          #   lives
+          def database_makefile_values(plan)
+            return dedicated_makefile_values unless plan.shared?
+
+            { "WHAT" => "one app box and a database on the shared instance #{plan.rds_stack}",
+              "REHEARSAL_NOTE" => "# true gives the box no Elastic IP. It never deletes the shared instance\n" \
+                                  "# or this site's database.\n",
+              "RDS_DEPLOY" => "", "DB_SECRET_PARAM" => "\t\tDbSecretArn=$$(aws secretsmanager describe-secret --secret-id " \
+                                                       "#{plan.infra_name}/database --query ARN --output text) \\",
+              "PHONY" => ".PHONY: stacks deploy provision", "PROVISION" => provision_target(plan) }
+          end
+
+          # @param plan [Settings::Plan] the resolved settings
+          # @return [String] the line of `deploy-box.sh` that finds the secret the containers read
+          def database_secret_line(plan)
+            return 'DB_SECRET=$(out "$RDS_STACK" DbSecretArn)' unless plan.shared?
+
+            "DB_SECRET=$(aws secretsmanager describe-secret --secret-id #{plan.infra_name}/database --query ARN --output text)"
+          end
+
+          # @param plan [Settings::Plan] the resolved settings
+          # @return [String] `provision-database.sh`, which creates the site's database, role and
+          #   secret
+          def provision_database_sh(plan)
+            template("provision-database.sh.tmpl", "STACK" => plan.infra_name, "RDS_STACK" => plan.rds_stack,
+                                                   "DATABASE" => plan.database_name)
+          end
+
+          private
+
+          def dedicated_makefile_values
+            { "WHAT" => "one app box and one RDS instance",
+              "REHEARSAL_NOTE" => "# true makes a throwaway pair: the database is deleted with its stack and the box has no\n" \
+                                  "# Elastic IP. The default is a production pair, with deletion protection.\n",
+              "RDS_DEPLOY" => rds_deploy_lines, "DB_SECRET_PARAM" => dedicated_secret_param,
+              "PHONY" => ".PHONY: stacks deploy", "PROVISION" => "" }
+          end
+
+          def rds_deploy_lines
+            "\taws cloudformation deploy --template-file rds.yaml --stack-name $(RDS_STACK) --capabilities CAPABILITY_IAM \\\n" \
+              "\t\t--parameter-overrides VpcId=$(VPC) PrivateSubnetIds=$(PRIVATE_SUBNETS) Rehearsal=$(REHEARSAL)\n"
+          end
+
+          def dedicated_secret_param
+            "\t\tDbSecretArn=$$(aws cloudformation describe-stacks --stack-name $(RDS_STACK) " \
+              "--query \"Stacks[0].Outputs[?OutputKey=='DbSecretArn'].OutputValue\" --output text) \\"
+          end
+
+          def provision_target(plan)
+            "# Creates #{plan.infra_name}'s database, role and secret on #{plan.rds_stack}. Run it once; run it again with\n" \
+              "# ROTATE=--rotate to change the role's password.  make provision BASTION=i-... [ROTATE=--rotate]\n" \
+              "provision:\n\tbash ./provision-database.sh $(BASTION) $(ROTATE)\n\n"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hecks/projections/deploy/box/templates/Makefile.tmpl

@@ -1,24 +1,23 @@
-# @@STACK@@: one app box and one RDS instance.
+# @@STACK@@: @@WHAT@@.
 #   make stacks VPC=vpc-... PRIVATE_SUBNETS=subnet-a,subnet-b PUBLIC_SUBNET=subnet-c [REHEARSAL=true]
 #   make deploy @@DEPLOY_HINT@@
 RDS_STACK = @@RDS_STACK@@
 BOX_STACK = @@BOX_STACK@@
-# true makes a throwaway pair: the database is deleted with its stack and the box has no
-# Elastic IP. The default is a production pair, with deletion protection.
+@@REHEARSAL_NOTE@@
 REHEARSAL ?= false
 
 HECKS ?= hecks
 
-.PHONY: stacks deploy
+@@PHONY@@
 stacks:
-	aws cloudformation deploy --template-file rds.yaml --stack-name $(RDS_STACK) --capabilities CAPABILITY_IAM \
-		--parameter-overrides VpcId=$(VPC) PrivateSubnetIds=$(PRIVATE_SUBNETS) Rehearsal=$(REHEARSAL)
+@@RDS_DEPLOY@@
 	aws cloudformation deploy --template-file box.yaml --stack-name $(BOX_STACK) --capabilities CAPABILITY_IAM \
 		--parameter-overrides VpcId=$(VPC) SubnetId=$(PUBLIC_SUBNET) \
 		DbSecurityGroupId=$$(aws cloudformation describe-stacks --stack-name $(RDS_STACK) --query "Stacks[0].Outputs[?OutputKey=='DbSecurityGroupId'].OutputValue" --output text) \
-		DbSecretArn=$$(aws cloudformation describe-stacks --stack-name $(RDS_STACK) --query "Stacks[0].Outputs[?OutputKey=='DbSecretArn'].OutputValue" --output text) \
+@@DB_SECRET_PARAM@@
 		AlertTopicArn=$$(aws cloudformation describe-stacks --stack-name $(RDS_STACK) --query "Stacks[0].Outputs[?OutputKey=='AlertTopicArn'].OutputValue" --output text) \
 		Rehearsal=$(REHEARSAL)
 
+@@PROVISION@@
 deploy:
 @@DEPLOY_RECIPE@@

data/lib/hecks/projections/deploy/box/templates/deploy-box.sh.tmpl

@@ -26,7 +26,7 @@ out() { aws cloudformation describe-stacks --stack-name "$1" --query "Stacks[0].
 
 BOX=$(out "$BOX_STACK" InstanceId)
 DB_HOST=$(out "$RDS_STACK" DbEndpoint)
-DB_SECRET=$(out "$RDS_STACK" DbSecretArn)
+@@DB_SECRET_LINE@@
 [ -n "$BOX" ] && [ "$BOX" != None ] || { echo "no instance in stack $BOX_STACK" >&2; exit 40; }
 ACCOUNT=$(aws sts get-caller-identity --query Account --output text)
 

data/lib/hecks/projections/deploy/box/templates/provision-database.sh.tmpl

@@ -0,0 +1,97 @@
+#!/bin/bash
+# Create @@STACK@@'s database and login role on the shared RDS instance (stack @@RDS_STACK@@), and the
+# secret @@STACK@@/database that the box's containers read. Run once for a new site, from a machine with
+# AWS credentials that may read the instance's master secret and start SSM sessions, through a bastion
+# that can reach the instance.
+#
+#   provision-database.sh <bastion-instance-id> [--rotate]
+#
+# The role owns only the database @@DATABASE@@, is not a superuser, and no other role may connect to that
+# database. The secret holds {username, password, host, port, dbname}; the box's role reads that secret
+# and never the instance's master secret.
+#
+# Safe to run again: a role and a database that exist are left alone, and a secret that exists keeps
+# its password. --rotate gives the role a new password and writes it to the secret; roll the box after it.
+#
+# Ends 70 (the shared stack has no instance).
+#
+# Needs psql, jq, openssl, the SSM plugin and AWS credentials.
+set -euo pipefail
+
+BASTION=${1:?bastion instance id}
+ROTATE=${2:-}
+[ -z "$ROTATE" ] || [ "$ROTATE" = --rotate ] || { echo "usage: provision-database.sh <bastion-instance-id> [--rotate]" >&2; exit 2; }
+RDS_STACK=@@RDS_STACK@@
+DB=@@DATABASE@@
+ROLE=@@DATABASE@@
+SECRET=@@STACK@@/database
+PORT=15434
+out() { aws cloudformation describe-stacks --stack-name "$1" --query "Stacks[0].Outputs[?OutputKey==\`$2\`].OutputValue" --output text; }
+
+HOST=$(out "$RDS_STACK" DbEndpoint)
+MASTER=$(out "$RDS_STACK" DbSecretArn)
+[ -n "$HOST" ] && [ "$HOST" != None ] || { echo "no instance in stack $RDS_STACK" >&2; exit 70; }
+
+WORK=$(mktemp -d)
+PIDS=()
+cleanup() { for p in "${PIDS[@]:-}"; do [ -n "$p" ] && kill "$p" 2>/dev/null || true; done; rm -rf "$WORK"; }
+trap cleanup EXIT
+
+master_field() { aws secretsmanager get-secret-value --secret-id "$MASTER" --query SecretString --output text | jq -r "$1"; }
+MASTER_USER=$(master_field '.username // "postgres"')
+MASTER_PW=$(master_field .password)
+
+aws ssm start-session --target "$BASTION" --document-name AWS-StartPortForwardingSessionToRemoteHost \
+  --parameters "{\"host\":[\"$HOST\"],\"portNumber\":[\"5432\"],\"localPortNumber\":[\"$PORT\"]}" >"$WORK/tunnel.log" 2>&1 &
+PIDS+=($!)
+disown
+for _ in $(seq 1 20); do grep -q "Waiting for connections" "$WORK/tunnel.log" 2>/dev/null && break; sleep 1; done
+
+# One statement, one answer. The master connects to the maintenance database.
+M() { PGPASSWORD=$MASTER_PW psql -h localhost -p $PORT -U "$MASTER_USER" -d postgres -v ON_ERROR_STOP=1 -Atc "$1"; }
+# A script on stdin, with the new password passed as a variable so it is never in a command line.
+MS() { PGPASSWORD=$MASTER_PW psql -h localhost -p $PORT -U "$MASTER_USER" -d postgres -v ON_ERROR_STOP=1 -v pw="$PW" -q -f -; }
+
+HAVE_ROLE=$(M "select count(*) from pg_roles where rolname='$ROLE'")
+HAVE_SECRET=no
+aws secretsmanager describe-secret --secret-id "$SECRET" >/dev/null 2>&1 && HAVE_SECRET=yes
+
+# Letters and digits only: the host puts the password in a URL without percent-encoding it.
+PW=""
+if [ "$HAVE_ROLE" = 0 ]; then
+  PW=$(openssl rand -hex 16)
+  MS <<SQL
+create role "$ROLE" login password :'pw' nosuperuser nocreatedb nocreaterole noinherit;
+SQL
+  echo "created role $ROLE"
+elif [ "$ROTATE" = --rotate ] || [ "$HAVE_SECRET" = no ]; then
+  PW=$(openssl rand -hex 16)
+  MS <<SQL
+alter role "$ROLE" password :'pw';
+SQL
+  echo "set a new password for role $ROLE"
+else
+  echo "role $ROLE exists and so does $SECRET; kept"
+fi
+
+# RDS's master is not a true superuser: it may create a database for a role only as a member of that role.
+if [ "$(M "select count(*) from pg_database where datname='$DB'")" = 0 ]; then
+  M "grant \"$ROLE\" to \"$MASTER_USER\"" >/dev/null
+  M "create database \"$DB\" owner \"$ROLE\"" >/dev/null
+  echo "created database $DB"
+else
+  echo "database $DB exists"
+fi
+# Nobody but the owner (and the master) may connect to it.
+M "revoke all on database \"$DB\" from public" >/dev/null
+
+if [ -n "$PW" ]; then
+  BODY=$(jq -n --arg u "$ROLE" --arg p "$PW" --arg h "$HOST" --arg d "$DB" '{username: $u, password: $p, host: $h, port: 5432, dbname: $d}')
+  if [ "$HAVE_SECRET" = yes ]; then
+    aws secretsmanager put-secret-value --secret-id "$SECRET" --secret-string "$BODY" >/dev/null
+  else
+    aws secretsmanager create-secret --name "$SECRET" --description "@@STACK@@'s database login on @@RDS_STACK@@" --secret-string "$BODY" >/dev/null
+  fi
+  echo "wrote secret $SECRET"
+fi
+echo "ok   $DB on $HOST, role $ROLE, secret $SECRET"

data/lib/hecks/projections/deploy/box/templates/restore-to-rds.sh.tmpl

@@ -7,6 +7,8 @@
 #
 #   FORCE=1   drop the target schemas first: a re-load before cutover, or a rollback copy in the other
 #             direction (swap the hosts and secrets, and SRC_DB and DST_DB)
+#   SCHEMAS   the schemas to copy, space separated (default @@SCHEMAS@@); another database's schemas, such as
+#             an analytics database's `public`, are copied by naming them here with SRC_DB and DST_DB
 #   SRC_DB    the source database (default @@SOURCE_DATABASE@@)
 #   DST_DB    the target database (default @@DATABASE@@)
 #   VERIFY_BY_COMMAND=1   end after the copy and leave the comparison to `hecks deploy data_copy.restore`
@@ -28,7 +30,7 @@ HERE=$(cd "$(dirname "$0")" && pwd)
 
 BASTION=${1:?bastion instance id}; SRC_HOST=${2:?source host}; SRC_SECRET=${3:?source secret arn}
 DST_HOST=${4:?target host}; DST_SECRET=${5:?target secret arn}
-SCHEMAS="@@SCHEMAS@@"
+SCHEMAS=${SCHEMAS:-"@@SCHEMAS@@"}
 SRC_DB=${SRC_DB:-@@SOURCE_DATABASE@@}; DST_DB=${DST_DB:-@@DATABASE@@}
 SRC_PORT=15432; DST_PORT=15433
 
@@ -93,4 +95,4 @@ done
 
 # Same structure, same exact row counts in every table, nothing unpopulated.
 [ -z "${VERIFY_BY_COMMAND:-}" ] || exit 0
-A_DB=$SRC_DB B_DB=$DST_DB bash "$HERE/verify-copy.sh" "$BASTION" "$SRC_HOST" "$SRC_SECRET" "$DST_HOST" "$DST_SECRET"
+A_DB=$SRC_DB B_DB=$DST_DB COMPARE_SCHEMAS=$(echo "$SCHEMAS" | tr ' ' ',') bash "$HERE/verify-copy.sh" "$BASTION" "$SRC_HOST" "$SRC_SECRET" "$DST_HOST" "$DST_SECRET"

data/lib/hecks/projections/deploy/box/templates/verify-copy.sh.tmpl

@@ -6,13 +6,14 @@
 #   verify-copy.sh <bastion-instance-id> <host-a> <secret-arn-a> <host-b> <secret-arn-b>
 #
 #   A_DB, B_DB   the database on each side (default @@DATABASE@@)
+#   COMPARE_SCHEMAS   the schemas to compare, comma separated (default @@SCHEMAS_CSV@@)
 #
 # Schemas compared: @@SCHEMAS@@
 # Exit 0 only if both are identical and no materialized view is unpopulated in either; 50 when they differ.
 set -euo pipefail
 
 BASTION=${1:?bastion instance id}; A_HOST=${2:?}; A_SECRET=${3:?}; B_HOST=${4:?}; B_SECRET=${5:?}
-SCHEMAS="@@SCHEMAS_CSV@@"
+SCHEMAS=${COMPARE_SCHEMAS:-"@@SCHEMAS_CSV@@"}
 A_DB=${A_DB:-@@DATABASE@@}; B_DB=${B_DB:-@@DATABASE@@}
 A_PORT=15434; B_PORT=15435
 

data/lib/hecks/projections/deploy/box.rb

@@ -7,6 +7,7 @@ require_relative "box/caddy"
 require_relative "box/access"
 require_relative "box/stack_yaml"
 require_relative "box/compose"
+require_relative "box/shared_database"
 require_relative "box/roll_scripts"
 
 module Hecks
@@ -22,6 +23,7 @@ module Hecks
         extend Access
         extend StackYaml
         extend Compose
+        extend SharedDatabase
         extend RollScripts
 
         projects_as :aws_box, needs_world: true, emits: :files
@@ -129,13 +131,23 @@ module Hecks
         # @return [Hash{String => String}] every generated file
         def render_all(plan, region)
           {
-            "rds.yaml" => rds_yaml(plan), "box.yaml" => box_yaml(plan, region),
+            "box.yaml" => box_yaml(plan, region),
             "Caddyfile" => caddyfile(plan), "services.json" => services_json(plan),
             "render-compose.sh" => render_compose_sh(plan, region),
             "fetch-secrets.sh" => File.read(File.join(TEMPLATE_DIR, "fetch-secrets.sh")),
             "deploy-box.sh" => deploy_box_sh(plan, region),
             "Makefile" => makefile(plan)
-          }.merge(migration_files(plan)).then { |files| Hosting.extend_files(files, plan: plan, region: region) }
+          }.merge(database_files(plan)).merge(migration_files(plan))
+            .then { |files| Hosting.extend_files(files, plan: plan, region: region) }
+        end
+
+        # @param plan [Settings::Plan] the resolved settings
+        # @return [Hash{String => String}] the database stack, or for a database on a shared
+        #   instance the script that provisions it; the shared stack is not this site's to generate
+        def database_files(plan)
+          return { "rds.yaml" => rds_yaml(plan) } unless plan.shared?
+
+          { "provision-database.sh" => provision_database_sh(plan) }
         end
 
         # Fills `@@NAME@@` markers. A marker alone on its line is replaced together with the line,