hecks 3.1.0 → 3.1.2

data/lib/hecks/projections/deploy/box/templates/restore-to-rds.sh.tmpl

@@ -0,0 +1,91 @@
+#!/bin/bash
+# Copy @@STACK@@'s schemas from the old database into the new RDS instance through a bastion that can
+# reach both, then verify the copy. Piped dump | restore, so nothing is written to disk. Never writes to
+# the source.
+#
+#   restore-to-rds.sh <bastion-instance-id> <source-host> <source-secret-arn> <rds-host> <rds-secret-arn>
+#
+#   FORCE=1   drop the target schemas first: a re-load before cutover, or a rollback copy in the other
+#             direction (swap the hosts and secrets, and SRC_DB and DST_DB)
+#   SRC_DB    the source database (default @@SOURCE_DATABASE@@)
+#   DST_DB    the target database (default @@DATABASE@@)
+#
+# Schemas copied: @@SCHEMAS@@
+#
+# Why not a plain pg_restore: a Hecks schema's materialized views call hecks_tr_extract() unqualified, and
+# pg_restore runs with an empty search_path, so the restore's REFRESH step errors on them. This restores
+# tolerating exactly that error, then refreshes the views with the schema on the search_path, and fails on
+# any other error.
+#
+# Needs pg_dump, pg_restore and psql 16 or newer, the SSM plugin, jq and AWS credentials that can read both
+# secrets and start SSM sessions.
+set -euo pipefail
+HERE=$(cd "$(dirname "$0")" && pwd)
+
+BASTION=${1:?bastion instance id}; SRC_HOST=${2:?source host}; SRC_SECRET=${3:?source secret arn}
+DST_HOST=${4:?target host}; DST_SECRET=${5:?target secret arn}
+SCHEMAS="@@SCHEMAS@@"
+SRC_DB=${SRC_DB:-@@SOURCE_DATABASE@@}; DST_DB=${DST_DB:-@@DATABASE@@}
+SRC_PORT=15432; DST_PORT=15433
+
+pg_major() { "$1" --version 2>/dev/null | sed -E 's/.* ([0-9]+)[.].*/\1/'; }
+# The default client on the PATH is often older than the server (Homebrew's is 14); prefer postgresql@16.
+if [ "$(pg_major pg_dump || echo 0)" -lt 16 ] && [ -x /opt/homebrew/opt/postgresql@16/bin/pg_dump ]; then
+  export PATH=/opt/homebrew/opt/postgresql@16/bin:$PATH
+fi
+for t in pg_dump pg_restore psql; do
+  [ "$(pg_major $t || echo 0)" -ge 16 ] || { echo "need $t 16 or newer on the PATH" >&2; exit 1; }
+done
+
+WORK=$(mktemp -d)
+PIDS=()
+cleanup() { for p in "${PIDS[@]:-}"; do [ -n "$p" ] && kill "$p" 2>/dev/null || true; done; rm -rf "$WORK"; }
+trap cleanup EXIT
+
+secret_field() { aws secretsmanager get-secret-value --secret-id "$1" --query SecretString --output text | jq -r "$2"; }
+SRC_PW=$(secret_field "$SRC_SECRET" .password); DST_PW=$(secret_field "$DST_SECRET" .password)
+SRC_USER=$(secret_field "$SRC_SECRET" '.username // "postgres"'); DST_USER=$(secret_field "$DST_SECRET" '.username // "postgres"')
+
+tunnel() { # host local-port
+  aws ssm start-session --target "$BASTION" --document-name AWS-StartPortForwardingSessionToRemoteHost \
+    --parameters "{\"host\":[\"$1\"],\"portNumber\":[\"5432\"],\"localPortNumber\":[\"$2\"]}" >"$WORK/tunnel_$2.log" 2>&1 &
+  PIDS+=($!)
+  disown # so shutting the tunnel down at exit does not print a "Terminated" line
+}
+tunnel "$SRC_HOST" $SRC_PORT; tunnel "$DST_HOST" $DST_PORT
+for i in $(seq 1 20); do
+  grep -q "Waiting for connections" "$WORK/tunnel_$SRC_PORT.log" 2>/dev/null && grep -q "Waiting for connections" "$WORK/tunnel_$DST_PORT.log" 2>/dev/null && break
+  sleep 1
+done
+
+SRC() { PGPASSWORD=$SRC_PW psql -h localhost -p $SRC_PORT -U "$SRC_USER" -d "$SRC_DB" -Atc "$1"; }
+DST() { PGPASSWORD=$DST_PW psql -h localhost -p $DST_PORT -U "$DST_USER" -d "$DST_DB" -Atc "$1"; }
+echo "source $(SRC 'show server_version'), target $(DST 'show server_version')"
+
+# Refuse to clobber a target that already holds the schemas.
+for S in $SCHEMAS; do
+  if [ "$(DST "select count(*) from pg_namespace where nspname='$S'")" != 0 ]; then
+    if [ "${FORCE:-}" = 1 ]; then DST "drop schema \"$S\" cascade" >/dev/null; echo "dropped target schema $S"
+    else echo "target already has schema $S; re-run with FORCE=1 to replace it" >&2; exit 1; fi
+  fi
+done
+
+for S in $SCHEMAS; do
+  echo "== copying schema $S"
+  PGPASSWORD=$SRC_PW pg_dump -h localhost -p $SRC_PORT -U "$SRC_USER" -d "$SRC_DB" --schema="$S" -Fc --no-owner --no-privileges \
+    | PGPASSWORD=$DST_PW pg_restore -h localhost -p $DST_PORT -U "$DST_USER" -d "$DST_DB" --no-owner --no-privileges 2>"$WORK/err_$S.txt" || true
+  TOTAL=$(grep -c 'error:' "$WORK/err_$S.txt" || true)
+  UNEXPECTED=$(grep 'error:' "$WORK/err_$S.txt" | grep -vc 'hecks_tr_extract' || true)
+  echo "   restore errors: $TOTAL (unexpected: $UNEXPECTED)"
+  if [ "$UNEXPECTED" != 0 ]; then grep 'error:' "$WORK/err_$S.txt" | grep -v 'hecks_tr_extract' | head -5 >&2; exit 1; fi
+done
+
+# Refresh what pg_restore could not (with the schema on the search_path).
+for S in $SCHEMAS; do
+  for MV in $(DST "select matviewname from pg_matviews where schemaname='$S' and not ispopulated"); do
+    PGPASSWORD=$DST_PW psql -h localhost -p $DST_PORT -U "$DST_USER" -d "$DST_DB" -qAtc "set search_path=\"$S\",public; refresh materialized view \"$S\".\"$MV\""
+  done
+done
+
+# Same structure, same exact row counts in every table, nothing unpopulated.
+A_DB=$SRC_DB B_DB=$DST_DB bash "$HERE/verify-copy.sh" "$BASTION" "$SRC_HOST" "$SRC_SECRET" "$DST_HOST" "$DST_SECRET"

data/lib/hecks/projections/deploy/box/templates/verify-copy.sh.tmpl

@@ -0,0 +1,58 @@
+#!/bin/bash
+# Compare two databases holding @@STACK@@'s schemas (structure and exact row counts of every table)
+# through a bastion that can reach both. Read-only on both. Used at the end of restore-to-rds.sh, for a
+# backup restore drill (production vs a database restored from its backup) and for checking a rollback copy.
+#
+#   verify-copy.sh <bastion-instance-id> <host-a> <secret-arn-a> <host-b> <secret-arn-b>
+#
+#   A_DB, B_DB   the database on each side (default @@DATABASE@@)
+#
+# Schemas compared: @@SCHEMAS@@
+# Exit 0 only if both are identical and no materialized view is unpopulated in either.
+set -euo pipefail
+
+BASTION=${1:?bastion instance id}; A_HOST=${2:?}; A_SECRET=${3:?}; B_HOST=${4:?}; B_SECRET=${5:?}
+SCHEMAS="@@SCHEMAS_CSV@@"
+A_DB=${A_DB:-@@DATABASE@@}; B_DB=${B_DB:-@@DATABASE@@}
+A_PORT=15434; B_PORT=15435
+
+pg_major() { "$1" --version 2>/dev/null | sed -E 's/.* ([0-9]+)[.].*/\1/'; }
+if [ "$(pg_major psql || echo 0)" -lt 16 ] && [ -x /opt/homebrew/opt/postgresql@16/bin/psql ]; then
+  export PATH=/opt/homebrew/opt/postgresql@16/bin:$PATH
+fi
+
+WORK=$(mktemp -d); PIDS=()
+cleanup() { for p in "${PIDS[@]:-}"; do [ -n "$p" ] && kill "$p" 2>/dev/null || true; done; rm -rf "$WORK"; }
+trap cleanup EXIT
+
+secret_field() { aws secretsmanager get-secret-value --secret-id "$1" --query SecretString --output text | jq -r "$2"; }
+A_PW=$(secret_field "$A_SECRET" .password); B_PW=$(secret_field "$B_SECRET" .password)
+A_USER=$(secret_field "$A_SECRET" '.username // "postgres"'); B_USER=$(secret_field "$B_SECRET" '.username // "postgres"')
+tunnel() {
+  aws ssm start-session --target "$BASTION" --document-name AWS-StartPortForwardingSessionToRemoteHost \
+    --parameters "{\"host\":[\"$1\"],\"portNumber\":[\"5432\"],\"localPortNumber\":[\"$2\"]}" >"$WORK/t_$2.log" 2>&1 &
+  PIDS+=($!); disown
+}
+tunnel "$A_HOST" $A_PORT; tunnel "$B_HOST" $B_PORT
+for i in $(seq 1 20); do
+  grep -q "Waiting for connections" "$WORK/t_$A_PORT.log" 2>/dev/null && grep -q "Waiting for connections" "$WORK/t_$B_PORT.log" 2>/dev/null && break
+  sleep 1
+done
+A() { PGPASSWORD=$A_PW psql -h localhost -p $A_PORT -U "$A_USER" -d "$A_DB" -Atc "$1"; }
+B() { PGPASSWORD=$B_PW psql -h localhost -p $B_PORT -U "$B_USER" -d "$B_DB" -Atc "$1"; }
+
+SHAPE="select n.nspname||' '||c.relkind::text, count(*) from pg_class c join pg_namespace n on n.oid=c.relnamespace where n.nspname = any(string_to_array('$SCHEMAS', ',')) group by 1
+ union all select 'policies', count(*) from pg_policies where schemaname = any(string_to_array('$SCHEMAS', ','))
+ union all select 'functions', count(*) from pg_proc p join pg_namespace n on n.oid=p.pronamespace where n.nspname = any(string_to_array('$SCHEMAS', ',')) order by 1"
+COUNTS="select table_schema||'.'||table_name, (xpath('/row/c/text()', query_to_xml(format('select count(*) as c from %I.%I', table_schema, table_name), false, true, '')))[1]::text from information_schema.tables where table_type='BASE TABLE' and table_schema = any(string_to_array('$SCHEMAS', ',')) order by 1"
+UNPOP="select count(*) from pg_matviews where schemaname = any(string_to_array('$SCHEMAS', ',')) and not ispopulated"
+
+BAD=0
+A "$SHAPE" >"$WORK/shape_a"; B "$SHAPE" >"$WORK/shape_b"
+A "$COUNTS" >"$WORK/counts_a"; B "$COUNTS" >"$WORK/counts_b"
+[ -s "$WORK/counts_a" ] || { echo "FAIL: first database has no tables to compare" >&2; BAD=1; }
+[ "$(A "$UNPOP")" = 0 ] && [ "$(B "$UNPOP")" = 0 ] || { echo "FAIL: unpopulated materialized views" >&2; BAD=1; }
+diff "$WORK/shape_a" "$WORK/shape_b" >/dev/null || { echo "FAIL: structure differs" >&2; diff "$WORK/shape_a" "$WORK/shape_b" >&2 || true; BAD=1; }
+diff "$WORK/counts_a" "$WORK/counts_b" >/dev/null || { echo "FAIL: row counts differ" >&2; diff "$WORK/counts_a" "$WORK/counts_b" >&2 || true; BAD=1; }
+[ "$BAD" = 0 ] || exit 1
+echo "OK: $(wc -l <"$WORK/counts_a" | tr -d ' ') tables, identical row counts and structure"

data/lib/hecks/projections/deploy/box.rb

@@ -20,10 +20,18 @@ module Hecks
         # the legacy `compute-1` form.
         LEGACY_COMPUTE_REGION = "us-east-1".freeze
 
+        # The end of every Caddyfile: sites a rehearsal mounts under `caddy-extra`, such as a
+        # loopback listener that adds the origin secret so a smoke test runs without the CDN. A
+        # glob that matches nothing is not an error, so production, which mounts none, is unchanged.
+        CADDY_EXTRA = "# Rehearsal-only sites are mounted here; nothing matches in production. Restart the proxy\n" \
+                      "# after adding one: the admin API is off, so a reload cannot reach it.\n" \
+                      "import /etc/caddy/extra/*\n".freeze
+
         module_function
 
         # Generates `rds.yaml`, `box.yaml`, `Caddyfile`, `services.json`, `render-compose.sh`,
-        # `fetch-secrets.sh`, `deploy-box.sh` and a `Makefile`.
+        # `fetch-secrets.sh`, `deploy-box.sh` and a `Makefile`, and for a world that declares a
+        # `migration`, `restore-to-rds.sh`, `verify-copy.sh` and `MIGRATION.md`.
         #
         # @param bluebook [Bluebook::Behaviour::Chapter] the domain's own booted chapter
         # @param options [Hash] generation options; same shape as `Fargate.call`'s
@@ -82,15 +90,95 @@ module Hecks
           {
             "rds.yaml" => rds_yaml(plan), "box.yaml" => box_yaml(plan, region),
             "Caddyfile" => caddyfile(plan), "services.json" => services_json(plan),
-            "render-compose.sh" => template("render-compose.sh.tmpl", "STACK" => plan.infra_name, "REGION" => region,
-                                                                      "DB_NAME" => plan.database_name,
-                                                                      "PROXY_IMAGE" => plan.proxy_image),
+            "render-compose.sh" => render_compose_sh(plan, region),
             "fetch-secrets.sh" => File.read(File.join(TEMPLATE_DIR, "fetch-secrets.sh")),
             "deploy-box.sh" => deploy_box_sh(plan, region),
             "Makefile" => makefile(plan)
+          }.merge(migration_files(plan))
+        end
+
+        # The tooling that moves a project's data from its old database into the RDS instance, for
+        # a world that declares a `migration`.
+        #
+        # @param plan [Settings::Plan] the resolved settings
+        # @return [Hash{String => String}] `restore-to-rds.sh`, `verify-copy.sh` and `MIGRATION.md`,
+        #   or nothing
+        def migration_files(plan)
+          migration = plan.migration
+          return {} unless migration
+
+          values = { "STACK" => plan.infra_name, "SCHEMAS" => migration.schemas.join(" "),
+                     "SCHEMAS_CSV" => migration.schemas.join(","), "DATABASE" => migration.database,
+                     "SOURCE_DATABASE" => migration.source_database }
+          {
+            "restore-to-rds.sh" => template("restore-to-rds.sh.tmpl", values),
+            "verify-copy.sh"    => template("verify-copy.sh.tmpl", values),
+            "MIGRATION.md"      => migration_md(plan)
           }
         end
 
+        # @param plan [Settings::Plan] the resolved settings
+        # @return [String] the runbook for moving onto the box, in the order the steps are run
+        def migration_md(plan)
+          migration = plan.migration
+          deploy = plan.task_definition ? "make deploy TASKDEF=#{plan.task_definition}:<revision>" : "make deploy"
+          <<~MD
+            # Moving #{plan.infra_name} onto the box and RDS
+
+            Generated from the world's `migration` setting. Nothing here has been run for you.
+
+            Schemas to copy: #{migration.schemas.map { |name| "`#{name}`" }.join(', ')}, in database
+            `#{migration.source_database}` on the old server and `#{migration.database}` on RDS.
+
+            ## Before cutover
+
+            1. Create the stacks: `make stacks VPC=... PRIVATE_SUBNETS=... PUBLIC_SUBNET=...`. To rehearse first,
+               deploy `rds.yaml` and `box.yaml` under other stack names with `Rehearsal=true`: the database is then
+               deleted with its stack and the box gets no stable public address. Delete the rehearsal stacks after.
+            2. Copy the data. The bastion is any instance that can reach both databases.
+
+               ```
+               RDS_HOST=$(aws cloudformation describe-stacks --stack-name #{plan.rds_stack} --query "Stacks[0].Outputs[?OutputKey=='DbEndpoint'].OutputValue" --output text)
+               RDS_SECRET=$(aws cloudformation describe-stacks --stack-name #{plan.rds_stack} --query "Stacks[0].Outputs[?OutputKey=='DbSecretArn'].OutputValue" --output text)
+               bash restore-to-rds.sh <bastion-instance-id> <old-host> <old-secret-arn> "$RDS_HOST" "$RDS_SECRET"
+               ```
+
+               It copies each schema, refreshes the materialized views a plain `pg_restore` cannot, then runs
+               `verify-copy.sh`, which compares structure and the exact row count of every table and prints `OK`
+               only if both match. Re-run with `FORCE=1` to reload.
+            3. Deploy the app onto the box: `#{deploy}`. The deploy ends with health checks on the box.
+            4. Run the project's own smoke test against the box before any traffic moves.
+
+            ## Cutover
+
+            1. Stop writes on the old stack, then run `restore-to-rds.sh` again with `FORCE=1` for a final copy and
+               wait for `OK`.
+            2. Point the CDN's origin at the box stack's `AppOriginDomain` output.
+            3. Keep the old database untouched for several days, and take a final snapshot before deleting it.
+
+            ## Rollback
+
+            Until the first write lands on RDS, point the origin back. After that, writes taken by both databases
+            cannot be merged: choose one side and copy it over the other (swap the hosts and secrets, and set
+            `SRC_DB` and `DST_DB`, with `FORCE=1`). Do not change the domain's era in the same window, so a rollback
+            only has to move data.
+          MD
+        end
+
+        # @param plan [Settings::Plan] the resolved settings
+        # @param region [String] the validated region
+        # @return [String] the script that renders the Compose file, from `services.json` or from
+        #   an ECS task definition when the world names one
+        def render_compose_sh(plan, region)
+          if plan.task_definition
+            template("render-compose-taskdef.sh.tmpl", "STACK" => plan.infra_name, "FAMILY" => plan.task_definition,
+                                                        "PROXY_IMAGE" => plan.proxy_image)
+          else
+            template("render-compose.sh.tmpl", "STACK" => plan.infra_name, "REGION" => region,
+                                               "DB_NAME" => plan.database_name, "PROXY_IMAGE" => plan.proxy_image)
+          end
+        end
+
         # @param plan [Settings::Plan] the resolved settings
         # @return [String] the database stack
         def rds_yaml(plan)
@@ -108,9 +196,42 @@ module Hecks
                    "VOLUME_GB" => plan.volume_gb.to_s, "SNAPSHOTS_KEEP" => plan.snapshots_keep.to_s,
                    "AMI_PARAMETER" => ami_parameter(plan.instance_type),
                    "COMPUTE_DOMAIN" => compute_domain(region),
-                   "SECRET_RESOURCES" => secret_resources(plan), "TUNNEL_EGRESS" => tunnel_egress(plan),
+                   "SECRET_RESOURCES" => secret_resources(plan), "WRITABLE_SECRETS" => writable_secrets(plan),
+                   "TUNNEL_EGRESS" => tunnel_egress(plan),
                    "SWAP_COMMANDS" => swap_commands(plan), "ECR_REPOSITORIES" => ecr_repositories(plan),
-                   "ECR_OUTPUTS" => ecr_outputs(plan))
+                   "ECR_OUTPUTS" => ecr_outputs(plan), "S3_POLICY" => s3_policy(plan))
+        end
+
+        # The role's S3 policy: every declared bucket readable, and writable only on a production
+        # box, so a rehearsal never changes the real objects.
+        #
+        # @param plan [Settings::Plan] the resolved settings
+        # @return [String] a policy list item, indented into the role's `Policies`, or nothing
+        def s3_policy(plan)
+          return "" if plan.s3_buckets.empty?
+
+          arn = ->(path) { "arn:${AWS::Partition}:s3:::#{path}" }
+          reads = plan.s3_buckets.flat_map { |b| [arn.call(b.name), arn.call("#{b.name}/*")] }
+          writes = plan.s3_buckets.select(&:write).map { |b| arn.call("#{b.name}/*") }
+          rows = s3_read_rows(reads)
+          rows += s3_write_rows(writes) unless writes.empty?
+          "#{rows.map { |row| "        #{row}" }.join("\n")}\n"
+        end
+
+        # @param resources [Array<String>] the bucket and object ARNs a box may read
+        # @return [Array<String>] the policy's head and its read statement, relative to `Policies`
+        def s3_read_rows(resources)
+          ["- PolicyName: s3-access", "  PolicyDocument:", "    Version: \"2012-10-17\"", "    Statement:",
+           "      - Effect: Allow", "        Action: [s3:GetObject, s3:ListBucket]", "        Resource:"] +
+            resources.map { |r| "          - !Sub \"#{r}\"" }
+        end
+
+        # @param resources [Array<String>] the object ARNs a production box may write
+        # @return [Array<String>] the production-only write statement, relative to `Policies`
+        def s3_write_rows(resources)
+          ["      - !If", "        - IsProduction", "        - Effect: Allow",
+           "          Action: [s3:PutObject, s3:DeleteObject]", "          Resource:"] +
+            resources.map { |r| "            - !Sub \"#{r}\"" } + ["        - !Ref AWS::NoValue"]
         end
 
         # Graviton families end their generation digit with `g` (`t4g`, `m7gd`, `c6gn`).
@@ -141,6 +262,28 @@ module Hecks
           end.join.chomp
         end
 
+        # The statement that lets a production box overwrite the secrets named as writable.
+        #
+        # @param plan [Settings::Plan] the resolved settings
+        # @return [String] a policy statement for the role, or nothing when none are declared
+        def writable_secrets(plan)
+          return "" if plan.writable_secrets.empty?
+
+          arns = plan.writable_secrets.map { |name| name.end_with?("*") ? name : "#{name}-*" }.uniq
+          resources = arns.map do |pattern|
+            "#{' ' * 20}- !Sub \"arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:#{pattern}\"\n"
+          end.join
+          <<~YAML.chomp
+            #{' ' * 14}- !If
+            #{' ' * 16}- IsProduction
+            #{' ' * 16}- Effect: Allow
+            #{' ' * 18}Action: secretsmanager:PutSecretValue
+            #{' ' * 18}Resource:
+            #{resources.chomp}
+            #{' ' * 16}- !Ref AWS::NoValue
+          YAML
+        end
+
         # @param plan [Settings::Plan] the resolved settings
         # @return [String] the outbound rules a Cloudflare Tunnel sidecar needs, or nothing
         def tunnel_egress(plan)
@@ -174,11 +317,14 @@ module Hecks
           SH
         end
 
-        # One ECR repository per container, keeping the newest 30 images.
+        # One ECR repository per container, keeping the newest 30 images. A task definition names
+        # images that already have repositories, so none are made.
         #
         # @param plan [Settings::Plan] the resolved settings
         # @return [String] CloudFormation resources, each preceded by a blank line
         def ecr_repositories(plan)
+          return "" if plan.task_definition
+
           plan.containers.map do |container|
             id = "#{logical(container.name)}Repository"
             <<~YAML.lines.map { |line| line.strip.empty? ? line : "  #{line}" }.join
@@ -198,6 +344,8 @@ module Hecks
         # @param plan [Settings::Plan] the resolved settings
         # @return [String] one repository URI output per container
         def ecr_outputs(plan)
+          return "" if plan.task_definition
+
           plan.containers.map do |container|
             "  #{logical(container.name)}RepositoryUri:\n    Value: !GetAtt #{logical(container.name)}Repository.RepositoryUri\n"
           end.join.chomp
@@ -223,7 +371,7 @@ module Hecks
             else
               indent(route_blocks(plan))
             end
-          "#{caddy_header(plan)}#{caddy_global(guarded)}\n:80 {\n#{site}}\n"
+          "#{caddy_header(plan)}#{caddy_global(guarded)}\n:80 {\n#{site}}\n\n#{CADDY_EXTRA}"
         end
 
         # @param plan [Settings::Plan] the resolved settings
@@ -256,9 +404,12 @@ module Hecks
         # @param guarded [Boolean] whether an origin secret guards the site
         # @return [String] the global options block
         def caddy_global(guarded)
-          return "{\n\tauto_https off\n\tadmin off\n}\n" unless guarded
+          options = "\t# No certificate for the :80 site. disable_redirects, not off, so a listener a rehearsal\n" \
+                    "\t# mounts under caddy-extra can still ask for `tls internal`.\n" \
+                    "\tauto_https disable_redirects\n\tadmin off\n"
+          return "{\n#{options}}\n" unless guarded
 
-          "{\n\tauto_https off\n\tadmin off\n\tservers {\n\t\ttrusted_proxies static 0.0.0.0/0 ::/0\n\t}\n}\n"
+          "{\n#{options}\tservers {\n\t\ttrusted_proxies static 0.0.0.0/0 ::/0\n\t}\n}\n"
         end
 
         # @param text [String] lines to indent one level with a tab
@@ -272,18 +423,30 @@ module Hecks
         # @param plan [Settings::Plan] the resolved settings
         # @return [String] `services.json`
         def services_json(plan)
-          services = plan.containers.to_h do |c|
-            [c.name, { "name" => c.name, "repository" => c.repository, "port" => c.port,
-                       "env" => c.env, "secrets" => c.secrets }]
-          end
+          services = plan.containers.to_h { |c| [c.name, service_entry(plan, c)] }
           origin = plan.origin_secret ? { "header" => plan.origin_header, "secret" => plan.origin_secret } : nil
           document = { "services" => services, "origin" => origin }
+          document["task_definition"] = plan.task_definition if plan.task_definition
           document["tunnel"] = tunnel_entry(plan.tunnel_service) if plan.tunnel_service
           json = JSON.pretty_generate(document)
           # An empty object prints as `{}` or `{` newline `}`, depending on the json gem.
           "#{json.gsub(/\{\s*\}/, '{}')}\n"
         end
 
+        # A container's entry. With a task definition the image, environment and secrets are read
+        # from it at deploy time, so only the name and port are written here.
+        #
+        # @param plan [Settings::Plan] the resolved settings
+        # @param container [Settings::Container] the container
+        # @return [Hash{String => Object}] its entry in `services.json`
+        def service_entry(plan, container)
+          entry = { "name" => container.name, "port" => container.port }
+          return entry if plan.task_definition
+
+          { "name" => container.name, "repository" => container.repository, "port" => container.port,
+            "env" => container.env, "secrets" => container.secrets }
+        end
+
         # @param tunnel [Settings::Tunnel] the declared tunnel service
         # @return [Hash{String => Object}] its entry in `services.json`
         def tunnel_entry(tunnel)
@@ -296,7 +459,36 @@ module Hecks
         def deploy_box_sh(plan, region)
           template("deploy-box.sh.tmpl", "STACK" => plan.infra_name, "BOX_STACK" => plan.box_stack,
                                          "RDS_STACK" => plan.rds_stack, "REGION" => region,
-                                         "DIR" => "/opt/#{plan.infra_name}", "HEALTH_CHECKS" => health_checks(plan))
+                                         "DIR" => "/opt/#{plan.infra_name}", "HEALTH_CHECKS" => health_checks(plan),
+                                         "SMOKE_HEADER" => smoke_header(plan), "USAGE" => deploy_usage(plan))
+        end
+
+        # @param plan [Settings::Plan] the resolved settings
+        # @return [String] the line that adds the origin secret to the smoke listener's requests, or
+        #   nothing when the world has no origin guard
+        def smoke_header(plan)
+          plan.origin_header ? "\t\theader_up #{plan.origin_header} {$ORIGIN_SECRET}" : ""
+        end
+
+        # @param plan [Settings::Plan] the resolved settings
+        # @return [String] the comment lines that say how to call `deploy-box.sh`
+        def deploy_usage(plan)
+          if plan.task_definition
+            <<~USAGE
+              #   deploy-box.sh [task-definition]
+              #
+              # The task definition (a family or family:revision) defaults to the latest active revision of
+              # #{plan.task_definition}. Secret values are resolved on the box by fetch-secrets.sh and never
+              # pass through the SSM command.
+            USAGE
+          else
+            <<~USAGE
+              #   deploy-box.sh [name=tag ...]
+              #
+              # A container named without a tag runs its "latest" image. Secret values are resolved on the box by
+              # fetch-secrets.sh and never pass through the SSM command.
+            USAGE
+          end
         end
 
         # The probes the post-roll check runs on the box, against the proxy on localhost.
@@ -345,23 +537,27 @@ module Hecks
         def makefile(plan)
           <<~MAKE
             # #{plan.infra_name}: one app box and one RDS instance.
-            #   make stacks VPC=vpc-... PRIVATE_SUBNETS=subnet-a,subnet-b PUBLIC_SUBNET=subnet-c
-            #   make deploy [TAGS="web=20260101 worker=20260101"]
+            #   make stacks VPC=vpc-... PRIVATE_SUBNETS=subnet-a,subnet-b PUBLIC_SUBNET=subnet-c [REHEARSAL=true]
+            #   make deploy #{plan.task_definition ? '[TASKDEF=family:revision]' : '[TAGS="web=20260101 worker=20260101"]'}
             RDS_STACK = #{plan.rds_stack}
             BOX_STACK = #{plan.box_stack}
+            # true makes a throwaway pair: the database is deleted with its stack and the box has no
+            # Elastic IP. The default is a production pair, with deletion protection.
+            REHEARSAL ?= false
 
             .PHONY: stacks deploy
             stacks:
             \taws cloudformation deploy --template-file rds.yaml --stack-name $(RDS_STACK) --capabilities CAPABILITY_IAM \\
-            \t\t--parameter-overrides VpcId=$(VPC) PrivateSubnetIds=$(PRIVATE_SUBNETS)
+            \t\t--parameter-overrides VpcId=$(VPC) PrivateSubnetIds=$(PRIVATE_SUBNETS) Rehearsal=$(REHEARSAL)
             \taws cloudformation deploy --template-file box.yaml --stack-name $(BOX_STACK) --capabilities CAPABILITY_IAM \\
             \t\t--parameter-overrides VpcId=$(VPC) SubnetId=$(PUBLIC_SUBNET) \\
             \t\tDbSecurityGroupId=$$(aws cloudformation describe-stacks --stack-name $(RDS_STACK) --query "Stacks[0].Outputs[?OutputKey=='DbSecurityGroupId'].OutputValue" --output text) \\
             \t\tDbSecretArn=$$(aws cloudformation describe-stacks --stack-name $(RDS_STACK) --query "Stacks[0].Outputs[?OutputKey=='DbSecretArn'].OutputValue" --output text) \\
-            \t\tAlertTopicArn=$$(aws cloudformation describe-stacks --stack-name $(RDS_STACK) --query "Stacks[0].Outputs[?OutputKey=='AlertTopicArn'].OutputValue" --output text)
+            \t\tAlertTopicArn=$$(aws cloudformation describe-stacks --stack-name $(RDS_STACK) --query "Stacks[0].Outputs[?OutputKey=='AlertTopicArn'].OutputValue" --output text) \\
+            \t\tRehearsal=$(REHEARSAL)
 
             deploy:
-            \tbash ./deploy-box.sh $(TAGS)
+            \tbash ./deploy-box.sh $(#{plan.task_definition ? 'TASKDEF' : 'TAGS'})
           MAKE
         end
 

data/lib/hecks/projections/deploy/scripts/hosting.mk.tmpl

@@ -8,11 +8,15 @@
 # machine. It replaces any path to a working copy: ROOT below points the
 # Makefile's build and mint-era steps at it. Override HECKS_VERSION to build
 # from another release, or HECKS_ROOT to use a checkout you manage yourself.
+#
+# HECKS_VERSION=edge is the one exception to "pinned": it follows the newest commit on main,
+# is fetched afresh on every build, and prints the commit it got.
 
 HECKS_VERSION   ?= @@RELEASE@@
 HECKS_SOURCE    ?= @@SOURCE@@
 HECKS_CACHE_DIR ?= @@CACHE_DIR@@
 HECKS_ROOT      ?= $(HECKS_CACHE_DIR)/hecks-$(HECKS_VERSION)
+HECKS_TAG       := $(if $(filter edge,$(HECKS_VERSION)),edge,v$(HECKS_VERSION))
 ROOT            := $(HECKS_ROOT)
 SERVICE         ?= @@SERVICE@@
 URL             ?= @@URL@@
@@ -22,20 +26,32 @@ URL             ?= @@URL@@
 build: hecks-release
 
 # Fetches the pinned release's tag into HECKS_ROOT unless it is already there,
-# and refuses a checkout that is not exactly that tag.
+# and refuses a checkout that is not exactly that tag. The edge tag moves, so it is
+# fetched afresh on every build and its commit is printed, not checked.
 hecks-release:
+ifeq ($(HECKS_VERSION),edge)
+	@mkdir -p "$(HECKS_CACHE_DIR)"; \
+		if [ -d "$(HECKS_ROOT)/.git" ]; then \
+			git -C "$(HECKS_ROOT)" fetch --quiet --depth 1 origin "+refs/tags/edge:refs/tags/edge" && \
+			git -C "$(HECKS_ROOT)" checkout --quiet --detach edge; \
+		else \
+			git clone --quiet --depth 1 --branch edge "$(HECKS_SOURCE)" "$(HECKS_ROOT)"; \
+		fi; \
+		echo "hecks edge at $$(git -C "$(HECKS_ROOT)" rev-parse --short HEAD) in $(HECKS_ROOT)"
+else
 	@if [ -f "$(HECKS_ROOT)/exe/hecks" ]; then \
 		echo "hecks v$(HECKS_VERSION) is already at $(HECKS_ROOT)"; \
 	else \
 		echo "Fetching hecks v$(HECKS_VERSION) from $(HECKS_SOURCE) into $(HECKS_ROOT)..."; \
 		mkdir -p "$(HECKS_CACHE_DIR)" && \
-		git clone --quiet --depth 1 --branch "v$(HECKS_VERSION)" "$(HECKS_SOURCE)" "$(HECKS_ROOT)"; \
+		git clone --quiet --depth 1 --branch "$(HECKS_TAG)" "$(HECKS_SOURCE)" "$(HECKS_ROOT)"; \
 	fi
 	@tag="$$(git -C "$(HECKS_ROOT)" describe --tags --exact-match 2>/dev/null)"; \
-		if [ "$$tag" != "v$(HECKS_VERSION)" ]; then \
-			echo "$(HECKS_ROOT) is at '$$tag', not v$(HECKS_VERSION). Remove it and re-run, or set HECKS_ROOT."; \
+		if [ "$$tag" != "$(HECKS_TAG)" ]; then \
+			echo "$(HECKS_ROOT) is at '$$tag', not $(HECKS_TAG). Remove it and re-run, or set HECKS_ROOT."; \
 			exit 1; \
 		fi
+endif
 
 # Pushes one container's already-built local image under a fresh tag and rolls
 # it out. Build the image first (docker-build); see deploy-service.sh.

data/lib/hecks/projections/deploy/scripts/settings.rb

@@ -12,6 +12,8 @@ module Hecks
           REF       = %r{\A[\w./-]+\z}
           ERA       = /\A[0-9a-zA-Z._-]+\z/
           RELEASE   = /\Av?\d+\.\d+\.\d+([.-][0-9A-Za-z.-]+)?\z/
+          # The moving tag on main; the one release name that is not a version.
+          EDGE      = "edge".freeze
           SOURCE    = %r{\A[\w.@:/~-]+\z}
           MAKE_PATH = /\A[^\s#]+\z/
 
@@ -43,6 +45,8 @@ module Hecks
             release = fetch(:hecks_release, nil)
             raise ArgumentError, missing_release_message unless release
 
+            return EDGE if release.to_s == EDGE
+
             check(:hecks_release, release.to_s, RELEASE).delete_prefix("v")
           end
 
@@ -63,6 +67,8 @@ module Hecks
                   end
 
               so the domain image is built from a fixed Hecks release, not from a checkout on one machine.
+              `hecks_release "edge"` follows the newest commit on main instead: each build fetches it
+              afresh and prints the commit it got, so a build is not reproducible from the name alone.
             MSG
           end