hecks 3.2.0 → 3.3.0

data/lib/hecks/projections/deploy/box/settings.rb

@@ -1,3 +1,5 @@
+require_relative "hosting_settings"
+
 module Hecks
   module Projections
     module Deploy
@@ -34,6 +36,8 @@ module Hecks
                          "@sha256:226d1f059b75399fe19182893c7184591c07b97afc8dfcf44eeb80c9a77a530f".freeze
           TUNNEL_SHAPE = "tunnel: a hash needs `to` (the container it forwards to) and `token_secret` " \
                          "(the secret holding the tunnel token)".freeze
+          ORIGIN_ENV_SHAPE = "origin_env: the container environment variable names that hold the origin secret " \
+                             "need an origin_secret and a task_definition to compare against".freeze
           ORIGIN_PAIR = "origin_header and origin_secret go together: the header a CDN sends, " \
                         "and the secret that holds its value".freeze
 
@@ -45,7 +49,9 @@ module Hecks
           # @!attribute [r] env [Hash{String => String}] plain environment variables
           # @!attribute [r] secrets [Hash{String => String}] environment variable => secret name,
           #   resolved on the box at deploy time and never written into a template
-          Container = Struct.new(:name, :repository, :port, :env, :secrets, keyword_init: true)
+          # @!attribute [r] tag_parameter [String] the stack parameter that holds this container's
+          #   image tag, which a hosting `deploy-service.sh` sets
+          Container = Struct.new(:name, :repository, :port, :env, :secrets, :tag_parameter, keyword_init: true)
 
           # A set of URL paths one container serves.
           #
@@ -79,8 +85,8 @@ module Hecks
             :infra_name, :stack_prefix, :instance_type, :volume_gb, :swap_gb, :database_class,
             :storage_gb, :backup_days, :snapshots_keep, :database_name, :engine_version,
             :containers, :routes, :default_container, :origin_header, :origin_secret,
-            :secret_prefixes, :writable_secrets, :tunnel, :tunnel_service, :proxy_image, :task_definition,
-            :migration, :s3_buckets, keyword_init: true
+            :secret_prefixes, :writable_secrets, :origin_env, :tunnel, :tunnel_service, :proxy_image, :task_definition,
+            :migration, :s3_buckets, :hosting, keyword_init: true
           ) do
             # @return [String] the CloudFormation stack that holds the database
             def rds_stack = "#{stack_prefix}-#{infra_name}-rds"
@@ -110,6 +116,7 @@ module Hecks
             task_definition = read_task_definition(s[:task_definition], listed)
             containers = read_containers(listed, infra_name)
             header, secret = read_origin(s)
+            origin_env = read_origin_env(s, secret, task_definition)
             tunnel, tunnel_service = read_tunnel(s.fetch(:tunnel, false), containers)
 
             Plan.new(
@@ -118,12 +125,13 @@ module Hecks
               engine_version: check(:engine_version, s.fetch(:engine_version, "16").to_s, ENGINE),
               containers: containers, routes: read_routes(s.fetch(:routes, []), containers),
               default_container: read_default(s[:default_container], containers), origin_header: header,
-              origin_secret: secret, secret_prefixes: read_prefixes(s, infra_name),
+              origin_secret: secret, origin_env: origin_env, secret_prefixes: read_prefixes(s, infra_name),
               writable_secrets: read_writable_secrets(s),
               tunnel: tunnel, tunnel_service: tunnel_service,
               proxy_image: check(:proxy_image, s.fetch(:proxy_image, PROXY_IMAGE), IMAGE),
               task_definition: task_definition, migration: read_migration(s[:migration], database_name),
-              s3_buckets: read_s3_access(s.fetch(:s3_access, []))
+              s3_buckets: read_s3_access(s.fetch(:s3_access, [])),
+              hosting: HostingSettings.read(s, task_definition)
             )
           end
 
@@ -221,10 +229,18 @@ module Hecks
               name: name, repository: check(:repository, spec.fetch(:repository, "#{infra_name}-#{name}"), REPOSITORY),
               port: integer(:port, spec.fetch(:port) { raise ArgumentError, "containers: #{name} has no port" }, 1, 65_535),
               env: string_map(:env, spec.fetch(:env, {}), ENV_KEY, ENV_VALUE),
-              secrets: string_map(:secrets, spec.fetch(:secrets, {}), ENV_KEY, SECRET_NAME)
+              secrets: string_map(:secrets, spec.fetch(:secrets, {}), ENV_KEY, SECRET_NAME),
+              tag_parameter: check(:tag_parameter, spec.fetch(:tag_parameter, default_tag_parameter(name)),
+                                   HostingSettings::PARAMETER)
             )
           end
 
+          # @param name [String] a container name such as `web-app`
+          # @return [String] the image-tag parameter a stack names it by, such as `WebAppImageTag`
+          def default_tag_parameter(name)
+            "#{name.split('-').map(&:capitalize).join}ImageTag"
+          end
+
           def read_routes(list, containers)
             raise ArgumentError, "routes: expected a list" unless list.is_a?(Array)
 
@@ -263,6 +279,24 @@ module Hecks
             [check(:origin_header, header, HEADER), check(:origin_secret, secret, SECRET_NAME)]
           end
 
+          # The container environment variables that hold the origin secret in the task
+          # definition. Only Caddy reads the named secret; the containers keep the task
+          # definition's copy, so a copy that differs makes every request through the CDN
+          # fail. Naming them lets deploy refuse a mismatch.
+          #
+          # @param settings [Hash{Symbol => Object}] the world's `AwsBox` settings
+          # @param secret [String, nil] the declared origin secret
+          # @param task_definition [String, nil] the declared task definition family
+          # @return [Array<String>] the variable names, empty when none are declared
+          # @raise [ArgumentError] when named without an origin secret and a task definition
+          def read_origin_env(settings, secret, task_definition)
+            names = Array(settings.fetch(:origin_env, []))
+            return [] if names.empty?
+            raise ArgumentError, ORIGIN_ENV_SHAPE if secret.nil? || task_definition.nil?
+
+            names.map { |name| check(:origin_env, name, ENV_KEY) }.uniq
+          end
+
           # `tunnel true` only opens the outbound port; a hash also runs cloudflared as a service.
           #
           # @param value [Boolean, Hash{Symbol => Object}] the world's `tunnel` setting

data/lib/hecks/projections/deploy/box/templates/deploy-box.sh.tmpl

@@ -55,6 +55,7 @@ ROLL=$(jq -n --arg compose "$(B64 "$WORK/compose.json")" --arg secrets "$(B64 "$
   --arg registry "$ACCOUNT.dkr.ecr.$REGION.amazonaws.com" --arg dir "$DIR" '
   {commands: ["cloud-init status --wait >/dev/null 2>&1 || true",
     "set -e", "mkdir -p \($dir)/caddy-extra && cd \($dir)", "umask 077",
+    "CADDY_BEFORE=$(cat Caddyfile 2>/dev/null | sha256sum)",
     "echo \($compose) | base64 -d > compose.json; echo \($secrets) | base64 -d > secrets.json",
     "echo \($caddy) | base64 -d > Caddyfile; echo \($fetch) | base64 -d > fetch-secrets.sh",
     "bash fetch-secrets.sh",
@@ -62,6 +63,7 @@ ROLL=$(jq -n --arg compose "$(B64 "$WORK/compose.json")" --arg secrets "$(B64 "$
     "aws ecr get-login-password --region '"$REGION"' | docker login --username AWS --password-stdin \($registry) >/dev/null",
     "docker compose -f compose.json pull --quiet",
     "docker compose -f compose.json up -d --remove-orphans",
+    "[ \"$(sha256sum < Caddyfile)\" = \"$CADDY_BEFORE\" ] || docker compose -f compose.json restart caddy",
     "for i in $(seq 1 40); do docker compose -f compose.json ps --format \"{{.Status}}\" | grep -qE \"^Up (Less than a second|[0-4] seconds?)|Restarting|Created|Exited\" || break; sleep 1; done; docker compose -f compose.json ps --format \"table {{.Service}}\\t{{.Status}}\""],
    executionTimeout: ["900"]}')
 

data/lib/hecks/projections/deploy/box/templates/deploy-service.services-roll.part

@@ -0,0 +1,33 @@
+# deploy-box.sh renders every container the roll does not name at "latest". Name the tag each other
+# container runs now, read from the box's own Compose file, so rolling one service leaves the rest alone.
+BOX_ID="$(aws cloudformation describe-stacks --stack-name "$BOX_STACK" --region "$REGION" \
+  --query "Stacks[0].Outputs[?OutputKey=='InstanceId'].OutputValue" --output text)"
+[ -n "$BOX_ID" ] && [ "$BOX_ID" != None ] || { echo "no instance in stack ${BOX_STACK}" >&2; exit 1; }
+
+echo "==> reading the tags the box runs now"
+READ_JSON="$(jq -n --arg c "cat ${BOX_DIR}/compose.json 2>/dev/null || echo '{}'" '{commands: [$c]}')"
+COMMAND_ID="$(aws ssm send-command --instance-ids "$BOX_ID" --document-name AWS-RunShellScript --region "$REGION" \
+  --parameters "$READ_JSON" --query 'Command.CommandId' --output text)"
+for _ in $(seq 1 30); do
+  sleep 2
+  state="$(aws ssm get-command-invocation --command-id "$COMMAND_ID" --instance-id "$BOX_ID" --region "$REGION" \
+    --query Status --output text 2>/dev/null || echo Pending)"
+  [ "$state" != InProgress ] && [ "$state" != Pending ] && break
+done
+[ "$state" = Success ] || { echo "could not read the box's Compose file (${state}); nothing was rolled" >&2; exit 1; }
+aws ssm get-command-invocation --command-id "$COMMAND_ID" --instance-id "$BOX_ID" --region "$REGION" \
+  --query StandardOutputContent --output text > "${SCRATCH_DIR}/compose.json"
+
+TAGS=()
+for name in $SERVICES; do
+  if [ "$name" = "$SERVICE_NAME" ]; then
+    TAGS+=("${name}=${TAG}")
+    continue
+  fi
+  image="$(jq -r --arg n "$name" '.services[$n].image // empty' "${SCRATCH_DIR}/compose.json")"
+  # A container the box does not run yet starts at its "latest" image.
+  if [ -n "$image" ]; then TAGS+=("${name}=${image##*:}"); fi
+done
+echo "==> rolling ${TAGS[*]}"
+
+bash ./deploy-box.sh "${TAGS[@]}"

data/lib/hecks/projections/deploy/box/templates/deploy-service.sh.tmpl

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# GENERATED by hecks deploy project -- re-run it to refresh this file rather than hand-editing.
+# Settings come from the domain's deployed_to("AwsBox") block (hosting_scripts true).
+#
+# Deploys ONE container's freshly built local image to @@STACK@@'s app box, under a fresh, unique ECR tag.
+#
+#   bash deploy-service.sh <service>
+#   EXISTING_TAG=<tag> bash deploy-service.sh <service>    redeploy a tag already in ECR (no push)
+#
+# Why it exists: pushing a floating tag and restarting a container looks like a deploy, but nothing
+# records which image the box was told to run, and a box that pulls a tag it already has keeps the old
+# image. This script pushes the local image under a never-reused tag (<service>-<UTC timestamp>),
+# refuses a tag that is already in ECR, and names that tag in the roll.
+@@MODE_SUMMARY@@
+#
+# It ends with smoke-after-deploy.sh, whose own exit codes (20-23) are distinct from a failed deploy
+# step's. SKIP_POST_DEPLOY_SMOKE=1 skips it.
+#
+# Environment:
+#   LOCAL_IMAGE               local image to push, default <ecr repository>:latest
+#   EXISTING_TAG              redeploy this tag from ECR instead of building and pushing
+#   SETTLE_TIMEOUT_SECS       default 1200, how long to wait for a stack update
+#   SKIP_POST_DEPLOY_SMOKE=1  passed through to smoke-after-deploy.sh
+set -euo pipefail
+
+@@SETTINGS@@
+
+@@SERVICES@@
+
+SERVICE_NAME="${1:?usage: deploy-service.sh <service> (one of: ${SERVICES})}"
+resolve_service "$SERVICE_NAME"
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+SETTLE_TIMEOUT_SECS="${SETTLE_TIMEOUT_SECS:-1200}"
+ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)"
+ECR_HOST="${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com"
+SCRATCH_DIR="$(mktemp -d)"
+trap 'rm -rf "$SCRATCH_DIR"' EXIT
+
+# True once $1 (a value of $SECONDS) is further in the past than the settle timeout.
+timed_out() { [ $((SECONDS - $1)) -gt "$SETTLE_TIMEOUT_SECS" ]; }
+
+@@REPOSITORY_STEP@@
+TAG="${EXISTING_TAG:-${SERVICE_NAME}-$(date -u +%Y%m%d%H%M%S)}"
+IMAGE_URI="${ECR_HOST}/${ECR_REPOSITORY}:${TAG}"
+LOCAL_IMAGE="${LOCAL_IMAGE:-${ECR_REPOSITORY}:latest}"
+
+if [ -n "${EXISTING_TAG:-}" ]; then
+  echo "==> reusing ${IMAGE_URI} (no push)"
+  aws ecr describe-images --repository-name "$ECR_REPOSITORY" --region "$REGION" \
+    --image-ids "imageTag=${TAG}" --query 'imageDetails[0].imageTags' --output text > /dev/null \
+    || { echo "tag ${TAG} is not in ECR" >&2; exit 1; }
+else
+  if aws ecr describe-images --repository-name "$ECR_REPOSITORY" --region "$REGION" \
+      --image-ids "imageTag=${TAG}" > /dev/null 2>&1; then
+    echo "tag ${TAG} is already in ECR; a tag is never reused. Run again." >&2
+    exit 1
+  fi
+  echo "==> tagging ${LOCAL_IMAGE} as ${IMAGE_URI}"
+  docker tag "$LOCAL_IMAGE" "$IMAGE_URI"
+
+  # A private docker config with no credential helper: a helper that blocks on a keychain prompt
+  # hangs forever in any unattended shell.
+  echo "==> ecr login"
+  DOCKER_CONFIG_DIR="${SCRATCH_DIR}/docker-config"
+  mkdir -p "$DOCKER_CONFIG_DIR"
+  echo '{}' > "${DOCKER_CONFIG_DIR}/config.json"
+  aws ecr get-login-password --region "$REGION" \
+    | docker --config "$DOCKER_CONFIG_DIR" login --username AWS --password-stdin "$ECR_HOST"
+
+  echo "==> pushing ${IMAGE_URI}"
+  docker --config "$DOCKER_CONFIG_DIR" push "$IMAGE_URI"
+fi
+
+@@ROLL_STEPS@@
+
+# The deploy itself is done. End with the smoke, which reports separately.
+echo "==> deploy succeeded"
+bash ./smoke-after-deploy.sh

data/lib/hecks/projections/deploy/box/templates/deploy-service.taskdef-roll.part

@@ -0,0 +1,77 @@
+echo "==> setting the stack's ${CFN_PARAM_KEY} to ${TAG} (registers the new task definition revision)"
+stack_parameters() {
+  aws cloudformation describe-stacks --stack-name "$IMAGE_STACK" --region "$REGION" \
+    --query 'Stacks[0].Parameters[].{k:ParameterKey,v:ParameterValue}' --output json
+}
+stack_parameters > "${SCRATCH_DIR}/params-before.json"
+ALL_PARAM_KEYS="$(jq -r '.[].k' "${SCRATCH_DIR}/params-before.json")"
+CFN_ARGS=()
+found=0
+for key in $ALL_PARAM_KEYS; do
+  if [ "$key" = "$CFN_PARAM_KEY" ]; then
+    CFN_ARGS+=("ParameterKey=${key},ParameterValue=${TAG}")
+    found=1
+  else
+    CFN_ARGS+=("ParameterKey=${key},UsePreviousValue=true")
+  fi
+done
+if [ "$found" != "1" ]; then
+  echo "stack ${IMAGE_STACK} has no parameter named ${CFN_PARAM_KEY}; the image was pushed but nothing was rolled" >&2
+  exit 1
+fi
+
+if UPDATE_OUT="$(aws cloudformation update-stack --stack-name "$IMAGE_STACK" --region "$REGION" \
+    --use-previous-template --capabilities CAPABILITY_IAM --parameters "${CFN_ARGS[@]}" \
+    --query 'StackId' --output text 2>&1)"; then
+  echo "==> waiting for the CloudFormation stack update to settle"
+  started=$SECONDS
+  while true; do
+    stack_status="$(aws cloudformation describe-stacks --stack-name "$IMAGE_STACK" --region "$REGION" \
+      --query 'Stacks[0].StackStatus' --output text)"
+    case "$stack_status" in
+      UPDATE_COMPLETE) echo "    -> ${stack_status}"; break ;;
+      *ROLLBACK*|*FAILED*)
+        echo "    -> ${stack_status}: the stack update failed, investigate before rolling anything" >&2
+        exit 1 ;;
+    esac
+    if timed_out "$started"; then
+      echo "==> the stack update did not settle within ${SETTLE_TIMEOUT_SECS}s (last: ${stack_status})" >&2
+      exit 1
+    fi
+    sleep 10
+  done
+elif echo "$UPDATE_OUT" | grep -q "No updates are to be performed"; then
+  echo "    -> the stack already has ${TAG}; nothing to update"
+else
+  echo "$UPDATE_OUT" >&2
+  exit 1
+fi
+
+echo "==> checking that only ${CFN_PARAM_KEY} changed"
+stack_parameters > "${SCRATCH_DIR}/params-after.json"
+CHANGED="$(jq -rn --slurpfile before "${SCRATCH_DIR}/params-before.json" --slurpfile after "${SCRATCH_DIR}/params-after.json" '
+  ($before[0] | map({(.k): .v}) | add // {}) as $b
+  | ($after[0] | map({(.k): .v}) | add // {}) as $a
+  | [($b + $a) | keys[] | select($b[.] != $a[.])] | join(" ")')"
+NOW="$(jq -r --arg k "$CFN_PARAM_KEY" '.[] | select(.k == $k) | .v' "${SCRATCH_DIR}/params-after.json")"
+if [ "$NOW" != "$TAG" ] || { [ -n "$CHANGED" ] && [ "$CHANGED" != "$CFN_PARAM_KEY" ]; }; then
+  echo "expected only ${CFN_PARAM_KEY} to change to ${TAG}; it is ${NOW} and changed: ${CHANGED:-nothing}" >&2
+  exit 1
+fi
+echo "    -> ${CFN_PARAM_KEY} is ${TAG}; no other parameter changed"
+
+# The newest active revision of the family is what the stack just registered. Roll it only if it
+# really carries the image pushed above: a stack update that registered something else would
+# otherwise roll an old image and still look like a deploy.
+TD="$(aws ecs describe-task-definition --task-definition "$FAMILY" --region "$REGION" \
+  --query 'taskDefinition.[family,revision]' --output text | tr '\t' ':')"
+RUNNING_IMAGE="$(aws ecs describe-task-definition --task-definition "$TD" --region "$REGION" \
+  --query "taskDefinition.containerDefinitions[?name=='${SERVICE_NAME}'].image | [0]" --output text)"
+if [ "$RUNNING_IMAGE" != "$IMAGE_URI" ]; then
+  echo "task definition ${TD} has ${RUNNING_IMAGE}, not ${IMAGE_URI}; refusing to roll" >&2
+  exit 1
+fi
+echo "==> task definition ${TD} carries ${SERVICE_NAME}=${TAG}"
+
+bash ./deploy-box.sh "$TD"
+export TASKDEF="$TD"

data/lib/hecks/projections/deploy/box/templates/deploy-service.taskdef.part

@@ -0,0 +1,12 @@
+# The active task definition names the image repository of every container, so the push goes where
+# the box already pulls from.
+active_image() {
+  aws ecs describe-task-definition --task-definition "$FAMILY" --region "$REGION" \
+    --query "taskDefinition.containerDefinitions[?name=='${SERVICE_NAME}'].image | [0]" --output text
+}
+CURRENT_IMAGE="$(active_image)"
+case "$CURRENT_IMAGE" in
+  None|"") echo "task definition family ${FAMILY} has no container named ${SERVICE_NAME}" >&2; exit 1 ;;
+esac
+ECR_REPOSITORY="${CURRENT_IMAGE#*/}"
+ECR_REPOSITORY="${ECR_REPOSITORY%%[:@]*}"

data/lib/hecks/projections/deploy/box/templates/hosting.mk.tmpl

@@ -0,0 +1,24 @@
+# GENERATED by hecks deploy project -- re-run it to refresh this file rather than hand-editing.
+# Included by the Makefile beside it; settings come from the domain's deployed_to("AwsBox") block
+# (hosting_scripts true).
+
+HECKS   ?= hecks
+SERVICE ?= @@SERVICE@@
+URL     ?= @@URL@@
+
+.PHONY: deploy-service smoke-after-deploy check-era
+
+# Pushes one container's already-built local image under a fresh tag and rolls it onto the box.
+# Build the image first; see deploy-service.sh.
+deploy-service:
+	bash ./deploy-service.sh $(SERVICE)
+
+# Waits for the last roll to settle, then dispatches and follows the smoke.
+# SKIP_POST_DEPLOY_SMOKE=1 opts out, DRY_RUN=1 dispatches nothing.
+smoke-after-deploy:
+	bash ./smoke-after-deploy.sh
+
+# Compares the era a live host reports at GET /version with expected-era.
+check-era:
+	@test -n "$(URL)" || { echo "usage: make check-era URL=https://host.example"; exit 1; }
+	HECKS_ENVIRONMENT=memory $(HECKS) host.check_era host="$(URL)" expected=expected-era --wait

data/lib/hecks/projections/deploy/box/templates/render-compose-taskdef.sh.tmpl

@@ -9,6 +9,10 @@
 # are replaced with the RDS stack's values where the task defines them, and every container runs on
 # the box's own network, so the proxy reaches each one on 127.0.0.1:<port>. Secrets (name + valueFrom)
 # are written to secrets.json for fetch-secrets.sh to resolve on the box.
+#
+# When the world names origin_env, the named origin secret is compared with those variables in the
+# task definition and the render refuses on any difference (Caddy checks the CDN's header against
+# the secret; the containers check it against their own copy). Values are never printed.
 set -euo pipefail
 HERE=$(cd "$(dirname "$0")" && pwd)
 DB_HOST=${1:?rds endpoint}
@@ -18,6 +22,19 @@ TD=${3:-@@FAMILY@@}
 DEF=$(aws ecs describe-task-definition --task-definition "$TD" --query 'taskDefinition.containerDefinitions' --output json)
 echo "rendering from task definition $TD" >&2
 
+ORIGIN_ENV=$(jq -r '.origin.env // [] | .[]' "$HERE/services.json")
+if [ -n "$ORIGIN_ENV" ]; then
+  ORIGIN_SECRET_ID=$(jq -r '.origin.secret' "$HERE/services.json")
+  ORIGIN_VALUE=$(aws secretsmanager get-secret-value --secret-id "$ORIGIN_SECRET_ID" --query SecretString --output text)
+  for NAME in $ORIGIN_ENV; do
+    HOLDERS=$(echo "$DEF" | jq -r --arg n "$NAME" '[.[] | select(any((.environment // [])[]; .name == $n)) | .name] | join(" ")')
+    [ -n "$HOLDERS" ] || { echo "refusing to render: origin_env names $NAME, which no container in $TD sets" >&2; exit 1; }
+    DIFFERENT=$(echo "$DEF" | jq -r --arg n "$NAME" --arg v "$ORIGIN_VALUE" \
+      '[.[] | select(any((.environment // [])[]; .name == $n and .value != $v)) | .name] | join(" ")')
+    [ -z "$DIFFERENT" ] || { echo "refusing to render: $NAME in $TD (container: $DIFFERENT) differs from the origin secret $ORIGIN_SECRET_ID; the proxy would refuse every request from the CDN. Make them equal." >&2; exit 1; }
+  done
+fi
+
 jq --argjson def "$DEF" --arg host "$DB_HOST" --arg secret "$DB_SECRET" '
   def log: {driver: "json-file", options: {"max-size": "10m", "max-file": "3"}};
   def env(c): ((c.environment // []) | map({(.name): .value}) | add // {})

data/lib/hecks/projections/deploy/box/templates/smoke-after-deploy.sh.tmpl

@@ -0,0 +1,271 @@
+#!/usr/bin/env bash
+# GENERATED by hecks deploy project -- re-run it to refresh this file rather than hand-editing.
+# Settings come from the domain's deployed_to("AwsBox") block (hosting_scripts true).
+#
+# Runs the production smoke once a roll of @@STACK@@'s app box has settled, so a deploy ends with a real
+# check of the live site instead of waiting for the next scheduled run. Called at the end of
+# deploy-service.sh and of `make deploy`; also runnable by hand:
+#
+#   bash smoke-after-deploy.sh
+#
+# What it does, in order:
+#   1. Waits for the roll to settle: two consecutive checks, one interval apart, must agree that
+@@SETTLE_CHECKS@@
+#      Gives up after the timeout. The box is read through SSM, the way deploy-box.sh rolls it.
+#   2. Dispatches the smoke workflow on the configured ref, finds the run that dispatch created (a
+#      workflow_dispatch run that did not exist before the dispatch, never a scheduled one), waits for it,
+#      and prints its conclusion, the failed check names and its URL.
+#
+# It only ever READS AWS; the smoke itself runs on GitHub against the live site.
+#
+# Exit codes (chosen to stay clear of the ones a failed deploy step produces: 1 from bash, 2 from bash
+# misuse, 254/255 from the aws CLI):
+#    0  the smoke passed, was skipped on purpose (SKIP_POST_DEPLOY_SMOKE=1), or was left running
+#       (SMOKE_ASYNC=1)
+#   20  the roll did not settle in time, or a stack ended in a failed state; the smoke was NOT run
+#   21  `gh` is missing or not signed in, or no repository is configured; the smoke was NOT run (the
+#       manual command is printed)
+#   22  the smoke ran and FAILED
+#   23  the smoke could not be dispatched, found or finished; result unknown
+#
+# Environment:
+#   SKIP_POST_DEPLOY_SMOKE=1   skip everything and say so
+#   SMOKE_ASYNC=1              dispatch the smoke and return as soon as its run is found, printing the
+#                              run URL, instead of waiting for it to finish (exit 0: the result is NOT
+#                              known yet)
+#   DRY_RUN=1                  wait for the roll as usual, then print what would be dispatched without
+#                              dispatching anything
+#   SETTLE_TIMEOUT_SECS        default 1200
+#   SETTLE_CHECK_INTERVAL_SECS default 5
+#   SMOKE_RUN_TIMEOUT_SECS     default 1800, how long to wait for the run
+#   SSM_POLL_SECS              default 2, how often to ask SSM whether a command on the box finished
+#   REPO                       GitHub repository (owner/name) holding the workflow
+#   WORKFLOW                   workflow file name
+#   SMOKE_REF                  ref to dispatch on
+#   TASKDEF                    the task definition the box should run (family or family:revision);
+#                              default: the latest revision of the family. deploy-service.sh and
+#                              `make deploy TASKDEF=...` pass the one they rolled.
+set -uo pipefail
+
+@@SETTINGS@@
+
+SETTLE_TIMEOUT_SECS="${SETTLE_TIMEOUT_SECS:-1200}"
+SETTLE_CHECK_INTERVAL_SECS="${SETTLE_CHECK_INTERVAL_SECS:-5}"
+SMOKE_RUN_TIMEOUT_SECS="${SMOKE_RUN_TIMEOUT_SECS:-1800}"
+SSM_POLL_SECS="${SSM_POLL_SECS:-2}"
+MANUAL_COMMAND="bash smoke-after-deploy.sh"
+
+if [ "${SKIP_POST_DEPLOY_SMOKE:-}" = "1" ]; then
+  echo "==> post-deploy smoke SKIPPED (SKIP_POST_DEPLOY_SMOKE=1)."
+  echo "    Run it yourself when ready: ${MANUAL_COMMAND}"
+  exit 0
+fi
+
+if ! command -v gh >/dev/null 2>&1 || ! gh auth status >/dev/null 2>&1; then
+  echo "==> post-deploy smoke NOT RUN: the GitHub CLI (gh) is missing or not signed in." >&2
+  echo "    Run it by hand: gh workflow run ${WORKFLOW} --ref ${SMOKE_REF}" >&2
+  echo "    or, once gh works: ${MANUAL_COMMAND}" >&2
+  exit 21
+fi
+
+if [ -z "$REPO" ]; then
+  REPO="$(gh repo view --json nameWithOwner --jq .nameWithOwner 2>/dev/null || true)"
+fi
+if [ -z "$REPO" ]; then
+  echo "==> post-deploy smoke NOT RUN: no GitHub repository is configured. Set REPO=owner/name, or the" >&2
+  echo "    smoke_repo setting, or run from a clone gh can read." >&2
+  exit 21
+fi
+MANUAL_DISPATCH="gh workflow run ${WORKFLOW} --repo ${REPO} --ref ${SMOKE_REF}"
+
+# The status of a stack, or nothing when it cannot be read.
+stack_status() {
+  aws cloudformation describe-stacks --stack-name "$1" --region "$REGION" \
+    --query 'Stacks[0].StackStatus' --output text 2>/dev/null
+}
+
+# Runs a script on the box over SSM and prints its output; fails when it does not succeed.
+box_run() {
+  local payload command_id state
+  payload="$(jq -n --arg c "$1" '{commands: [$c]}')"
+  command_id="$(aws ssm send-command --instance-ids "$BOX_ID" --document-name AWS-RunShellScript \
+    --region "$REGION" --parameters "$payload" --query 'Command.CommandId' --output text 2>/dev/null)" || return 1
+  for _ in $(seq 1 60); do
+    sleep "$SSM_POLL_SECS"
+    state="$(aws ssm get-command-invocation --command-id "$command_id" --instance-id "$BOX_ID" \
+      --region "$REGION" --query Status --output text 2>/dev/null || echo Pending)"
+    [ "$state" != InProgress ] && [ "$state" != Pending ] && break
+  done
+  [ "$state" = Success ] || return 1
+  aws ssm get-command-invocation --command-id "$command_id" --instance-id "$BOX_ID" --region "$REGION" \
+    --query StandardOutputContent --output text
+}
+
+# Prints "ok <signature>" when the box looks settled, otherwise "wait <why>". The signature names
+# every image the box runs, so two checks agree only when nothing changed between them.
+box_state() {
+  local report compose ps name image wanted signature
+  report="$(box_run "cd ${BOX_DIR} && cat compose.json && echo ===PS=== && docker compose -f compose.json ps --format '{{.Service}} {{.Status}}'")" \
+    || { echo "wait could not read the box"; return; }
+  compose="${report%%===PS===*}"
+  ps="${report#*===PS===}"
+  for name in $EXPECTED_SERVICES; do
+    printf '%s\n' "$ps" | grep -E "^${name} Up " | grep -qvE " Up (Less than a second|[0-4] seconds?)|Restarting" \
+      || { echo "wait ${name} is not up and settled"; return; }
+  done
+  signature=""
+  for name in $EXPECTED_SERVICES; do
+    image="$(printf '%s' "$compose" | jq -r --arg n "$name" '.services[$n].image // empty' 2>/dev/null)"
+    [ "$name" = caddy ] || [ -n "$image" ] || { echo "wait ${name} is not in the box's Compose file"; return; }
+    if [ -n "$TASK_DEFINITION" ] && [ "$name" != caddy ]; then
+      wanted="$(aws ecs describe-task-definition --task-definition "$TASK_DEFINITION" --region "$REGION" \
+        --query "taskDefinition.containerDefinitions[?name=='${name}'].image | [0]" --output text 2>/dev/null)"
+      [ "$image" = "$wanted" ] || { echo "wait ${name} runs ${image}, ${TASK_DEFINITION} has ${wanted}"; return; }
+    fi
+    signature="${signature}${name}=${image} "
+  done
+  echo "ok ${signature}"
+}
+
+wait_for_settled_roll() {
+  echo "==> waiting for the roll to settle (box ${BOX_STACK}@@IMAGE_STACK_WAIT@@; timeout ${SETTLE_TIMEOUT_SECS}s)"
+  local start=$SECONDS previous="" status stacks state signature why s
+  BOX_ID="$(aws cloudformation describe-stacks --stack-name "$BOX_STACK" --region "$REGION" \
+    --query "Stacks[0].Outputs[?OutputKey=='InstanceId'].OutputValue" --output text 2>/dev/null)"
+  if [ -z "$BOX_ID" ] || [ "$BOX_ID" = None ]; then
+    echo "==> no instance in stack ${BOX_STACK}. The smoke was NOT run." >&2
+    return 20
+  fi
+  while true; do
+    stacks=""
+    for s in $WATCHED_STACKS; do
+      status="$(stack_status "$s")"
+      case "$status" in
+        *ROLLBACK*|*FAILED*)
+          echo "==> roll did not settle: stack ${s} is ${status}. The smoke was NOT run." >&2
+          return 20 ;;
+      esac
+      stacks="${stacks}${s}=${status:-unreadable} "
+    done
+    if printf '%s' "$stacks" | tr ' ' '\n' | grep -v '^$' | grep -qvE '=(CREATE|UPDATE)_COMPLETE$'; then
+      state="wait ${stacks}"
+    else
+      state="$(box_state)"
+    fi
+    if [ "${state%% *}" = "ok" ]; then
+      signature="${stacks}${state#ok }"
+      if [ "$signature" = "$previous" ]; then
+        echo "    -> settled on ${state#ok } (two consecutive checks agree)"
+        return 0
+      fi
+      echo "    -> looks settled (${state#ok }); confirming in ${SETTLE_CHECK_INTERVAL_SECS}s"
+      previous="$signature"
+    else
+      why="${state#wait }"
+      echo "    -> not yet: ${why}"
+      previous=""
+    fi
+    if [ $((SECONDS - start + SETTLE_CHECK_INTERVAL_SECS)) -gt "$SETTLE_TIMEOUT_SECS" ]; then
+      echo "==> roll did not settle within ${SETTLE_TIMEOUT_SECS}s (last: ${state#wait }). The smoke was NOT run." >&2
+      return 20
+    fi
+    sleep "$SETTLE_CHECK_INTERVAL_SECS"
+  done
+}
+
+# Ids of the workflow_dispatch runs that exist now, one per line.
+dispatch_run_ids() {
+  gh run list --repo "$REPO" --workflow "$WORKFLOW" --event workflow_dispatch --limit 50 \
+    --json databaseId --jq '.[].databaseId' 2>/dev/null
+}
+
+print_failed_checks() {
+  local run_id="$1" log names
+  log="$(gh run view "$run_id" --repo "$REPO" --log 2>/dev/null || true)"
+  # Log lines are "<job>\t<step>\t<timestamp> <text>"; a smoke that prints "  <check>... FAILED: <why>"
+  # for each failing check is listed by name.
+  names="$(printf '%s\n' "$log" | awk -F'\t' 'NF >= 3 { print $3 }' | sed -E 's/^[0-9T:.-]+Z //' \
+    | grep -E '\.\.\. FAILED:' | sed -E 's/^ +//' || true)"
+  if [ -n "$names" ]; then
+    echo "    failed checks:"
+    printf '%s\n' "$names" | sed 's/^/      - /'
+  else
+    echo "    (no failed check names in the log; the run may have failed before the checks. Last log lines:)"
+    printf '%s\n' "$log" | tail -n 15 | sed 's/^/      /'
+  fi
+}
+
+run_smoke() {
+  local before run_id="" waited=0 url conclusion status
+  if ! before="$(dispatch_run_ids)"; then
+    echo "==> could not list the smoke workflow's runs, so it was NOT dispatched (it could not be told apart afterwards). Run it by hand: ${MANUAL_DISPATCH}" >&2
+    return 23
+  fi
+
+  echo "==> dispatching the production smoke: ${MANUAL_DISPATCH}"
+  if ! gh workflow run "$WORKFLOW" --repo "$REPO" --ref "$SMOKE_REF"; then
+    echo "==> could not dispatch the smoke workflow. Deploy succeeded; run it by hand: ${MANUAL_DISPATCH}" >&2
+    return 23
+  fi
+
+  # The dispatch call does not return the run, so find the workflow_dispatch run that was not there
+  # before it (scheduled runs are never candidates).
+  while [ -z "$run_id" ] && [ "$waited" -lt 180 ]; do
+    sleep 2
+    waited=$((waited + 2))
+    run_id="$(comm -13 <(printf '%s\n' "$before" | sort) <(dispatch_run_ids | sort) | sort -n | tail -n 1)"
+  done
+  if [ -z "$run_id" ]; then
+    echo "==> dispatched, but the new run never appeared. Deploy succeeded; check https://github.com/${REPO}/actions/workflows/${WORKFLOW}" >&2
+    return 23
+  fi
+  url="https://github.com/${REPO}/actions/runs/${run_id}"
+  echo "==> smoke run ${run_id}: ${url}"
+  if [ "${SMOKE_ASYNC:-}" = "1" ]; then
+    echo "==> deploy succeeded; the smoke is still running (SMOKE_ASYNC=1), so its result is NOT known yet."
+    echo "    Watch it: gh run watch ${run_id} --repo ${REPO}   or open ${url}"
+    return 0
+  fi
+
+  waited=0
+  while true; do
+    status="$(gh run view "$run_id" --repo "$REPO" --json status --jq .status 2>/dev/null || true)"
+    [ "$status" = "completed" ] && break
+    if [ "$waited" -ge "$SMOKE_RUN_TIMEOUT_SECS" ]; then
+      echo "==> deploy succeeded, but the smoke run did not finish within ${SMOKE_RUN_TIMEOUT_SECS}s: see ${url}" >&2
+      return 23
+    fi
+    sleep 5
+    waited=$((waited + 5))
+  done
+
+  conclusion="$(gh run view "$run_id" --repo "$REPO" --json conclusion --jq .conclusion || true)"
+  echo "==> smoke conclusion: ${conclusion}  ${url}"
+  if [ "$conclusion" = "success" ]; then
+    echo "==> deploy succeeded, post-deploy smoke passed."
+    return 0
+  fi
+  print_failed_checks "$run_id"
+  echo "==> deploy succeeded, post-deploy smoke FAILED: see ${url}" >&2
+  return 22
+}
+
+rc=0
+wait_for_settled_roll || rc=$?
+[ "$rc" -eq 0 ] || exit "$rc"
+
+if [ "${DRY_RUN:-}" = "1" ]; then
+  echo "==> DRY_RUN=1: would dispatch: ${MANUAL_DISPATCH}"
+  newest="$(dispatch_run_ids | sort -n | tail -n 1 || true)"
+  if [ -n "$newest" ]; then
+    echo "    then pick the workflow_dispatch run that appears after it; the newest existing one is"
+    echo "    https://github.com/${REPO}/actions/runs/${newest} (not the one it would wait for)"
+  else
+    echo "    (no workflow_dispatch runs exist yet)"
+  fi
+  echo "    then wait for it and report its conclusion and failed checks. Nothing dispatched."
+  exit 0
+fi
+
+run_smoke || rc=$?
+exit "$rc"