hecks 3.4.1 → 3.5.0

data/lib/hecks/hecks/hecks.hecksagon

@@ -105,6 +105,7 @@ Hecks.hecksagon "Hecks" do
     end
 
     answers_query "Check"
+    answers_query "Digest"
   end
 
   Hecks::Door.port "Workspace" do
@@ -400,6 +401,71 @@ Hecks.hecksagon "Hecks" do
     end
   end
 
+  Deploy::SmokeRun.port "DeployToolchain" do
+    asks "Smoke", to: SmokeRun do
+      attribute :run, RunKey
+      attribute :after, Origin, default: { value: "alone" }
+      answers "SmokeAnswered"
+      refuses "SmokeRefused"
+    end
+  end
+
+  Deploy::ServiceRoll.port "DeployToolchain" do
+    asks "RollService", to: ServiceRoll do
+      attribute :run, RunKey
+      answers "ServiceRollAnswered"
+      refuses "ServiceRollRefused"
+    end
+  end
+
+  Deploy::BoxRoll.port "DeployToolchain" do
+    asks "RollBox", to: BoxRoll do
+      attribute :run, RunKey
+      answers "BoxRollAnswered"
+      refuses "BoxRollRefused"
+    end
+  end
+
+  Deploy::DataCopy.port "DeployToolchain" do
+    asks "CopyData", to: DataCopy do
+      attribute :run, RunKey
+      answers "DataCopyAnswered"
+      refuses "DataCopyRefused"
+    end
+  end
+
+  Deploy::CopyVerification.port "DeployToolchain" do
+    asks "CompareCopy", to: CopyVerification do
+      attribute :run, RunKey
+      answers "CopyVerificationAnswered"
+      refuses "CopyVerificationRefused"
+    end
+  end
+
+  Deploy::BluebookDiff.port "DeployToolchain" do
+    asks "CompareBluebooks", to: BluebookDiff do
+      attribute :run, RunKey
+      answers "BluebookDiffAnswered"
+      refuses "BluebookDiffRefused"
+    end
+  end
+
+  Deploy::PreviewRun.port "DeployToolchain" do
+    asks "RunPreview", to: PreviewRun do
+      attribute :run, RunKey
+      answers "PreviewRunAnswered"
+      refuses "PreviewRunRefused"
+    end
+  end
+
+  Deploy::CompanionRoll.port "DeployToolchain" do
+    asks "RollCompanion", to: CompanionRoll do
+      attribute :run, RunKey
+      answers "CompanionRollAnswered"
+      refuses "CompanionRollRefused"
+    end
+  end
+
   Deploy::TemplateComparison.port "DeployToolchain" do
     asks "Compare", to: TemplateComparison do
       attribute :run, RunKey

data/lib/hecks/hecks/hecks.world

@@ -40,9 +40,14 @@ Hecks.world "Hecks" do
                        failure_states: %w[flagged failed drifted unreachable refused faulted halted stopped
                                           abandoned red needs_fix],
                        settled: %w[package.vendor package.revendor package.release
-                                  operation.bootstrap_admin],
+                                  operation.bootstrap_admin smoke_run.run
+                                  service_roll.run box_roll.run data_copy.restore
+                                  data_copy.verify bluebook_diff.run preview_run.name
+                                  preview_run.url preview_run.list preview_run.deploy
+                                  preview_run.destroy preview_run.login companion_roll.run],
                        names: { "mcp" => "door.serve_mcp", "console" => "operation.open_console", "init" => "door.init",
                                 "interview" => "door.interview", "package.check" => "registry.check",
+                                "package.digest" => "registry.digest",
                                 "package.release" => "registry.release" },
                        streams: %w[Follow],
                        executable: "exe/hecks",

data/lib/hecks/hecks/site.world

@@ -5,9 +5,11 @@
 # `run_keys` mints the `run` key of a projection, and `--wait` reports a faulted projection (a
 # routes.ts out of date under `--check`, or a route table the projector refused) as exit 1, and
 # `settled` makes `site_projection.check_live` wait for its reactions without the flag, so a live
-# distribution that differs from the project is an exit status.
+# distribution that differs from the project is an exit status. `report` prints that command's
+# answer as the report alone: the differences, or the line saying the distribution matches.
 Hecks.world "Site" do
   launcher "Launcher", run_keys: true,
                        failure_states: %w[faulted refused halted stopped abandoned red needs_fix],
-                       settled: %w[site_projection.check_live]
+                       settled: %w[site_projection.check_live],
+                       report: %w[site_projection.check_live]
 end

data/lib/hecks/projections/deploy/box/hosting.rb

@@ -33,8 +33,10 @@ module Hecks
           # @param plan [Settings::Plan] the resolved settings
           # @return [String] the Makefile fragment the box `Makefile` includes
           def hosting_mk(plan)
-            Box.template("hosting.mk.tmpl", "SERVICE" => plan.containers.first.name,
-                                            "URL"     => plan.hosting.public_url.to_s)
+            Box.template("hosting.mk.tmpl", "SERVICE"        => plan.containers.first.name,
+                                            "URL"            => plan.hosting.public_url.to_s,
+                                            "SMOKE_RECIPE"   => Box::SMOKE_RECIPE,
+                                            "SERVICE_RECIPE" => service_recipe)
           end
 
           # @param plan [Settings::Plan] the resolved settings
@@ -49,6 +51,14 @@ module Hecks
                          "ROLL_STEPS" => part(roll))
           end
 
+          # @return [String] the `deploy-service` recipe: the `service_roll.run` command
+          def service_recipe
+            args = "service=$(SERVICE) $(if $(EXISTING_TAG),existing_tag=$(EXISTING_TAG)) " \
+                   "$(if $(LOCAL_IMAGE),local_image=$(LOCAL_IMAGE))"
+            direct = "bash ./deploy-service.sh $(SERVICE)"
+            Box::RollRecipe.call(command: "service_roll.run", args: args, direct: direct)
+          end
+
           # @param plan [Settings::Plan] the resolved settings
           # @param region [String] the validated AWS region
           # @return [String] `smoke-after-deploy.sh`
@@ -147,7 +157,7 @@ module Hecks
               resolve_service() {
                 case "$1" in
               #{arms.join("\n")}
-                  *) echo "unknown service '$1', must be one of: ${SERVICES}" >&2; exit 1 ;;
+                  *) echo "unknown service '$1', must be one of: ${SERVICES}" >&2; exit 2 ;;
                 esac
               }
             BASH

data/lib/hecks/projections/deploy/box/roll_recipe.rb

@@ -0,0 +1,62 @@
+module Hecks
+  module Projections
+    module Deploy
+      module Box
+        # The Makefile recipes that run a roll as `box_roll.run` or `service_roll.run`.
+        #
+        # The command rolls with the generated script and, through its policy, requests the
+        # smoke, so one deploy leaves a `BoxRoll` or `ServiceRoll` (which also records the smoke's
+        # outcome) and a `SmokeRun` in the Hecks database. A failed roll or smoke is the command's
+        # exit 1, with the script's status in its message. With no database the command never
+        # starts, so the recipe runs the script itself, states the database error apart from the
+        # deploy's result, and fails (`Error 24` if all passed).
+        module RollRecipe
+          module_function
+
+          # @param plan [Settings::Plan] the resolved settings
+          # @return [String] the `deploy` target's recipe, tab-indented: the `box_roll.run` command
+          def deploy(plan)
+            variable = plan.task_definition ? "TASKDEF" : "TAGS"
+            args = plan.task_definition ? "$(if $(TASKDEF),taskdef=$(TASKDEF))" : "$(if $(TAGS),tags=\"$(TAGS)\")"
+            direct = "bash ./deploy-box.sh $(#{variable})"
+            direct += " || exit $$?; TASKDEF=\"$(TASKDEF)\" bash ./smoke-after-deploy.sh" if plan.hosting
+            "\t#{call(command: 'box_roll.run', args: args, direct: direct)}"
+          end
+
+          # @param command [String] the Deploy command, such as `box_roll.run`
+          # @param args [String] the Makefile text that turns variables into the arguments
+          # @param direct [String] the shell that runs the script when there is no database
+          # @return [String] the recipe lines, joined for a Makefile rule
+          def call(command:, args:, direct:)
+            [*run_lines(command, args), *no_database_lines(direct), *outcome_lines].join("\n\t").freeze
+          end
+
+          def run_lines(command, args)
+            ["@err=$$(mktemp); run=deploy-$$(date -u +%Y%m%d%H%M%S); \\",
+             "$(HECKS) deploy #{command} project=\"$(CURDIR)\" run=\"$$run\" #{args} \\",
+             '  $(if $(SKIP_POST_DEPLOY_SMOKE),skip_smoke=true) --wait 2>"$$err"; rc=$$?; \\']
+          end
+
+          def no_database_lines(direct)
+            ['if [ $$rc -ne 0 ] && grep -q \'^cannot open Hecks\' "$$err"; then \\',
+             '  echo "==> the deploy record was NOT written: $$(grep -m1 \'^cannot open Hecks\' "$$err")" >&2; \\',
+             '  echo "    one-time setup on this machine: a database AND a non-superuser role that owns it" >&2; \\',
+             '  echo "    (a superuser skips the era write-fence, so createdb alone fails), then set" >&2; \\',
+             '  echo "    HECKS_DATABASE=postgres://<role>@localhost/<db>" >&2; \\',
+             '  echo "    deploying anyway so the deploy is not lost" >&2; \\',
+             "  rm -f \"$$err\"; #{direct}; rc=$$?; \\",
+             '  echo "==> deploy exit $$rc. The deploy record was NOT written: the database is unavailable (see above)." >&2; \\',
+             "  [ $$rc -ne 0 ] || rc=24; exit $$rc; \\",
+             "fi; \\"]
+          end
+
+          def outcome_lines
+            ['cat "$$err" >&2; rm -f "$$err"; exit $$rc']
+          end
+
+          private_class_method :run_lines, :no_database_lines, :outcome_lines
+        end
+      end
+    end
+  end
+end

data/lib/hecks/projections/deploy/box/templates/deploy-box.sh.tmpl

@@ -2,6 +2,13 @@
 # Roll @@STACK@@'s app box: render the compose files, push them over SSM, start the containers and check
 # them. Exits non-zero if the roll fails or the box does not answer as expected.
 #
+# Exit codes (the ones `hecks deploy box_roll.run` names; a step that fails under `set -e` ends with its
+# own status, 1 from bash and 254/255 from the aws CLI):
+#   0   the roll succeeded and the box answered healthy
+#   40  the box stack has no instance
+#   41  the roll did not succeed on the box
+#   42  the roll succeeded but the box is not healthy after it
+#
 @@USAGE@@
 set -euo pipefail
 HERE=$(cd "$(dirname "$0")" && pwd)
@@ -14,7 +21,7 @@ out() { aws cloudformation describe-stacks --stack-name "$1" --query "Stacks[0].
 BOX=$(out "$BOX_STACK" InstanceId)
 DB_HOST=$(out "$RDS_STACK" DbEndpoint)
 DB_SECRET=$(out "$RDS_STACK" DbSecretArn)
-[ -n "$BOX" ] && [ "$BOX" != None ] || { echo "no instance in stack $BOX_STACK" >&2; exit 1; }
+[ -n "$BOX" ] && [ "$BOX" != None ] || { echo "no instance in stack $BOX_STACK" >&2; exit 40; }
 ACCOUNT=$(aws sts get-caller-identity --query Account --output text)
 
 WORK=$(mktemp -d); trap 'rm -rf "$WORK"' EXIT
@@ -68,7 +75,7 @@ ROLL=$(jq -n --arg compose "$(B64 "$WORK/compose.json")" --arg secrets "$(B64 "$
    executionTimeout: ["900"]}')
 
 echo "==> rolling $BOX_STACK ($BOX) at $(date -u +%H:%M:%SZ)"
-run_on_box "$ROLL" || { echo "==> the roll did not succeed" >&2; exit 1; }
+run_on_box "$ROLL" || { echo "==> the roll did not succeed" >&2; exit 41; }
 
 echo "==> checking the box"
 CHECK=$(cat <<'CHECK_EOF'
@@ -81,5 +88,5 @@ if [ -z "$DOWN" ]; then echo "ok   every container is up"; else echo "FAIL conta
 CHECK_EOF
 )
 CHECK_JSON=$(jq -n --arg c "$(printf '%s' "$CHECK" | base64 | tr -d '\n')" '{commands: ["echo \($c) | base64 -d | bash"]}')
-run_on_box "$CHECK_JSON" || { echo "==> the box is NOT healthy after the roll" >&2; exit 1; }
+run_on_box "$CHECK_JSON" || { echo "==> the box is NOT healthy after the roll" >&2; exit 42; }
 echo "==> box roll done ($(date -u +%H:%M:%SZ))"

data/lib/hecks/projections/deploy/box/templates/deploy-service.services-roll.part

@@ -2,7 +2,7 @@
 # container runs now, read from the box's own Compose file, so rolling one service leaves the rest alone.
 BOX_ID="$(aws cloudformation describe-stacks --stack-name "$BOX_STACK" --region "$REGION" \
   --query "Stacks[0].Outputs[?OutputKey=='InstanceId'].OutputValue" --output text)"
-[ -n "$BOX_ID" ] && [ "$BOX_ID" != None ] || { echo "no instance in stack ${BOX_STACK}" >&2; exit 1; }
+[ -n "$BOX_ID" ] && [ "$BOX_ID" != None ] || { echo "no instance in stack ${BOX_STACK}" >&2; exit 33; }
 
 echo "==> reading the tags the box runs now"
 READ_JSON="$(jq -n --arg c "cat ${BOX_DIR}/compose.json 2>/dev/null || echo '{}'" '{commands: [$c]}')"
@@ -14,7 +14,7 @@ for _ in $(seq 1 30); do
     --query Status --output text 2>/dev/null || echo Pending)"
   [ "$state" != InProgress ] && [ "$state" != Pending ] && break
 done
-[ "$state" = Success ] || { echo "could not read the box's Compose file (${state}); nothing was rolled" >&2; exit 1; }
+[ "$state" = Success ] || { echo "could not read the box's Compose file (${state}); nothing was rolled" >&2; exit 32; }
 aws ssm get-command-invocation --command-id "$COMMAND_ID" --instance-id "$BOX_ID" --region "$REGION" \
   --query StandardOutputContent --output text > "${SCRATCH_DIR}/compose.json"
 

data/lib/hecks/projections/deploy/box/templates/deploy-service.sh.tmpl

@@ -14,13 +14,24 @@
 @@MODE_SUMMARY@@
 #
 # It ends with smoke-after-deploy.sh, whose own exit codes (20-23) are distinct from a failed deploy
-# step's. SKIP_POST_DEPLOY_SMOKE=1 skips it.
+# step's. SKIP_POST_DEPLOY_SMOKE=1 skips it. `hecks deploy service_roll.run` runs this script with
+# SMOKE_BY_COMMAND=1, which ends after the roll and leaves the smoke to the smoke_run.run its policy requests.
+#
+# Exit codes (the ones `service_roll.run` names; a step that fails under `set -e` ends with its own
+# status, 1 from bash and 254/255 from the aws CLI):
+#   2   unknown service
+#   30  EXISTING_TAG is not in ECR              31  the fresh tag is already in ECR
+#   32  the box's Compose file is unreadable    33  the box stack has no instance
+#   34  the stack has no parameter for the container   35  the stack update failed or did not settle
+#   36  a parameter other than the container's changed  37  the task definition lacks the pushed image
+#   38  the task definition has no such container      40-42  deploy-box.sh's own
 #
 # Environment:
 #   LOCAL_IMAGE               local image to push, default <ecr repository>:latest
 #   EXISTING_TAG              redeploy this tag from ECR instead of building and pushing
 #   SETTLE_TIMEOUT_SECS       default 1200, how long to wait for a stack update
 #   SKIP_POST_DEPLOY_SMOKE=1  passed through to smoke-after-deploy.sh
+#   SMOKE_BY_COMMAND=1        end after the roll and print `rolled taskdef=<td> tag=<tag>`; the caller smokes
 set -euo pipefail
 
 @@SETTINGS@@
@@ -49,12 +60,12 @@ if [ -n "${EXISTING_TAG:-}" ]; then
   echo "==> reusing ${IMAGE_URI} (no push)"
   aws ecr describe-images --repository-name "$ECR_REPOSITORY" --region "$REGION" \
     --image-ids "imageTag=${TAG}" --query 'imageDetails[0].imageTags' --output text > /dev/null \
-    || { echo "tag ${TAG} is not in ECR" >&2; exit 1; }
+    || { echo "tag ${TAG} is not in ECR" >&2; exit 30; }
 else
   if aws ecr describe-images --repository-name "$ECR_REPOSITORY" --region "$REGION" \
       --image-ids "imageTag=${TAG}" > /dev/null 2>&1; then
     echo "tag ${TAG} is already in ECR; a tag is never reused. Run again." >&2
-    exit 1
+    exit 31
   fi
   echo "==> tagging ${LOCAL_IMAGE} as ${IMAGE_URI}"
   docker tag "$LOCAL_IMAGE" "$IMAGE_URI"
@@ -74,6 +85,7 @@ fi
 
 @@ROLL_STEPS@@
 
-# The deploy itself is done. End with the smoke, which reports separately.
+# The deploy itself is done. End with the smoke, which reports separately, unless the caller runs it.
 echo "==> deploy succeeded"
+[ -z "${SMOKE_BY_COMMAND:-}" ] || { echo "==> rolled taskdef=${TASKDEF:-} tag=${TAG}"; exit 0; }
 bash ./smoke-after-deploy.sh

data/lib/hecks/projections/deploy/box/templates/deploy-service.taskdef-roll.part

@@ -17,7 +17,7 @@ for key in $ALL_PARAM_KEYS; do
 done
 if [ "$found" != "1" ]; then
   echo "stack ${IMAGE_STACK} has no parameter named ${CFN_PARAM_KEY}; the image was pushed but nothing was rolled" >&2
-  exit 1
+  exit 34
 fi
 
 if UPDATE_OUT="$(aws cloudformation update-stack --stack-name "$IMAGE_STACK" --region "$REGION" \
@@ -32,11 +32,11 @@ if UPDATE_OUT="$(aws cloudformation update-stack --stack-name "$IMAGE_STACK" --r
       UPDATE_COMPLETE) echo "    -> ${stack_status}"; break ;;
       *ROLLBACK*|*FAILED*)
         echo "    -> ${stack_status}: the stack update failed, investigate before rolling anything" >&2
-        exit 1 ;;
+        exit 35 ;;
     esac
     if timed_out "$started"; then
       echo "==> the stack update did not settle within ${SETTLE_TIMEOUT_SECS}s (last: ${stack_status})" >&2
-      exit 1
+      exit 35
     fi
     sleep 10
   done
@@ -44,7 +44,7 @@ elif echo "$UPDATE_OUT" | grep -q "No updates are to be performed"; then
   echo "    -> the stack already has ${TAG}; nothing to update"
 else
   echo "$UPDATE_OUT" >&2
-  exit 1
+  exit 35
 fi
 
 echo "==> checking that only ${CFN_PARAM_KEY} changed"
@@ -56,7 +56,7 @@ CHANGED="$(jq -rn --slurpfile before "${SCRATCH_DIR}/params-before.json" --slurp
 NOW="$(jq -r --arg k "$CFN_PARAM_KEY" '.[] | select(.k == $k) | .v' "${SCRATCH_DIR}/params-after.json")"
 if [ "$NOW" != "$TAG" ] || { [ -n "$CHANGED" ] && [ "$CHANGED" != "$CFN_PARAM_KEY" ]; }; then
   echo "expected only ${CFN_PARAM_KEY} to change to ${TAG}; it is ${NOW} and changed: ${CHANGED:-nothing}" >&2
-  exit 1
+  exit 36
 fi
 echo "    -> ${CFN_PARAM_KEY} is ${TAG}; no other parameter changed"
 
@@ -69,7 +69,7 @@ RUNNING_IMAGE="$(aws ecs describe-task-definition --task-definition "$TD" --regi
   --query "taskDefinition.containerDefinitions[?name=='${SERVICE_NAME}'].image | [0]" --output text)"
 if [ "$RUNNING_IMAGE" != "$IMAGE_URI" ]; then
   echo "task definition ${TD} has ${RUNNING_IMAGE}, not ${IMAGE_URI}; refusing to roll" >&2
-  exit 1
+  exit 37
 fi
 echo "==> task definition ${TD} carries ${SERVICE_NAME}=${TAG}"
 

data/lib/hecks/projections/deploy/box/templates/deploy-service.taskdef.part

@@ -6,7 +6,7 @@ active_image() {
 }
 CURRENT_IMAGE="$(active_image)"
 case "$CURRENT_IMAGE" in
-  None|"") echo "task definition family ${FAMILY} has no container named ${SERVICE_NAME}" >&2; exit 1 ;;
+  None|"") echo "task definition family ${FAMILY} has no container named ${SERVICE_NAME}" >&2; exit 38 ;;
 esac
 ECR_REPOSITORY="${CURRENT_IMAGE#*/}"
 ECR_REPOSITORY="${ECR_REPOSITORY%%[:@]*}"

data/lib/hecks/projections/deploy/box/templates/hosting.mk.tmpl

@@ -8,15 +8,21 @@ URL     ?= @@URL@@
 
 .PHONY: deploy-service smoke-after-deploy check-era
 
-# Pushes one container's already-built local image under a fresh tag and rolls it onto the box.
-# Build the image first; see deploy-service.sh.
+# Pushes one container's already-built local image under a fresh tag and rolls it onto the box, as the
+# service_roll.run command: it runs deploy-service.sh, records the roll in the Hecks database
+# (HECKS_DATABASE, default postgres://hecks@localhost/hecks) and requests the smoke. Build the image
+# first; see deploy-service.sh. EXISTING_TAG=<tag> redeploys a tag already in ECR. Without the
+# database the script runs by itself and the target exits non-zero (24 if it all passed).
 deploy-service:
-	bash ./deploy-service.sh $(SERVICE)
+	@@SERVICE_RECIPE@@
 
-# Waits for the last roll to settle, then dispatches and follows the smoke.
+# Waits for the last roll to settle, then dispatches and follows the smoke, as the smoke_run.run
+# command, which runs smoke-after-deploy.sh and records how it ended in the Hecks database
+# (HECKS_DATABASE, default postgres://hecks@localhost/hecks). Without that database the smoke still
+# runs and its result is printed, and the target exits non-zero (24 if the smoke passed).
 # SKIP_POST_DEPLOY_SMOKE=1 opts out, DRY_RUN=1 dispatches nothing.
 smoke-after-deploy:
-	bash ./smoke-after-deploy.sh
+	@@SMOKE_RECIPE@@
 
 # Compares the era a live host reports at GET /version with expected-era.
 check-era:

data/lib/hecks/projections/deploy/box/templates/restore-to-rds.sh.tmpl

@@ -9,6 +9,10 @@
 #             direction (swap the hosts and secrets, and SRC_DB and DST_DB)
 #   SRC_DB    the source database (default @@SOURCE_DATABASE@@)
 #   DST_DB    the target database (default @@DATABASE@@)
+#   VERIFY_BY_COMMAND=1   end after the copy and leave the comparison to `hecks deploy data_copy.restore`
+#
+# Ends 60 (a client older than 16), 61 (the target already has a schema), 62 (unexpected restore
+# errors), or, run by hand, verify-copy.sh's 50 (the databases differ).
 #
 # Schemas copied: @@SCHEMAS@@
 #
@@ -34,7 +38,7 @@ if [ "$(pg_major pg_dump || echo 0)" -lt 16 ] && [ -x /opt/homebrew/opt/postgres
   export PATH=/opt/homebrew/opt/postgresql@16/bin:$PATH
 fi
 for t in pg_dump pg_restore psql; do
-  [ "$(pg_major $t || echo 0)" -ge 16 ] || { echo "need $t 16 or newer on the PATH" >&2; exit 1; }
+  [ "$(pg_major $t || echo 0)" -ge 16 ] || { echo "need $t 16 or newer on the PATH" >&2; exit 60; }
 done
 
 WORK=$(mktemp -d)
@@ -66,7 +70,7 @@ echo "source $(SRC 'show server_version'), target $(DST 'show server_version')"
 for S in $SCHEMAS; do
   if [ "$(DST "select count(*) from pg_namespace where nspname='$S'")" != 0 ]; then
     if [ "${FORCE:-}" = 1 ]; then DST "drop schema \"$S\" cascade" >/dev/null; echo "dropped target schema $S"
-    else echo "target already has schema $S; re-run with FORCE=1 to replace it" >&2; exit 1; fi
+    else echo "target already has schema $S; re-run with FORCE=1 to replace it" >&2; exit 61; fi
   fi
 done
 
@@ -77,7 +81,7 @@ for S in $SCHEMAS; do
   TOTAL=$(grep -c 'error:' "$WORK/err_$S.txt" || true)
   UNEXPECTED=$(grep 'error:' "$WORK/err_$S.txt" | grep -vc 'hecks_tr_extract' || true)
   echo "   restore errors: $TOTAL (unexpected: $UNEXPECTED)"
-  if [ "$UNEXPECTED" != 0 ]; then grep 'error:' "$WORK/err_$S.txt" | grep -v 'hecks_tr_extract' | head -5 >&2; exit 1; fi
+  if [ "$UNEXPECTED" != 0 ]; then grep 'error:' "$WORK/err_$S.txt" | grep -v 'hecks_tr_extract' | head -5 >&2; exit 62; fi
 done
 
 # Refresh what pg_restore could not (with the schema on the search_path).
@@ -88,4 +92,5 @@ for S in $SCHEMAS; do
 done
 
 # Same structure, same exact row counts in every table, nothing unpopulated.
+[ -z "${VERIFY_BY_COMMAND:-}" ] || exit 0
 A_DB=$SRC_DB B_DB=$DST_DB bash "$HERE/verify-copy.sh" "$BASTION" "$SRC_HOST" "$SRC_SECRET" "$DST_HOST" "$DST_SECRET"

data/lib/hecks/projections/deploy/box/templates/verify-copy.sh.tmpl

@@ -8,7 +8,7 @@
 #   A_DB, B_DB   the database on each side (default @@DATABASE@@)
 #
 # Schemas compared: @@SCHEMAS@@
-# Exit 0 only if both are identical and no materialized view is unpopulated in either.
+# Exit 0 only if both are identical and no materialized view is unpopulated in either; 50 when they differ.
 set -euo pipefail
 
 BASTION=${1:?bastion instance id}; A_HOST=${2:?}; A_SECRET=${3:?}; B_HOST=${4:?}; B_SECRET=${5:?}
@@ -54,5 +54,5 @@ A "$COUNTS" >"$WORK/counts_a"; B "$COUNTS" >"$WORK/counts_b"
 [ "$(A "$UNPOP")" = 0 ] && [ "$(B "$UNPOP")" = 0 ] || { echo "FAIL: unpopulated materialized views" >&2; BAD=1; }
 diff "$WORK/shape_a" "$WORK/shape_b" >/dev/null || { echo "FAIL: structure differs" >&2; diff "$WORK/shape_a" "$WORK/shape_b" >&2 || true; BAD=1; }
 diff "$WORK/counts_a" "$WORK/counts_b" >/dev/null || { echo "FAIL: row counts differ" >&2; diff "$WORK/counts_a" "$WORK/counts_b" >&2 || true; BAD=1; }
-[ "$BAD" = 0 ] || exit 1
+[ "$BAD" = 0 ] || exit 50
 echo "OK: $(wc -l <"$WORK/counts_a" | tr -d ' ') tables, identical row counts and structure"

data/lib/hecks/projections/deploy/box.rb

@@ -2,6 +2,7 @@ require "json"
 require_relative "../../projector"
 require_relative "box/settings"
 require_relative "box/hosting"
+require_relative "box/roll_recipe"
 
 module Hecks
   module Projections
@@ -28,6 +29,27 @@ module Hecks
                       "# after adding one: the admin API is off, so a reload cannot reach it.\n" \
                       "import /etc/caddy/extra/*\n".freeze
 
+        # The `smoke-after-deploy` recipe: the smoke runs as `smoke_run.run`, which finds
+        # `smoke-after-deploy.sh` beside the Makefile and records how it ended in the Hecks
+        # database. When that database cannot be opened the command never starts, so the recipe
+        # runs the script itself, states the database error apart from the smoke's result, and
+        # exits non-zero (24 when the smoke passed).
+        SMOKE_RECIPE = [
+          "@err=$$(mktemp); \\",
+          '$(HECKS) deploy smoke_run.run project="$(CURDIR)" --wait $(if $(TASKDEF),taskdef=$(TASKDEF)) 2>"$$err"; rc=$$?; \\',
+          'if [ $$rc -ne 0 ] && grep -q \'^cannot open Hecks\' "$$err"; then \\',
+          '  echo "==> the deploy record was NOT written: $$(grep -m1 \'^cannot open Hecks\' "$$err")" >&2; \\',
+          '  echo "    one-time setup on this machine: a database AND a non-superuser role that owns it" >&2; \\',
+          '  echo "    (a superuser skips the era write-fence, so createdb alone fails), then set" >&2; \\',
+          '  echo "    HECKS_DATABASE=postgres://<role>@localhost/<db>" >&2; \\',
+          '  echo "    running the smoke anyway so its result is not lost" >&2; \\',
+          '  rm -f "$$err"; TASKDEF="$(TASKDEF)" bash ./smoke-after-deploy.sh; rc=$$?; \\',
+          '  echo "==> smoke exit $$rc. The deploy record was NOT written: the database is unavailable (see above)." >&2; \\',
+          "  [ $$rc -ne 0 ] || rc=24; exit $$rc; \\",
+          "fi; \\",
+          'cat "$$err" >&2; rm -f "$$err"; exit $$rc'
+        ].join("\n\t").freeze
+
         module_function
 
         # Generates `rds.yaml`, `box.yaml`, `Caddyfile`, `services.json`, `render-compose.sh`,
@@ -557,6 +579,8 @@ module Hecks
             # Elastic IP. The default is a production pair, with deletion protection.
             REHEARSAL ?= false
 
+            HECKS ?= hecks
+
             .PHONY: stacks deploy
             stacks:
             \taws cloudformation deploy --template-file rds.yaml --stack-name $(RDS_STACK) --capabilities CAPABILITY_IAM \\
@@ -569,20 +593,9 @@ module Hecks
             \t\tRehearsal=$(REHEARSAL)
 
             deploy:
-            \tbash ./deploy-box.sh $(#{plan.task_definition ? 'TASKDEF' : 'TAGS'})
+            #{RollRecipe.deploy(plan)}
           MAKE
-          "#{base}#{hosting_makefile_tail(plan)}"
-        end
-
-        # With hosting scripts, a deploy ends with the post-deploy smoke and the Makefile includes
-        # the fragment that adds the hosting targets.
-        #
-        # @param plan [Settings::Plan] the resolved settings
-        # @return [String] the Makefile lines that follow the `deploy` recipe, or nothing
-        def hosting_makefile_tail(plan)
-          return "" unless plan.hosting
-
-          "\t#{'TASKDEF="$(TASKDEF)" ' if plan.task_definition}bash ./smoke-after-deploy.sh\n\ninclude hosting.mk\n"
+          plan.hosting ? "#{base}\ninclude hosting.mk\n" : base
         end
 
         # Fills `@@NAME@@` markers. A marker alone on its line is replaced together with the line,