hecks 2.5.1 → 2.7.0

data/lib/hecks/projections/deploy/scripts/smoke-after-deploy.sh.tmpl

@@ -0,0 +1,232 @@
+#!/usr/bin/env bash
+# GENERATED by bin/project_deploy -- re-run it to refresh this file rather
+# than hand-editing. Settings come from the domain's deployed_to("AwsFargate")
+# block; see Hecks::Projections::Deploy::Scripts for each key.
+#
+# Runs the production smoke once a roll of the ECS service has settled, so a
+# deploy ends with a real check of the live site instead of waiting for the
+# next scheduled run. Called at the end of deploy-service.sh; also runnable by
+# hand:
+#
+#   bash smoke-after-deploy.sh
+#
+# What it does, in order:
+#   1. Waits for the roll to settle: the stack is UPDATE_COMPLETE or
+#      CREATE_COMPLETE and the ECS service has a single PRIMARY, COMPLETED
+#      deployment with desired == running on the service's own task definition,
+#      on two consecutive checks one interval apart. Gives up after the timeout.
+#   2. Dispatches the smoke workflow on the configured ref, finds the run that
+#      dispatch created (a workflow_dispatch run that did not exist before the
+#      dispatch, never a scheduled one), waits for it, and prints its
+#      conclusion, the failed check names and its URL.
+#
+# It only ever READS AWS (describe-stacks, describe-services); the smoke itself
+# runs on GitHub against the live site.
+#
+# Exit codes (chosen to stay clear of the ones a failed deploy step produces:
+# 1 from bash or python, 2 from bash misuse, 254/255 from the aws CLI):
+#    0  the smoke passed, or was skipped on purpose (SKIP_POST_DEPLOY_SMOKE=1)
+#   20  the roll did not settle in time, or the stack ended in a failed state;
+#       the smoke was NOT run
+#   21  `gh` is missing or not signed in, or no repository or workflow is
+#       configured; the smoke was NOT run (the manual command is printed)
+#   22  the smoke ran and FAILED
+#   23  the smoke could not be dispatched, found or finished; result unknown
+#
+# Environment:
+#   SKIP_POST_DEPLOY_SMOKE=1   skip everything and say so
+#   DRY_RUN=1                  wait for the roll as usual, then print what would
+#                              be dispatched without dispatching anything
+#   SETTLE_TIMEOUT_SECS        default 1200
+#   SETTLE_CHECK_INTERVAL_SECS default 60
+#   SMOKE_RUN_TIMEOUT_SECS     default 1800, how long to wait for the run
+#   REPO                       GitHub repository (owner/name) holding the workflow
+#   WORKFLOW                   workflow file name
+#   SMOKE_REF                  ref to dispatch on
+set -euo pipefail
+
+#@@SETTINGS@@
+
+SETTLE_TIMEOUT_SECS="${SETTLE_TIMEOUT_SECS:-1200}"
+SETTLE_CHECK_INTERVAL_SECS="${SETTLE_CHECK_INTERVAL_SECS:-60}"
+SMOKE_RUN_TIMEOUT_SECS="${SMOKE_RUN_TIMEOUT_SECS:-1800}"
+MANUAL_COMMAND="bash smoke-after-deploy.sh"
+MANUAL_DISPATCH="gh workflow run ${WORKFLOW} --repo ${REPO} --ref ${SMOKE_REF}"
+
+if [ "${SKIP_POST_DEPLOY_SMOKE:-}" = "1" ]; then
+  echo "==> post-deploy smoke SKIPPED (SKIP_POST_DEPLOY_SMOKE=1)."
+  echo "    Run it yourself when ready: ${MANUAL_COMMAND}"
+  exit 0
+fi
+
+if [ -z "$REPO" ] || [ -z "$WORKFLOW" ]; then
+  echo "==> post-deploy smoke NOT RUN: no smoke repository or workflow is configured." >&2
+  echo "    Set REPO and WORKFLOW, or the smoke_repo and smoke_workflow settings, and re-run." >&2
+  exit 21
+fi
+
+if ! command -v gh >/dev/null 2>&1 || ! gh auth status >/dev/null 2>&1; then
+  echo "==> post-deploy smoke NOT RUN: the GitHub CLI (gh) is missing or not signed in." >&2
+  echo "    Run it by hand: ${MANUAL_DISPATCH}" >&2
+  echo "    or, once gh works: ${MANUAL_COMMAND}" >&2
+  exit 21
+fi
+
+stack_status() {
+  aws cloudformation describe-stacks --stack-name "$STACK" --region "$REGION" \
+    --query 'Stacks[0].StackStatus' --output text 2>/dev/null || true
+}
+
+# Prints "ok <task definition arn>" when the service looks settled, otherwise
+# "wait <why>".
+ecs_state() {
+  aws ecs describe-services --cluster "$CLUSTER" --services "$ECS_SERVICE" --region "$REGION" \
+    --query 'services[0].{td:taskDefinition,desired:desiredCount,running:runningCount,pending:pendingCount,deployments:deployments[].{status:status,rollout:rolloutState,td:taskDefinition}}' \
+    --output json 2>/dev/null | python3 -c '
+import json, sys
+try:
+    s = json.load(sys.stdin)
+except ValueError:
+    print("wait could not read the service"); raise SystemExit
+if not s:
+    print("wait service not found"); raise SystemExit
+deps = s.get("deployments") or []
+if len(deps) != 1:
+    print("wait %d deployments in flight" % len(deps)); raise SystemExit
+d = deps[0]
+if d.get("status") != "PRIMARY" or d.get("rollout") != "COMPLETED":
+    print("wait deployment is %s / %s" % (d.get("status"), d.get("rollout"))); raise SystemExit
+if d.get("td") != s.get("td"):
+    print("wait deployment task definition differs from the service"); raise SystemExit
+if not s.get("desired") or s.get("desired") != s.get("running") or s.get("pending"):
+    print("wait desired=%s running=%s pending=%s" % (s.get("desired"), s.get("running"), s.get("pending"))); raise SystemExit
+print("ok " + s["td"])
+' || echo "wait could not read the service"
+}
+
+wait_for_settled_roll() {
+  echo "==> waiting for the roll to settle (stack ${STACK}, ECS service ${ECS_SERVICE}; timeout ${SETTLE_TIMEOUT_SECS}s)"
+  local start=$SECONDS previous="" status ecs signature
+  while true; do
+    status="$(stack_status)"
+    case "$status" in
+      *ROLLBACK*|*FAILED*)
+        echo "==> roll did not settle: stack ${STACK} is ${status}. The smoke was NOT run." >&2
+        return 20 ;;
+    esac
+    if [ "$status" = "UPDATE_COMPLETE" ] || [ "$status" = "CREATE_COMPLETE" ]; then
+      ecs="$(ecs_state)"
+    else
+      ecs="wait stack is ${status:-unreadable}"
+    fi
+    if [ "${ecs%% *}" = "ok" ]; then
+      signature="${status} ${ecs#ok }"
+      if [ "$signature" = "$previous" ]; then
+        echo "    -> settled on ${ecs#ok } (two consecutive checks agree)"
+        return 0
+      fi
+      echo "    -> looks settled (${status}, ${ecs#ok }); confirming in ${SETTLE_CHECK_INTERVAL_SECS}s"
+      previous="$signature"
+    else
+      echo "    -> not yet: ${ecs#wait }"
+      previous=""
+    fi
+    if [ $((SECONDS - start + SETTLE_CHECK_INTERVAL_SECS)) -gt "$SETTLE_TIMEOUT_SECS" ]; then
+      echo "==> roll did not settle within ${SETTLE_TIMEOUT_SECS}s (last: ${ecs#wait }). The smoke was NOT run." >&2
+      return 20
+    fi
+    sleep "$SETTLE_CHECK_INTERVAL_SECS"
+  done
+}
+
+# Ids of the workflow_dispatch runs that exist now, one per line.
+dispatch_run_ids() {
+  gh run list --repo "$REPO" --workflow "$WORKFLOW" --event workflow_dispatch --limit 50 \
+    --json databaseId --jq '.[].databaseId' 2>/dev/null
+}
+
+print_failed_checks() {
+  local run_id="$1" log names
+  log="$(gh run view "$run_id" --repo "$REPO" --log 2>/dev/null || true)"
+  # Log lines are "<job>\t<step>\t<timestamp> <text>"; a smoke that prints
+  # "  <check>... FAILED: <why>" for each failing check is listed by name.
+  names="$(printf '%s\n' "$log" | awk -F'\t' 'NF >= 3 { print $3 }' | sed -E 's/^[0-9T:.-]+Z //' \
+    | grep -E '\.\.\. FAILED:' | sed -E 's/^ +//' || true)"
+  if [ -n "$names" ]; then
+    echo "    failed checks:"
+    printf '%s\n' "$names" | sed 's/^/      - /'
+  else
+    echo "    (no failed check names in the log; the run may have failed before the checks. Last log lines:)"
+    printf '%s\n' "$log" | tail -n 15 | sed 's/^/      /'
+  fi
+}
+
+run_smoke() {
+  local before run_id="" waited=0 url conclusion status
+  if ! before="$(dispatch_run_ids)"; then
+    echo "==> could not list the smoke workflow's runs, so it was NOT dispatched (it could not be told apart afterwards). Run it by hand: ${MANUAL_DISPATCH}" >&2
+    return 23
+  fi
+
+  echo "==> dispatching the production smoke: ${MANUAL_DISPATCH}"
+  if ! gh workflow run "$WORKFLOW" --repo "$REPO" --ref "$SMOKE_REF"; then
+    echo "==> could not dispatch the smoke workflow. Deploy succeeded; run it by hand: ${MANUAL_DISPATCH}" >&2
+    return 23
+  fi
+
+  # The dispatch call does not return the run, so find the workflow_dispatch
+  # run that was not there before it (scheduled runs are never candidates).
+  while [ -z "$run_id" ] && [ "$waited" -lt 180 ]; do
+    sleep 5
+    waited=$((waited + 5))
+    run_id="$(comm -13 <(printf '%s\n' "$before" | sort) <(dispatch_run_ids | sort) | sort -n | tail -n 1)"
+  done
+  if [ -z "$run_id" ]; then
+    echo "==> dispatched, but the new run never appeared. Deploy succeeded; check https://github.com/${REPO}/actions/workflows/${WORKFLOW}" >&2
+    return 23
+  fi
+  url="https://github.com/${REPO}/actions/runs/${run_id}"
+  echo "==> smoke run ${run_id}: ${url}"
+
+  waited=0
+  while true; do
+    status="$(gh run view "$run_id" --repo "$REPO" --json status --jq .status 2>/dev/null || true)"
+    [ "$status" = "completed" ] && break
+    if [ "$waited" -ge "$SMOKE_RUN_TIMEOUT_SECS" ]; then
+      echo "==> deploy succeeded, but the smoke run did not finish within ${SMOKE_RUN_TIMEOUT_SECS}s: see ${url}" >&2
+      return 23
+    fi
+    sleep 15
+    waited=$((waited + 15))
+  done
+
+  conclusion="$(gh run view "$run_id" --repo "$REPO" --json conclusion --jq .conclusion || true)"
+  echo "==> smoke conclusion: ${conclusion}  ${url}"
+  if [ "$conclusion" = "success" ]; then
+    echo "==> deploy succeeded, post-deploy smoke passed."
+    return 0
+  fi
+  print_failed_checks "$run_id"
+  echo "==> deploy succeeded, post-deploy smoke FAILED: see ${url}" >&2
+  return 22
+}
+
+rc=0
+wait_for_settled_roll || rc=$?
+[ "$rc" -eq 0 ] || exit "$rc"
+
+if [ "${DRY_RUN:-}" = "1" ]; then
+  echo "==> DRY_RUN=1: would dispatch: ${MANUAL_DISPATCH}"
+  newest="$(dispatch_run_ids | sort -n | tail -n 1 || true)"
+  if [ -n "$newest" ]; then
+    echo "    then pick the workflow_dispatch run that appears after it; the newest existing one is"
+    echo "    https://github.com/${REPO}/actions/runs/${newest} (not the one it would wait for)"
+  else
+    echo "    (no workflow_dispatch runs exist yet)"
+  fi
+  echo "    then wait for it and report its conclusion and failed checks. Nothing dispatched."
+  exit 0
+fi
+
+run_smoke || rc=$?
+exit "$rc"

data/lib/hecks/projections/deploy/scripts.rb

@@ -0,0 +1,171 @@
+require "shellwords"
+require_relative "scripts/settings"
+
+module Hecks
+  module Projections
+    module Deploy
+      # The hosting scripts an AWS Fargate deploy ships beside its template:
+      # the steps an operator runs after `make deploy` has created the stack.
+      # `Fargate.call` adds them when the domain's
+      # `deployed_to("AwsFargate")` block opts in, and leaves its file map
+      # untouched otherwise.
+      #
+      # ## What it emits
+      #
+      # - `deploy-service.sh` reads the service's active task definition,
+      #   pushes a local image under a never-reused tag, swaps one container's
+      #   image, and syncs that container's CloudFormation `*ImageTag`
+      #   parameter with `--use-previous-template`, checking afterwards that no
+      #   other parameter changed.
+      # - `smoke-after-deploy.sh` waits for the roll to settle, then dispatches
+      #   a GitHub workflow and reports its result. Exit codes 20 to 23 are its
+      #   own.
+      # - `hosting.mk` is included by the generated `Makefile`. It pins the
+      #   Hecks release the domain image is built from (`HECKS_ROOT` resolves
+      #   to a cached checkout of that release's tag, never to a path on one
+      #   machine) and adds the `deploy-service`, `smoke-after-deploy` and
+      #   `check-era` targets.
+      # - `expected-era` is the allow-list `bin/check_era` compares a host's
+      #   `GET /version` era against.
+      #
+      # ## Settings
+      #
+      # Read from `deployed_to("AwsFargate")`. Everything but `hosting_scripts`
+      # and `hecks_release` has a default; the ones the stack template already
+      # names (`stack_name`, `stack_prefix`, `region`) are not repeated here.
+      #
+      # - `hosting_scripts` opts in to every file above, when `true`. Off by default.
+      # - `hecks_release` is the Hecks release the domain image is built from,
+      #   such as `"2.5.1"`. Required.
+      # - `hecks_source` is the repository URL the release tag is fetched from.
+      #   Defaults to the Hecks repository.
+      # - `hecks_cache_dir` is where release checkouts are kept, a Make expression.
+      #   Defaults to `$(HOME)/.cache/hecks`.
+      # - `ecs_cluster` and `ecs_service` name the cluster and service. They default to the
+      #   names the generated template gives them: the stack name, or the `names` override.
+      # - `smoke_repo` is the GitHub `owner/name` holding the smoke workflow, and
+      #   `smoke_workflow` its file name. With neither set, `REPO` and `WORKFLOW` must
+      #   come from the environment.
+      # - `smoke_ref` is the ref the workflow is dispatched on. Defaults to `main`.
+      # - `expected_eras` lists the era ids `expected-era` holds. Defaults to none, so a
+      #   host is only checked to report an era.
+      # - `public_url` is the host URL `make check-era` requests. Without it, `URL` must
+      #   be passed to `make`.
+      #
+      # ## Names come from the template
+      #
+      # The containers, their ECR repositories and their image-tag parameters are not
+      # settings of the scripts. `deploy-service.sh` accepts exactly the containers the
+      # template's `Fargate::Containers::Layout` holds (the domain container, then each
+      # entry of the world's `containers`), and for each one pushes to the repository and
+      # syncs the parameter that layout names: the values the template renders. Rename them
+      # with `domain_container` and `containers` (see `Fargate`), and the scripts, and the
+      # `Makefile`'s `deploy`, follow.
+      module Scripts
+        SCRIPT_DIR = File.join(__dir__, "scripts").freeze
+
+        module_function
+
+        # Adds the hosting scripts to a Fargate file map when the domain opts in.
+        #
+        # @param files [Hash{String => String}] the Fargate target's own files, keyed by path
+        # @param deploy_settings [Hash{Symbol => Object}] the world's `deployed_to("AwsFargate")`
+        #   settings
+        # @param plan [Fargate::Settings::Plan] the template generator's resolved settings, the
+        #   source of every container, repository and parameter name
+        # @param stack_name [String] the CloudFormation stack name
+        # @param region [String] the validated AWS region
+        # @return [Hash{String => String}] `files` itself when the domain has not opted in, else
+        #   a copy with the scripts added and the `Makefile` including `hosting.mk`
+        # @raise [ArgumentError] if a hosting setting is missing or invalid; see `Settings`
+        def extend_files(files, deploy_settings:, plan:, stack_name:, region:)
+          return files unless deploy_settings[:hosting_scripts] == true
+
+          settings = Settings.new(deploy_settings: deploy_settings, plan: plan,
+                                  stack_name: stack_name, region: region)
+          files.merge(
+            "Makefile"              => "#{files.fetch('Makefile')}\ninclude hosting.mk\n",
+            "hosting.mk"            => hosting_mk(settings),
+            "deploy-service.sh"     => deploy_service_sh(settings),
+            "smoke-after-deploy.sh" => smoke_after_deploy_sh(settings),
+            "expected-era"          => expected_era(settings)
+          )
+        end
+
+        # Renders `deploy-service.sh`.
+        #
+        # @param settings [Settings] the checked hosting settings
+        # @return [String] the script
+        def deploy_service_sh(settings)
+          render("deploy-service.sh.tmpl", "SETTINGS" => stack_constants(settings),
+                                           "SERVICES" => services_block(settings))
+        end
+
+        # Renders `smoke-after-deploy.sh`.
+        #
+        # @param settings [Settings] the checked hosting settings
+        # @return [String] the script
+        def smoke_after_deploy_sh(settings)
+          smoke = [
+            "REPO=\"${REPO:-#{settings.smoke_repo}}\"",
+            "WORKFLOW=\"${WORKFLOW:-#{settings.smoke_workflow}}\"",
+            "SMOKE_REF=\"${SMOKE_REF:-#{settings.smoke_ref}}\""
+          ]
+          render("smoke-after-deploy.sh.tmpl", "SETTINGS" => [stack_constants(settings), *smoke].join("\n"))
+        end
+
+        # Renders the `expected-era` allow-list.
+        #
+        # @param settings [Settings] the checked hosting settings
+        # @return [String] the header comment followed by one era id per line
+        def expected_era(settings)
+          header = File.read(File.join(SCRIPT_DIR, "expected-era.header"))
+          "#{([header.chomp, ''] + settings.expected_eras).join("\n")}\n"
+        end
+
+        # Renders `hosting.mk`.
+        #
+        # @param settings [Settings] the checked hosting settings
+        # @return [String] the Makefile fragment
+        def hosting_mk(settings)
+          render("hosting.mk.tmpl", "RELEASE" => settings.hecks_release, "SOURCE" => settings.hecks_source,
+                                    "CACHE_DIR" => settings.hecks_cache_dir, "URL" => settings.public_url.to_s,
+                                    "SERVICE" => settings.containers.first.name)
+        end
+
+        def stack_constants(settings)
+          {
+            "REGION" => settings.region, "CLUSTER" => settings.cluster,
+            "ECS_SERVICE" => settings.service, "STACK" => settings.stack
+          }.map { |name, value| "#{name}=#{Shellwords.escape(value)}" }.join("\n")
+        end
+
+        def services_block(settings)
+          arms = settings.containers.map do |container|
+            repository = Shellwords.escape(container.repository_name)
+            parameter = Shellwords.escape(container.tag_parameter)
+            "    #{Shellwords.escape(container.name)}) ECR_REPOSITORY=#{repository}; CFN_PARAM_KEY=#{parameter} ;;"
+          end
+          <<~BASH.chomp
+            SERVICES='#{settings.containers.map(&:name).join(' ')}'
+
+            # Sets ECR_REPOSITORY and CFN_PARAM_KEY for one service, or exits.
+            resolve_service() {
+              case "$1" in
+            #{arms.join("\n")}
+                *) echo "unknown service '$1', must be one of: ${SERVICES}" >&2; exit 1 ;;
+              esac
+            }
+          BASH
+        end
+
+        def render(template, values)
+          text = File.read(File.join(SCRIPT_DIR, template))
+          values.reduce(text) { |out, (marker, value)| out.gsub(/#?@@#{marker}@@/) { value } }
+        end
+
+        private_class_method :stack_constants, :services_block, :render
+      end
+    end
+  end
+end