mandrel-platform 1.1.0 → 1.3.0

package/scripts/stryker-base-config.test.mjs

@@ -0,0 +1,256 @@
+#!/usr/bin/env node
+/**
+ * stryker-base-config.test.mjs — regression guard for the shared Stryker base
+ * config's bail and timeout shape.
+ *
+ * The bug this pins: with Stryker's default bail (`disableBail: false`), the
+ * vitest runner can mark a mutant Survived having completed zero tests — the
+ * mutant carries a non-empty covering-test list while its completed-test count
+ * is zero. A consumer measured 92 of 345 mutants flipping verdict between two
+ * identical runs, and a recorded score of 53.5% against a real 72.29%. A
+ * committed baseline taken under bail is therefore a floor under a number
+ * nothing measured: the gate is not merely noisy, it is wrong in the direction
+ * that hides surviving mutants.
+ *
+ * Disabling bail is not free — every mutant now runs its full covering set, so
+ * the run lengthens. The timeouts in this same base config must move with it,
+ * or the suite trades a wrong number for a silent overrun, and an overrun that
+ * preserves the prior result and exits clean is the same failure wearing a
+ * different mask. That coupling is why bail and the timeouts are asserted
+ * together here rather than in two independent tests: re-tightening either half
+ * alone reintroduces the defect.
+ *
+ * This repository ships the config but runs no mutation suite of its own, so
+ * the contract is asserted by shape. The run-to-run stability it buys is
+ * observable only in a consumer.
+ *
+ * The delivery half is asserted end-to-end rather than by shape: the suite
+ * imports the base through its published package specifier — the same
+ * resolution a consumer's `stryker.config.mjs` performs — and checks the
+ * settings that arrive. Stryker has no `extends` option (adjudicated against
+ * @stryker-mutator/core and @stryker-mutator/api 9.6.1), so the spread import
+ * is the only mechanism that delivers anything at all, and it is the only one
+ * documented.
+ *
+ * Run: node --test scripts/stryker-base-config.test.mjs
+ */
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { readFileSync } from "node:fs";
+
+const CONFIG_PATH = "config/stryker.base.json";
+const PACKAGE_PATH = "package.json";
+
+/**
+ * The specifier a consumer's `stryker.config.mjs` imports. Node resolves it
+ * through this package's own `exports` map (self-reference), so importing it
+ * here exercises the same resolution a consumer gets rather than a stand-in
+ * for it.
+ */
+const PACKAGE_SPECIFIER = "mandrel-platform/stryker.base.json";
+
+/** Stryker's own defaults, per https://stryker-mutator.io/docs/stryker-js/configuration. */
+const STRYKER_DEFAULTS = Object.freeze({
+  timeoutMS: 5000,
+  timeoutFactor: 1.5,
+  dryRunTimeoutMinutes: 5,
+});
+
+/**
+ * The timeout floor this config carried *before* bail was disabled. Bail
+ * cut every mutant's run short, so 60s absolute was survivable; without it the
+ * covering set runs to completion and the budget has to grow. Asserting
+ * strictly above the old value is what makes "raised alongside the bail
+ * change" mechanically checkable rather than a claim in a commit message.
+ */
+const PRE_CHANGE_TIMEOUT_MS = 60000;
+
+function readConfig() {
+  return JSON.parse(readFileSync(CONFIG_PATH, "utf8"));
+}
+
+test("base config disables bail so every mutant runs its full covering set", () => {
+  const config = readConfig();
+
+  assert.equal(
+    config.disableBail,
+    true,
+    `${CONFIG_PATH} must set "disableBail": true. Stryker defaults it to false, ` +
+      "and under bail the runner can score a mutant Survived having completed " +
+      "zero of its covering tests.",
+  );
+});
+
+test("timeouts are raised above both Stryker's defaults and the pre-change floor", async (t) => {
+  const config = readConfig();
+
+  await t.test("timeoutMS clears the pre-bail-change floor", () => {
+    assert.equal(
+      typeof config.timeoutMS,
+      "number",
+      `${CONFIG_PATH} must pin "timeoutMS" explicitly, not inherit it.`,
+    );
+    assert.ok(
+      config.timeoutMS > PRE_CHANGE_TIMEOUT_MS,
+      `"timeoutMS" is ${config.timeoutMS}; it must exceed the pre-change ` +
+        `${PRE_CHANGE_TIMEOUT_MS} because disabling bail lengthens every mutant's run.`,
+    );
+  });
+
+  await t.test("timeoutFactor is pinned above Stryker's default", () => {
+    assert.equal(
+      typeof config.timeoutFactor,
+      "number",
+      `${CONFIG_PATH} must pin "timeoutFactor" explicitly, not inherit it.`,
+    );
+    assert.ok(
+      config.timeoutFactor > STRYKER_DEFAULTS.timeoutFactor,
+      `"timeoutFactor" is ${config.timeoutFactor}; it must exceed Stryker's ` +
+        `${STRYKER_DEFAULTS.timeoutFactor} default so a full covering set is not ` +
+        "clipped as a false Timeout.",
+    );
+  });
+
+  await t.test("dryRunTimeoutMinutes is pinned above Stryker's default", () => {
+    assert.equal(
+      typeof config.dryRunTimeoutMinutes,
+      "number",
+      `${CONFIG_PATH} must pin "dryRunTimeoutMinutes" explicitly, not inherit it.`,
+    );
+    assert.ok(
+      config.dryRunTimeoutMinutes > STRYKER_DEFAULTS.dryRunTimeoutMinutes,
+      `"dryRunTimeoutMinutes" is ${config.dryRunTimeoutMinutes}; it must exceed ` +
+        `Stryker's ${STRYKER_DEFAULTS.dryRunTimeoutMinutes}-minute default.`,
+    );
+  });
+});
+
+test("a Timeout is not silently absorbed into the score", () => {
+  const config = readConfig();
+
+  // `ignoreStatic` legitimately drops static mutants from the denominator.
+  // Nothing else may: an option that reclassifies or suppresses a timed-out or
+  // errored mutant would let a run that overran still report the prior number
+  // and exit clean — the exact failure disabling bail is meant to end.
+  assert.equal(
+    Object.hasOwn(config, "maxTestRunnerReuse"),
+    false,
+    'Do not pin "maxTestRunnerReuse" in the shared base; it masks runner-level ' +
+      "instability that the timeout budget is supposed to surface.",
+  );
+  assert.notEqual(
+    config.allowEmpty,
+    true,
+    '"allowEmpty" must stay false/absent: a dry run that executed no tests must ' +
+      "fail loudly rather than score an empty suite.",
+  );
+});
+
+test("the documented spread-import mechanism delivers the bail-free settings", async () => {
+  // This is the consumer's own resolution path, not a proxy for it: the
+  // specifier below is resolved through the published `exports` map by Node,
+  // exactly as `stryker.config.mjs` in a consuming repo resolves it. Asserting
+  // the exports-map *string* instead would pass while the file it points at
+  // carried the wrong values, or while the entry was absent from `files` — the
+  // two ways the recipe can be documented correctly and still deliver nothing.
+  const { default: base } = await import(PACKAGE_SPECIFIER, {
+    with: { type: "json" },
+  });
+
+  assert.equal(
+    base.disableBail,
+    true,
+    `Importing "${PACKAGE_SPECIFIER}" must yield "disableBail": true. A consumer ` +
+      "spreading this object into stryker.config.mjs gets whatever this " +
+      "resolves to, so a broken export or a stale published file silently " +
+      "restores bail.",
+  );
+  assert.ok(
+    base.timeoutMS > PRE_CHANGE_TIMEOUT_MS,
+    `Importing "${PACKAGE_SPECIFIER}" must also carry the raised timeout budget ` +
+      `that bail-free runs need; got timeoutMS ${base.timeoutMS}.`,
+  );
+
+  // The exports map must reach *this* file, or the assertions in the rest of
+  // this suite are guarding a config no consumer receives.
+  assert.deepEqual(
+    base,
+    readConfig(),
+    `"${PACKAGE_SPECIFIER}" must resolve to ${CONFIG_PATH} — the file every ` +
+      "other test here asserts.",
+  );
+});
+
+test("the config declares no `extends`, which Stryker does not support", () => {
+  const config = readConfig();
+
+  // Adjudicated against @stryker-mutator/core 9.6.1 and @stryker-mutator/api
+  // 9.6.1: the config reader loads exactly one config file and deep-merges CLI
+  // arguments over it — there is no extends resolution step anywhere in it —
+  // and `extends` is absent from the 45 top-level properties in the published
+  // stryker-core.json schema. A base that advertises an `extends` recipe sends
+  // consumers down a path where the settings below arrive not at all.
+  assert.equal(
+    Object.hasOwn(config, "extends"),
+    false,
+    `${CONFIG_PATH} must not declare "extends". Stryker has no such option; ` +
+      "the supported mechanism is importing this file by its package " +
+      "specifier and spreading it (see the README).",
+  );
+});
+
+test("annotations use the `_comment` suffix Stryker's validator exempts", () => {
+  const config = readConfig();
+
+  // Stryker warns "Unknown stryker config option \"<key>\"" for any top-level
+  // key that is neither in its schema nor suffixed `_comment`. A prefix-named
+  // key like `_comment_disableBail` fails that suffix check, so documenting
+  // the base costs every consumer a warning on every run.
+  const STRYKER_OPTIONS = new Set([
+    "$schema",
+    "packageManager",
+    "reporters",
+    "coverageAnalysis",
+    "ignoreStatic",
+    "cleanTempDir",
+    "disableBail",
+    "timeoutMS",
+    "timeoutFactor",
+    "dryRunTimeoutMinutes",
+    "thresholds",
+  ]);
+
+  const wouldWarn = Object.keys(config).filter(
+    (key) => !STRYKER_OPTIONS.has(key) && !key.endsWith("_comment"),
+  );
+
+  assert.deepEqual(
+    wouldWarn,
+    [],
+    `${CONFIG_PATH} keys ${JSON.stringify(wouldWarn)} are neither pinned Stryker ` +
+      'options nor suffixed "_comment", so Stryker reports each as an unknown ' +
+      "config option in every consumer run. Rename annotations to " +
+      "`<topic>_comment`.",
+  );
+});
+
+test("every pinned Stryker option is still exported to consumers", () => {
+  const pkg = JSON.parse(readFileSync(PACKAGE_PATH, "utf8"));
+
+  // The import test above proves resolution works *here*, where Node's
+  // self-reference falls back to the local file. Publication is what carries it
+  // to a consumer, and that needs both the exports entry and the files
+  // allowlist.
+  assert.equal(
+    pkg.exports["./stryker.base.json"],
+    `./${CONFIG_PATH}`,
+    "The ./stryker.base.json export must point at the file this test asserts, " +
+      "or consumers import a config nothing guards.",
+  );
+  assert.ok(
+    pkg.files.includes("config/"),
+    'The "files" allowlist must publish config/, or the export resolves to a ' +
+      "file absent from the tarball.",
+  );
+});

package/templates/runbooks/runner-provisioning.md

@@ -142,6 +142,8 @@ Copy the kit from the platform payload into the runner root:
 ```bash
 cp node_modules/mandrel-platform/templates/runner/job-cleanup.sh <RUNNER_DIR>/job-cleanup.sh
 chmod +x <RUNNER_DIR>/job-cleanup.sh
+cp node_modules/mandrel-platform/templates/runner/check-runner-env-drift.sh <RUNNER_DIR>/check-runner-env-drift.sh
+chmod +x <RUNNER_DIR>/check-runner-env-drift.sh
 cp node_modules/mandrel-platform/templates/runner/.env.example <RUNNER_DIR>/.env
 ```
 
@@ -167,8 +169,48 @@ with the runner root's absolute path. The resulting file wires:
   (two env names, one dir; some actions read the legacy name).
 - `LANG=en_US.UTF-8`.
 
-The hook needs no per-runner editing: it derives `RUNNER_DIR` from its own
-location, so the same file works verbatim on every runner.
+Neither shipped script needs per-runner editing: each derives its paths from
+its own location, so the same files work verbatim on every runner.
+
+### Confirm the pool is uniform (`check-runner-env-drift.sh`)
+
+Run this **after provisioning each runner**, and again whenever two runners
+behave differently on the same job. It walks the pool and names the runners
+missing any of the four mandated keys:
+
+```bash
+cd <RUNNER_DIR>
+./check-runner-env-drift.sh                       # pool root = this dir's parent
+./check-runner-env-drift.sh --pool-root <POOL_ROOT>
+```
+
+The pool root is the directory holding one subdirectory per runner (§1); a
+child directory counts as a runner iff it contains `config.sh`. The checker is
+read-only — it never writes into a runner root and never touches a service.
+
+| Exit | Meaning | Operator response |
+|------|---------|-------------------|
+| `0` | No drift. Every mandated key is set on every runner, or unset on every runner. | None. |
+| `1` | **Drift** — a key is set on some runners but not all. | Copy `.env.example` onto each runner the report names, substitute its `<RUNNER_DIR>`, then `./svc.sh stop && ./svc.sh start` on those runners so they reload `.env`. |
+| `2` | Usage error — bad flag, or a pool root holding no runner directories. | Re-check `--pool-root`. |
+
+A key absent from **every** runner is reported as a uniform gap and does *not*
+exit non-zero, so a fleet that has deliberately not adopted a key is not a
+standing alarm.
+
+**Why this check exists at all:** `scripts/check-runner-health.mjs` monitors
+the fleet through the GitHub runners API, which reports a runner's name,
+labels and online status — it cannot see `<RUNNER_DIR>/.env`. So partial
+provisioning never surfaces as a configuration fault; it surfaces as an
+unattributable behavioural difference between two runs of the same job. In
+issue #343, 16 of 19 runners on one host carried the hook, and the resulting
+`Set up runner` spread (5m29s against 54s) took far longer to attribute than
+reading nineteen `.env` files would have.
+
+Do **not** wire this into `ACTIONS_RUNNER_HOOK_JOB_STARTED`. That hook runs
+inside the job's clock, where every read is billed to `Set up runner` and
+counts against the job's `timeout-minutes` — the exact cost model that made
+#343 a job-killer. This is an operator-run tool.
 
 ## 5. Install as a launchd service (`svc.sh`)
 
@@ -199,10 +241,12 @@ The runner loads `.env` at service start — after any `.env` change, restart:
   `svc.sh`.
 - **Kit updates.** The hook and `.env.example` are versioned in
   mandrel-platform. On a platform release that touches `templates/runner/`,
-  re-copy `job-cleanup.sh` (verbatim — it is parameterized) and diff
-  `.env.example` against the live `.env`, then `./svc.sh stop && ./svc.sh
-  start`. There is no `mandrel sync` equivalent for a runner host's
-  filesystem — this is an operator-applied step.
+  re-copy `job-cleanup.sh` and `check-runner-env-drift.sh` (both verbatim —
+  they are parameterized) and diff `.env.example` against the live `.env`,
+  then `./svc.sh stop && ./svc.sh start`. There is no `mandrel sync`
+  equivalent for a runner host's filesystem — this is an operator-applied
+  step. Re-run `./check-runner-env-drift.sh` afterwards: a kit update applied
+  to some runners and not others is exactly the drift it reports.
 - **Token/registration rotation.** Registration tokens are one-shot at
   config time; nothing persists to rotate. To move a runner between repos or
   rename it: `./svc.sh stop && ./svc.sh uninstall && ./config.sh remove

package/templates/runner/check-runner-env-drift.sh

@@ -0,0 +1,248 @@
+#!/usr/bin/env bash
+#
+# check-runner-env-drift.sh — report per-runner `.env` configuration drift
+# across one host's runner pool (mandrel-platform runner kit).
+#
+# ── WHY THIS EXISTS ─────────────────────────────────────────────────────────
+#
+# Nothing else observes a runner's LOCAL configuration. The fleet monitor
+# (`scripts/check-runner-health.mjs`) reaches runners through
+# `GET /repos/{owner}/{repo}/actions/runners`, which reports a runner's name,
+# labels and online status — the endpoint cannot see `<RUNNER_DIR>/.env`, so
+# hook configuration is invisible to it.
+#
+# The consequence is that a partially-provisioned pool never presents as a
+# configuration fault. It presents as an unattributable behavioural difference
+# between two runs of the SAME job on the SAME repo. That is issue #343: on one
+# host, 16 of 19 runners carried ACTIONS_RUNNER_HOOK_JOB_STARTED; two of the
+# hooked ones sat 5m29s in `Set up runner` while the same job on an unhooked
+# runner finished in 54 seconds. Attributing that took far longer than reading
+# nineteen `.env` files would have — which is precisely what this script does.
+#
+# ── WHAT IT REPORTS ─────────────────────────────────────────────────────────
+#
+# PRESENCE — never value — of the four keys `.env.example` mandates:
+# ACTIONS_RUNNER_HOOK_JOB_STARTED, RUNNER_TOOL_CACHE, AGENT_TOOLSDIRECTORY,
+# LANG. Values are deliberately not compared: every one of them embeds the
+# runner's own absolute root path, so they are SUPPOSED to differ per runner.
+#
+# The drift signal is a key set on SOME runners but not all — the 16-of-19
+# shape. A key absent from EVERY runner is a uniform gap: reported as such, and
+# not on its own a non-zero exit. A fleet that has deliberately not adopted a
+# key must not be a standing alarm, or the operator learns to ignore the exit
+# code and the signal is worth nothing when it does fire.
+#
+# ── OPERATOR CONTRACT ───────────────────────────────────────────────────────
+#
+#   check-runner-env-drift.sh [--pool-root <dir>]
+#
+#   --pool-root <dir>  Directory holding one subdirectory per runner. Defaults
+#                      to the PARENT of the directory containing this script:
+#                      the kit installs it into <RUNNER_DIR>, and the runbook
+#                      mandates one directory per runner under a common root,
+#                      so the default is correct on any kit-provisioned host.
+#
+#   A child directory counts as a runner iff it contains `config.sh`. That
+#   predicate keeps unrelated siblings (shared caches, scratch dirs) out of the
+#   report without inventing a naming convention.
+#
+#   exit 0 — no drift: every mandated key is uniform across the pool (set
+#            everywhere, or unset everywhere).
+#   exit 1 — drift: at least one key is set on some runners but not all. The
+#            non-zero exit IS the alert channel, matching the posture
+#            `scripts/check-runner-health.mjs` already uses, so this can be
+#            scheduled.
+#   exit 2 — usage error: unknown flag, or a pool root that is not a directory
+#            or holds no runners. Deliberately distinct from 0: reporting "no
+#            drift" over an empty walk would read as evidence the fleet is
+#            uniform.
+#
+# READ-ONLY, and never fails soft on a broken runner. It writes nothing into a
+# runner root and never touches a launchd service. A runner whose `.env` is
+# missing or unreadable is recorded as all four keys unset and the walk
+# continues — one broken runner must not shrink the sample the verdict is
+# computed over.
+#
+# This is an OPERATOR-run tool, not a job hook. Do not wire it into
+# ACTIONS_RUNNER_HOOK_JOB_STARTED: that hook runs inside the job's clock, where
+# every read is billed to `Set up runner` and counts against the job's
+# `timeout-minutes` (issue #343). A pool-wide walk belongs outside that clock.
+#
+# ── PORTABILITY ─────────────────────────────────────────────────────────────
+#
+# Runs on the host with no repo checkout and no Node runtime, and stays
+# compatible with macOS's system bash 3.2 — the same constraint
+# `.github/actions/gitleaks-scan/action.yml` documents for this fleet. That
+# rules out `declare -A`, `mapfile`/`readarray`, and `${var,,}`; it does NOT
+# rule out plain INDEXED arrays, which 3.2 supports and which the accumulators
+# below use. Only the per-key tally needs a second pass, because keeping a
+# key->count table is the one thing an indexed array cannot do.
+#
+# Accumulating into arrays rather than splitting a delimited string on a
+# reassigned `IFS` is deliberate and load-bearing: reassigning IFS globally is
+# flagged by the platform's own SAST ruleset (`bash.lang.security.ifs-tampering`)
+# because it silently changes the splitting behaviour of every later unquoted
+# expansion in the script. Arrays give the same grouping with no global state
+# and no quoting hazard for a runner directory whose name contains whitespace.
+
+set -u
+
+MANDATED_KEYS=(
+  ACTIONS_RUNNER_HOOK_JOB_STARTED
+  RUNNER_TOOL_CACHE
+  AGENT_TOOLSDIRECTORY
+  LANG
+)
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+POOL_ROOT=$(dirname "$SCRIPT_DIR")
+
+usage() {
+  cat <<'USAGE'
+Usage: check-runner-env-drift.sh [--pool-root <dir>]
+
+Reports which runners in a pool are missing the `.env` keys the runner kit
+mandates. Read-only.
+
+  --pool-root <dir>   Pool root (default: the parent of this script's dir).
+  -h, --help          Show this help.
+
+Exit: 0 no drift · 1 drift · 2 usage error.
+USAGE
+}
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --pool-root)
+      if [ $# -lt 2 ]; then
+        printf 'check-runner-env-drift: --pool-root requires a directory\n' >&2
+        exit 2
+      fi
+      POOL_ROOT=$2
+      shift 2
+      ;;
+    --pool-root=*)
+      POOL_ROOT=${1#--pool-root=}
+      shift
+      ;;
+    -h | --help)
+      usage
+      exit 0
+      ;;
+    *)
+      printf 'check-runner-env-drift: unknown argument: %s\n' "$1" >&2
+      usage >&2
+      exit 2
+      ;;
+  esac
+done
+
+if [ ! -d "$POOL_ROOT" ]; then
+  printf 'check-runner-env-drift: pool root is not a directory: %s\n' "$POOL_ROOT" >&2
+  exit 2
+fi
+POOL_ROOT=$(cd "$POOL_ROOT" && pwd)
+
+# Presence test for one key in one runner's `.env`.
+#
+# The assignment form is `^[[:space:]]*KEY=` on a non-comment line. A commented
+# line cannot match (the `#` is not whitespace), which is the case that matters:
+# `.env.example` ships every key inside a block of explanatory prose, so a
+# half-applied copy where the operator never uncommented a line is the most
+# likely real drift shape — and matching the key name anywhere in the file would
+# report that runner as fully provisioned. The trailing `=` is equally
+# load-bearing: without it `LANGUAGE=` would satisfy `LANG`.
+#
+# A missing, unreadable, or non-regular `.env` returns "unset" rather than
+# aborting, so the caller records all keys unset and keeps walking.
+env_has_key() {
+  env_file=$1
+  env_key=$2
+
+  [ -f "$env_file" ] || return 1
+  [ -r "$env_file" ] || return 1
+  grep -Eq "^[[:space:]]*${env_key}=" "$env_file" 2>/dev/null
+}
+
+key_total=${#MANDATED_KEYS[@]}
+
+# Enumerate runners. A glob matching nothing stays literal and fails the `-d`
+# test, so an empty pool root falls through to the exit-2 branch below.
+runner_names=()
+for candidate in "$POOL_ROOT"/*/; do
+  [ -d "$candidate" ] || continue
+  [ -f "${candidate}config.sh" ] || continue
+  runner_names+=("$(basename "$candidate")")
+done
+runner_count=${#runner_names[@]}
+
+if [ "$runner_count" -eq 0 ]; then
+  printf 'check-runner-env-drift: no runner directories under %s\n' "$POOL_ROOT" >&2
+  printf 'check-runner-env-drift: a runner is a child directory containing config.sh — is this the pool root?\n' >&2
+  exit 2
+fi
+
+printf 'runner .env configuration drift report\n'
+printf '  pool root: %s\n' "$POOL_ROOT"
+printf '  runners:   %d\n' "$runner_count"
+printf '\n'
+
+printf 'per-runner:\n'
+for name in "${runner_names[@]}"; do
+  unset_keys=()
+  for key in "${MANDATED_KEYS[@]}"; do
+    if ! env_has_key "$POOL_ROOT/$name/.env" "$key"; then
+      unset_keys+=("$key")
+    fi
+  done
+
+  if [ "${#unset_keys[@]}" -eq 0 ]; then
+    printf '  %s: all %d mandated keys set\n' "$name" "$key_total"
+  else
+    # `${arr[*]}` joins on the first character of IFS — a space, since this
+    # script never reassigns it. Safe for key names, which carry no whitespace;
+    # runner names are printed one per line below for exactly that reason.
+    printf '  %s: unset %s\n' "$name" "${unset_keys[*]}"
+  fi
+done
+printf '\n'
+
+printf 'per-key:\n'
+drift_count=0
+for key in "${MANDATED_KEYS[@]}"; do
+  set_count=0
+  unset_names=()
+  for name in "${runner_names[@]}"; do
+    if env_has_key "$POOL_ROOT/$name/.env" "$key"; then
+      set_count=$((set_count + 1))
+    else
+      unset_names+=("$name")
+    fi
+  done
+
+  if [ "$set_count" -eq "$runner_count" ]; then
+    printf '  %s: ok — set on %d of %d runners\n' "$key" "$set_count" "$runner_count"
+  elif [ "$set_count" -eq 0 ]; then
+    printf '  %s: uniformly unset — set on 0 of %d runners; a uniform gap, not drift\n' "$key" "$runner_count"
+  else
+    # Reached only when 0 < set_count < runner_count, so unset_names is
+    # non-empty here — one name per line, because a runner directory name may
+    # contain whitespace and a joined list would make it unactionable.
+    drift_count=$((drift_count + 1))
+    printf '  %s: DRIFT — set on %d of %d runners; unset on:\n' "$key" "$set_count" "$runner_count"
+    for name in "${unset_names[@]}"; do
+      printf '      %s\n' "$name"
+    done
+  fi
+done
+printf '\n'
+
+if [ "$drift_count" -eq 0 ]; then
+  printf 'no drift: every mandated key is uniform across the pool.\n'
+  exit 0
+fi
+
+printf 'DRIFT: %d of %d mandated keys are set on some runners but not all.\n' "$drift_count" "$key_total"
+printf 'Provision the runners named above from templates/runner/.env.example,\n'
+printf 'then restart each one so it reloads .env: ./svc.sh stop && ./svc.sh start\n'
+exit 1