mandrel-platform 1.13.2 → 1.14.0

package/scripts/release-asset-download-flags.test.mjs

@@ -0,0 +1,333 @@
+#!/usr/bin/env node
+/**
+ * release-asset-download-flags.test.mjs — node:test guard over the retry flags
+ * on every pinned release-asset download in a first-party composite action
+ * (Story #523).
+ *
+ * WHY A SECOND GUARD
+ * ------------------
+ * `scripts/check-action-download-retries.mjs` already lints that every asset
+ * download carries `--retry`, `--retry-connrefused` and `--max-time`. That
+ * contract turned out to be insufficient for the failure the release CDN
+ * actually produces: on 2026-09-14 a consumer's `ci / Security` job logged
+ *
+ *     Downloading pinned gitleaks: https://github.com/gitleaks/gitleaks/…
+ *     curl: (56) The requested URL returned error: 504
+ *
+ * and the step took **196 ms** — three retries with curl's default backoff
+ * take ≥ 7 s, so no retry fired. curl treats HTTP 408/429/5xx as transient
+ * only when it can read them as a response; a 504 that arrives as a broken
+ * transfer surfaces as `CURLE_RECV_ERROR` (exit 56), outside that set.
+ * `--retry-all-errors` (curl ≥ 7.71) is what makes it retryable.
+ *
+ * So the flag this file pins is deliberately NOT folded into the existing
+ * lint's `REQUIRED_FLAGS`: that lint governs *every* asset download in the
+ * action surface, whereas the exit-56 case is specific to the GitHub release
+ * CDN. This guard scopes itself to `releases/download` URLs and complements
+ * the lint rather than restating it — it reuses the lint's exported shell
+ * parsing (`collapseContinuations`, `shellWords`) so the two cannot disagree
+ * about what a `curl` line even is.
+ *
+ * WHAT IS ASSERTED
+ * ----------------
+ *   1. Each of the three action manifests that fetch a pinned release binary
+ *      still fetches one, and every such line carries `--retry-all-errors`
+ *      alongside `--retry` (a flag set that retries nothing is worse than no
+ *      flags, because it reads as covered).
+ *   2. No OTHER action manifest has quietly grown a release-asset download
+ *      that this file's list would miss — a fourth site must join the
+ *      contract, not escape it.
+ *   3. Checksum verification is still fail-closed in each manifest: the retry
+ *      widens what is attempted, never what is trusted (AC-3).
+ *   4. The detector itself is exercised on synthetic fixtures, so a change
+ *      that makes it match nothing fails here rather than passing vacuously.
+ *
+ * Run: node --test scripts/release-asset-download-flags.test.mjs
+ */
+
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { test } from "node:test";
+import { fileURLToPath } from "node:url";
+
+import {
+  collapseContinuations,
+  findActionManifests,
+  shellWords,
+  stripShellComment,
+} from "./check-action-download-retries.mjs";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = join(HERE, "..");
+
+/**
+ * The action manifests that fetch a pinned release binary, with how many such
+ * fetches each one owns. The counts are part of the contract: a site that
+ * disappears is as much a regression signal as one that ships uncovered.
+ */
+const RELEASE_ASSET_ACTIONS = Object.freeze([
+  Object.freeze({ path: ".github/actions/gitleaks-scan/action.yml", downloads: 1 }),
+  Object.freeze({ path: ".github/actions/workflow-lint/action.yml", downloads: 2 }),
+  Object.freeze({ path: ".github/actions/osv-scan/action.yml", downloads: 1 }),
+]);
+
+/**
+ * Flags every release-asset download must carry, with why each one matters.
+ * `--retry` without `--retry-all-errors` is the state Story #523 found: the
+ * flags read as resilient and covered nothing the CDN produces.
+ */
+const REQUIRED_RETRY_FLAGS = Object.freeze(["--retry", "--retry-all-errors"]);
+
+/** A GitHub release-asset URL, in any of the spellings the actions use. */
+const RELEASE_ASSET_URL = /releases\/download\//;
+/** `curl` invoked as a command — after start-of-line, whitespace or an operator. */
+const CURL_COMMAND = /(^|[\s;&|(])curl(\s|$)/;
+/** A shell assignment: `name=value`, optionally exported. */
+const ASSIGNMENT = /^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.+)$/;
+/** Characters that continue a bare `$name` reference. */
+const NAME_CHAR = /[A-Za-z0-9_]/;
+
+/**
+ * Pure: does `word` reference the shell variable `name`, as `${name}` or as a
+ * bare `$name` that ends there? The bare form needs the boundary check or
+ * `$url` would match `$urls`.
+ *
+ * @param {string} word
+ * @param {string} name
+ * @returns {boolean}
+ */
+function referencesVariable(word, name) {
+  if (word.includes(`\${${name}}`)) return true;
+  const bare = `$${name}`;
+  let from = 0;
+  for (;;) {
+    const at = word.indexOf(bare, from);
+    if (at === -1) return false;
+    const next = word[at + bare.length];
+    if (next === undefined || !NAME_CHAR.test(next)) return true;
+    from = at + 1;
+  }
+}
+
+/**
+ * Pure: the names of shell variables assigned a GitHub release-asset URL.
+ *
+ * Every real download in these actions builds the URL one line above the
+ * fetch (`url="https://github.com/…/releases/download/v${V}/${asset}"`) and
+ * then curls `"$url"`, so a detector that only looks for the literal in the
+ * curl line finds nothing at all — which is a guard that passes vacuously.
+ *
+ * @param {string} source
+ * @returns {Set<string>}
+ */
+export function releaseAssetUrlVariables(source) {
+  const names = new Set();
+  for (const { text } of collapseContinuations(source)) {
+    const match = ASSIGNMENT.exec(stripShellComment(text).trim());
+    if (match === null) continue;
+    if (!RELEASE_ASSET_URL.test(match[2])) continue;
+    names.add(match[1]);
+  }
+  return names;
+}
+
+/**
+ * Pure: every logical line in `source` that fetches a GitHub release asset
+ * with curl — whether the URL is written inline or carried in a variable
+ * assigned one earlier in the same script. Line numbers are 1-based and name
+ * the line the command started on, so a failure points at something a reader
+ * can open.
+ *
+ * @param {string} source
+ * @returns {Array<{ line: number, text: string, words: string[] }>}
+ */
+export function releaseAssetDownloads(source) {
+  const urlVariables = releaseAssetUrlVariables(source);
+  const found = [];
+  for (const { line, text } of collapseContinuations(source)) {
+    const command = stripShellComment(text);
+    if (!CURL_COMMAND.test(command)) continue;
+    const words = shellWords(text);
+    const fetchesRelease = words.some(
+      (word) =>
+        RELEASE_ASSET_URL.test(word) ||
+        [...urlVariables].some((name) => referencesVariable(word, name)),
+    );
+    if (!fetchesRelease) continue;
+    found.push({ line, text, words });
+  }
+  return found;
+}
+
+/**
+ * Pure: which of the required retry flags this download is missing, in
+ * contract order. Matched as whole shell words (or `--flag=value`), never as
+ * substrings — `--retry-all-errors` contains `--retry`, and the reverse
+ * containment is exactly the bug this guard exists for.
+ *
+ * @param {string[]} words
+ * @returns {string[]}
+ */
+export function missingRetryFlags(words) {
+  return REQUIRED_RETRY_FLAGS.filter(
+    (flag) => !words.some((word) => word === flag || word.startsWith(`${flag}=`)),
+  );
+}
+
+/**
+ * Read one repo-relative file.
+ *
+ * @param {string} relative
+ * @returns {string}
+ */
+function readRepoFile(relative) {
+  return readFileSync(join(REPO_ROOT, relative), "utf8");
+}
+
+for (const { path, downloads } of RELEASE_ASSET_ACTIONS) {
+  test(`${path} — every release-asset download retries an exit-56 failure`, () => {
+    const found = releaseAssetDownloads(readRepoFile(path));
+
+    assert.equal(
+      found.length,
+      downloads,
+      `expected ${downloads} release-asset download(s) in ${path}, found ${found.length} — update RELEASE_ASSET_ACTIONS if a site was deliberately added or removed`,
+    );
+
+    for (const { line, words } of found) {
+      const missing = missingRetryFlags(words);
+      assert.deepEqual(
+        missing,
+        [],
+        `${path}:${line} is missing ${missing.join(", ")}. A release-CDN 504 reaches curl as exit 56 (CURLE_RECV_ERROR), which --retry alone does not treat as transient; --retry-all-errors is what covers it.`,
+      );
+    }
+  });
+
+  test(`${path} — a checksum mismatch still fails the download closed`, () => {
+    const source = readRepoFile(path);
+    assert.match(
+      source,
+      /checksum mismatch/,
+      `${path} no longer fails on a checksum mismatch — the retry may widen what is attempted, never what is trusted`,
+    );
+  });
+}
+
+test("no other action manifest fetches a release asset uncovered", () => {
+  const known = new Set(RELEASE_ASSET_ACTIONS.map(({ path }) => path));
+  const uncovered = [];
+
+  for (const manifest of findActionManifests(join(REPO_ROOT, ".github", "actions"))) {
+    const relative = manifest.slice(REPO_ROOT.length + 1);
+    if (known.has(relative)) continue;
+    if (releaseAssetDownloads(readFileSync(manifest, "utf8")).length > 0) {
+      uncovered.push(relative);
+    }
+  }
+
+  assert.deepEqual(
+    uncovered,
+    [],
+    `these action manifests grew a release-asset download outside the retry contract: ${uncovered.join(", ")} — add them to RELEASE_ASSET_ACTIONS`,
+  );
+});
+
+test("detects a release-asset download and reports its missing flags", () => {
+  const source = [
+    "        echo 'Downloading pinned thing'",
+    '        curl -fsSL --retry 3 --retry-connrefused --max-time 300 "https://github.com/o/r/releases/download/v1/asset.tar.gz" -o "${tmp}/asset.tar.gz"',
+  ].join("\n");
+
+  const found = releaseAssetDownloads(source);
+  assert.equal(found.length, 1);
+  assert.equal(found[0].line, 2);
+  assert.deepEqual(missingRetryFlags(found[0].words), ["--retry-all-errors"]);
+});
+
+test("a shell-variable URL split across continuations is still one download", () => {
+  const source = [
+    '        curl -fsSL --retry 3 --retry-all-errors \\',
+    '          --retry-connrefused --retry-delay 2 --max-time 300 \\',
+    '          "https://github.com/o/r/releases/download/v1/a" -o "$out"',
+  ].join("\n");
+
+  const found = releaseAssetDownloads(source);
+  assert.equal(found.length, 1, "line continuations must fold into one command");
+  assert.equal(found[0].line, 1, "the reported line is where the command starts");
+  assert.deepEqual(missingRetryFlags(found[0].words), []);
+});
+
+test("a URL carried in a shell variable is still a release-asset download", () => {
+  const source = [
+    '        asset="tool_${V}_linux_amd64.tar.gz"',
+    '        url="https://github.com/o/r/releases/download/v${V}/${asset}"',
+    '        echo "Downloading pinned tool: ${url}"',
+    '        curl -fsSL --retry 3 --retry-connrefused --max-time 300 "$url" -o "${tmp}/${asset}"',
+  ].join("\n");
+
+  const found = releaseAssetDownloads(source);
+  assert.equal(found.length, 1, "the fetch resolves through the url variable");
+  assert.equal(found[0].line, 4);
+  assert.deepEqual(missingRetryFlags(found[0].words), ["--retry-all-errors"]);
+});
+
+test("a variable holding a non-release URL does not pull its curl into scope", () => {
+  const source = [
+    '        api="https://api.github.com/repos/o/r/releases/latest"',
+    '        curl -fsSL --max-time 30 "$api" -o meta.json',
+  ].join("\n");
+
+  assert.deepEqual(
+    releaseAssetDownloads(source),
+    [],
+    "an API probe is not a pinned asset fetch and carries no retry contract here",
+  );
+});
+
+test("a bare $name reference does not match a longer variable name", () => {
+  const source = [
+    '        url="https://github.com/o/r/releases/download/v1/a"',
+    "        curl -fsSL $urls -o out",
+  ].join("\n");
+
+  assert.deepEqual(
+    releaseAssetDownloads(source),
+    [],
+    "$urls is a different variable from $url",
+  );
+});
+
+test("non-release fetches and commented-out lines are out of scope", () => {
+  const notRelease = 'curl -fsSL --max-time 5 "https://api.github.com/repos/o/r" -o out.json';
+  assert.deepEqual(releaseAssetDownloads(notRelease), []);
+
+  const commented =
+    '        # curl -fsSL "https://github.com/o/r/releases/download/v1/a" -o "$out"';
+  assert.deepEqual(
+    releaseAssetDownloads(commented),
+    [],
+    "a curl inside a shell comment is prose, not a download",
+  );
+
+  const trailingComment =
+    '        echo hi  # see https://github.com/o/r/releases/download/v1/a for the asset';
+  assert.deepEqual(releaseAssetDownloads(trailingComment), []);
+});
+
+test("--retry-all-errors alone does not satisfy --retry", () => {
+  const words = shellWords('curl -fsSL --retry-all-errors "$url" -o out');
+  assert.deepEqual(
+    missingRetryFlags(words),
+    ["--retry"],
+    "substring matching would report this as covered; it retries zero times",
+  );
+});
+
+test("the --flag=value spelling counts", () => {
+  const words = shellWords(
+    'curl --retry=3 --retry-all-errors "https://github.com/o/r/releases/download/v1/a" -o out',
+  );
+  assert.deepEqual(missingRetryFlags(words), []);
+});

package/scripts/runner-env-drift.test.mjs

@@ -87,13 +87,16 @@ const HERE = dirname(fileURLToPath(import.meta.url));
 const SCRIPT = join(HERE, "..", "templates", "runner", "check-runner-env-drift.sh");
 const RUNBOOK = join(HERE, "..", "templates", "runbooks", "runner-provisioning.md");
 
-/** The four keys `templates/runner/.env.example` mandates. */
+/** The five keys `templates/runner/.env.example` mandates. */
 const HOOK = "ACTIONS_RUNNER_HOOK_JOB_STARTED";
-const MANDATED = [HOOK, "RUNNER_TOOL_CACHE", "AGENT_TOOLSDIRECTORY", "LANG"];
+/** The job-END hook (Story #524) — mandated on the same footing as the start one. */
+const HOOK_COMPLETED = "ACTIONS_RUNNER_HOOK_JOB_COMPLETED";
+const MANDATED = [HOOK, HOOK_COMPLETED, "RUNNER_TOOL_CACHE", "AGENT_TOOLSDIRECTORY", "LANG"];
 
 /** Representative values — the checker reports PRESENCE, never value. */
 const VALUES = {
   ACTIONS_RUNNER_HOOK_JOB_STARTED: "/Users/ci/runners/a/job-cleanup.sh",
+  ACTIONS_RUNNER_HOOK_JOB_COMPLETED: "/Users/ci/runners/a/job-completed.sh",
   RUNNER_TOOL_CACHE: "/Users/ci/runners/a/_work/_tool",
   AGENT_TOOLSDIRECTORY: "/Users/ci/runners/a/_work/_tool",
   LANG: "en_US.UTF-8",
@@ -128,7 +131,7 @@ function envWith(keys) {
  * Build a synthetic pool root.
  *
  * Each entry maps a child directory name to its spec:
- *   `keys`     — mandated keys to set in that runner's `.env` (default: all four)
+ *   `keys`     — mandated keys to set in that runner's `.env` (default: all five)
  *   `env`      — raw `.env` body, overriding `keys`
  *   `noEnv`    — create no `.env` at all
  *   `isRunner` — false to omit `config.sh`, i.e. not a runner directory
@@ -242,6 +245,52 @@ test("AC-2: names every runner missing a key that others have, and exits non-zer
   assert.match(stdout, new RegExp(HOOK), "the drifting key must be named");
 });
 
+test("Story #524: a runner missing only the job-COMPLETED hook is named as drifted", () => {
+  // The two hooks close different halves of one gap — started defends the next
+  // job against the previous one's orphans, completed makes each job reap its
+  // own tree — so a runner carrying only the started hook is half-provisioned,
+  // and this checker is the only observer that can see it. Provisioning the
+  // completed hook across a fleet is an operator-applied, per-host step, which
+  // is exactly the shape that lands on some runners and not others.
+  const root = makePool({
+    "runner-a": { keys: MANDATED },
+    "runner-b": { keys: MANDATED.filter((key) => key !== HOOK_COMPLETED) },
+  });
+
+  const { status, stdout } = runChecker(["--pool-root", root]);
+
+  assert.notEqual(status, 0, "a missing completed hook must drift like a missing started hook");
+  assert.match(stdout, new RegExp(`${HOOK_COMPLETED}: DRIFT`), "the drifting key must be named");
+  assert.match(stdout, /runner-b/, "the runner missing it must be named, not just counted");
+});
+
+test("Story #524: `.env.example` sets every key the checker mandates", () => {
+  // The runbook's provisioning step is `cp .env.example <RUNNER_DIR>/.env`. If
+  // the checker mandated a key the example never sets, an operator who
+  // followed the runbook exactly could not satisfy it — the alert would fire
+  // on a correctly-provisioned fleet, and the operator would learn to ignore
+  // the exit code.
+  const example = readFileSync(join(HERE, "..", "templates", "runner", ".env.example"), "utf8");
+  const mandatedInScript = readFileSync(SCRIPT, "utf8")
+    .split("MANDATED_KEYS=(")[1]
+    .split(")")[0]
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(Boolean);
+
+  assert.deepEqual(
+    mandatedInScript,
+    MANDATED,
+    "the checker's mandated keys and this suite's list have diverged",
+  );
+  for (const key of mandatedInScript) {
+    assert.ok(
+      new RegExp(`^\\s*${key}=`, "m").test(example),
+      `.env.example does not set ${key}, but the checker reports a runner without it`,
+    );
+  }
+});
+
 test("AC-3: a fully provisioned pool exits 0", () => {
   const root = makePool({
     "runner-a": { keys: MANDATED },
@@ -480,7 +529,7 @@ test("a commented-out assignment does not count as set", () => {
   const root = makePool({
     "runner-a": { keys: MANDATED },
     "runner-b": {
-      env: `# ${HOOK}=/Users/ci/runners/b/job-cleanup.sh\nRUNNER_TOOL_CACHE=${VALUES.RUNNER_TOOL_CACHE}\nAGENT_TOOLSDIRECTORY=${VALUES.AGENT_TOOLSDIRECTORY}\nLANG=${VALUES.LANG}\n`,
+      env: `# ${HOOK}=/Users/ci/runners/b/job-cleanup.sh\n${HOOK_COMPLETED}=${VALUES[HOOK_COMPLETED]}\nRUNNER_TOOL_CACHE=${VALUES.RUNNER_TOOL_CACHE}\nAGENT_TOOLSDIRECTORY=${VALUES.AGENT_TOOLSDIRECTORY}\nLANG=${VALUES.LANG}\n`,
     },
   });
 
@@ -494,7 +543,7 @@ test("a leading-whitespace assignment counts as set", () => {
   const root = makePool({
     "runner-a": { keys: MANDATED },
     "runner-b": {
-      env: `  ${HOOK}=/Users/ci/runners/b/job-cleanup.sh\n\tRUNNER_TOOL_CACHE=${VALUES.RUNNER_TOOL_CACHE}\nAGENT_TOOLSDIRECTORY=${VALUES.AGENT_TOOLSDIRECTORY}\nLANG=${VALUES.LANG}\n`,
+      env: `  ${HOOK}=/Users/ci/runners/b/job-cleanup.sh\n\t${HOOK_COMPLETED}=${VALUES[HOOK_COMPLETED]}\n\tRUNNER_TOOL_CACHE=${VALUES.RUNNER_TOOL_CACHE}\nAGENT_TOOLSDIRECTORY=${VALUES.AGENT_TOOLSDIRECTORY}\nLANG=${VALUES.LANG}\n`,
     },
   });
 
@@ -509,7 +558,7 @@ test("a longer key that merely starts with a mandated key does not count as set"
   const root = makePool({
     "runner-a": { keys: MANDATED },
     "runner-b": {
-      env: `${HOOK}=/Users/ci/runners/b/job-cleanup.sh\nRUNNER_TOOL_CACHE=${VALUES.RUNNER_TOOL_CACHE}\nAGENT_TOOLSDIRECTORY=${VALUES.AGENT_TOOLSDIRECTORY}\nLANGUAGE=en_US\n`,
+      env: `${HOOK}=/Users/ci/runners/b/job-cleanup.sh\n${HOOK_COMPLETED}=${VALUES[HOOK_COMPLETED]}\nRUNNER_TOOL_CACHE=${VALUES.RUNNER_TOOL_CACHE}\nAGENT_TOOLSDIRECTORY=${VALUES.AGENT_TOOLSDIRECTORY}\nLANGUAGE=en_US\n`,
     },
   });
 

package/templates/runbooks/runner-provisioning.md

@@ -4,8 +4,9 @@
 > this directory, there is no canonical `docs/runbooks/` counterpart to link —
 > this file IS the process. It provisions a **persistent** (non-ephemeral)
 > GitHub Actions runner on a macOS host using the mandrel-platform runner kit
-> (`templates/runner/`), which ships the job-start hygiene hook
-> (`job-cleanup.sh`) and the per-runner `.env` (`.env.example`).
+> (`templates/runner/`), which ships the two job hooks — job-start hygiene
+> (`job-cleanup.sh`) and job-end reaping (`job-completed.sh`) — and the
+> per-runner `.env` (`.env.example`).
 >
 > Placeholder convention: `<UPPER_SNAKE>` between angle brackets — search for
 > `<` after copying to find everything that still needs a value.
@@ -113,13 +114,15 @@ installs the pnpm shim to a **runner-scoped** destination:
 There is **no runner-side override** for the pnpm `dest` — it is a workflow
 input — which is why this step is a rollout gate, not an `.env` line.
 
-## 4. Install the hygiene kit (hook + `.env`)
+## 4. Install the hygiene kit (hooks + `.env`)
 
 Copy the kit from the platform payload into the runner root:
 
 ```bash
 cp node_modules/mandrel-platform/templates/runner/job-cleanup.sh <RUNNER_DIR>/job-cleanup.sh
 chmod +x <RUNNER_DIR>/job-cleanup.sh
+cp node_modules/mandrel-platform/templates/runner/job-completed.sh <RUNNER_DIR>/job-completed.sh
+chmod +x <RUNNER_DIR>/job-completed.sh
 cp node_modules/mandrel-platform/templates/runner/check-runner-env-drift.sh <RUNNER_DIR>/check-runner-env-drift.sh
 chmod +x <RUNNER_DIR>/check-runner-env-drift.sh
 cp node_modules/mandrel-platform/templates/runner/.env.example <RUNNER_DIR>/.env
@@ -142,19 +145,52 @@ with the runner root's absolute path. The resulting file wires:
   jobs were killed before their first real step — surfacing as `cancelled`
   on unrelated diffs. If you add a sweep to this hook, root it at
   `_work/_temp`, never at `$TMPDIR`.
+- `ACTIONS_RUNNER_HOOK_JOB_COMPLETED=<RUNNER_DIR>/job-completed.sh` — the
+  job-end hook. After the last step of every job it terminates whatever of
+  **this** job's process tree is still alive — SIGTERM, a bounded grace, then
+  SIGKILL — matching processes whose command line resolves inside
+  `<RUNNER_DIR>/_work/` plus their descendants. It never signals itself, its
+  own ancestors, or the runner's `Runner.Worker`/`Runner.Listener`, never
+  fails a job (always exits 0), and does nothing at all when the job left
+  nothing behind.
 - `RUNNER_TOOL_CACHE=<RUNNER_DIR>/_work/_tool` and
   `AGENT_TOOLSDIRECTORY=<RUNNER_DIR>/_work/_tool` — runner-scoped tool cache
   (two env names, one dir; some actions read the legacy name).
 - `LANG=en_US.UTF-8`.
 
-Neither shipped script needs per-runner editing: each derives its paths from
-its own location, so the same files work verbatim on every runner.
+None of the shipped scripts needs per-runner editing: each derives its paths
+from its own location, so the same files work verbatim on every runner.
+
+### Why both hooks
+
+They cover opposite ends of the same job and neither substitutes for the
+other:
+
+| Hook | Runs | Defends against |
+|------|------|-----------------|
+| `job-cleanup.sh` (`..._JOB_STARTED`) | before the first step | the **previous** job's leftovers — orphaned pnpm/node processes and stale `_work/_temp` artifacts already on the runner |
+| `job-completed.sh` (`..._JOB_COMPLETED`) | after the last step | **this** job's own survivors reaching the **next** job |
+
+The started hook cannot close the second case: it runs before the new job's
+processes exist, so once a job is minutes in, nothing it did can help. A
+**cancelled** job is where survivors are most likely — the runner terminates
+the step it is executing, not everything that step forked — so a coalesced
+push (`concurrency: cancel-in-progress`) is the routine way a runner ends up
+hosting a previous job's vitest forks or dev server. The observed symptom is
+the next job on that runner exiting 143 (SIGTERM) mid-run, with no
+cancellation request in the runner's `Worker_*.log` and every concurrent job
+on the pool's other runners passing.
+
+The completed hook runs on the **job's** clock, like the started one, so it is
+held to the same cost rule: its whole input is one `ps` snapshot, and it
+sleeps only while waiting out the grace period of a tree it actually
+signalled. A job with nothing to reap pays a few milliseconds.
 
 ### Confirm the pool is uniform (`check-runner-env-drift.sh`)
 
 Run this **after provisioning each runner**, and again whenever two runners
 behave differently on the same job. It walks the pool and names the runners
-missing any of the four mandated keys:
+missing any of the five mandated keys:
 
 ```bash
 cd <RUNNER_DIR>
@@ -200,8 +236,10 @@ cd <RUNNER_DIR>
 
 Verify end-to-end: push a trivial workflow run targeting
 `runs-on: [self-hosted, macOS, ARM64, <REPO>-runner]` and confirm (a) the job
-is picked up and (b) the job log shows the `Set up runner` hook phase running
-`job-cleanup.sh` before the first step.
+is picked up, (b) the job log shows the `Set up runner` hook phase running
+`job-cleanup.sh` before the first step, and (c) the job log shows
+`job-completed.sh` running after the last step (it prints either what it
+reaped or `nothing to reap`).
 
 The runner loads `.env` at service start — after any `.env` change, restart:
 
@@ -216,10 +254,11 @@ The runner loads `.env` at service start — after any `.env` change, restart:
   pinned or the self-update wedges, stop the service, download/unpack the new
   tarball over `<RUNNER_DIR>` (config and `.env` survive), and restart via
   `svc.sh`.
-- **Kit updates.** The hook and `.env.example` are versioned in
+- **Kit updates.** The hooks and `.env.example` are versioned in
   mandrel-platform. On a platform release that touches `templates/runner/`,
-  re-copy `job-cleanup.sh` and `check-runner-env-drift.sh` (both verbatim —
-  they are parameterized) and diff `.env.example` against the live `.env`,
+  re-copy `job-cleanup.sh`, `job-completed.sh` and
+  `check-runner-env-drift.sh` (all verbatim — they are parameterized) and
+  diff `.env.example` against the live `.env`,
   then `./svc.sh stop && ./svc.sh start`. There is no `mandrel sync`
   equivalent for a runner host's filesystem — this is an operator-applied
   step. Re-run `./check-runner-env-drift.sh` afterwards: a kit update applied

package/templates/runner/.env.example

@@ -29,6 +29,21 @@ LANG=en_US.UTF-8
 # cost never becomes a function of host-wide temp churn (issue #343).
 ACTIONS_RUNNER_HOOK_JOB_STARTED=<RUNNER_DIR>/job-cleanup.sh
 
+# Job-end reap hook. Runs templates/runner/job-completed.sh (installed into
+# the runner root) after the last step of every job: terminates whatever of
+# THIS job's process tree is still alive — SIGTERM, a bounded grace, then
+# SIGKILL — scoped to processes under this runner's `_work/`.
+#
+# Both hooks are needed, and neither substitutes for the other. The started
+# hook is defence against the PREVIOUS job; it runs before the new job's own
+# processes exist, so it cannot help a job that is already minutes in. The
+# completed hook makes each job clean up after itself while the runner still
+# knows whose processes these are — which is what a CANCELLED job never does
+# on its own (the runner terminates the step it is executing, not everything
+# that step forked). Like the started hook, it is runner-scoped and always
+# exits 0.
+ACTIONS_RUNNER_HOOK_JOB_COMPLETED=<RUNNER_DIR>/job-completed.sh
+
 # Runner-scoped tool cache. Without this, actions/setup-node & friends
 # default the tool cache to a host-shared location and co-resident runners
 # race on extraction. `_work/_tool` is inside this runner's own work tree,

package/templates/runner/check-runner-env-drift.sh

@@ -20,10 +20,17 @@
 #
 # ── WHAT IT REPORTS ─────────────────────────────────────────────────────────
 #
-# PRESENCE — never value — of the four keys `.env.example` mandates:
-# ACTIONS_RUNNER_HOOK_JOB_STARTED, RUNNER_TOOL_CACHE, AGENT_TOOLSDIRECTORY,
-# LANG. Values are deliberately not compared: every one of them embeds the
-# runner's own absolute root path, so they are SUPPOSED to differ per runner.
+# PRESENCE — never value — of the five keys `.env.example` mandates:
+# ACTIONS_RUNNER_HOOK_JOB_STARTED, ACTIONS_RUNNER_HOOK_JOB_COMPLETED,
+# RUNNER_TOOL_CACHE, AGENT_TOOLSDIRECTORY, LANG. Values are deliberately not
+# compared: every one of them embeds the runner's own absolute root path, so
+# they are SUPPOSED to differ per runner.
+#
+# Both hook keys are mandated, because the two hooks close different halves of
+# the same gap: the started hook defends the NEXT job against the previous
+# one's orphans, the completed hook makes each job reap its own tree before it
+# releases the runner. A runner carrying only one of them is exactly the
+# half-provisioned shape this tool exists to name.
 #
 # The drift signal is a key set on SOME runners but not all — the 16-of-19
 # shape. A key absent from EVERY runner is a uniform gap: reported as such, and
@@ -56,12 +63,12 @@
 #
 # READ-ONLY, and never fails soft on a broken runner. It writes nothing into a
 # runner root and never touches a launchd service. A runner whose `.env` is
-# missing or unreadable is recorded as all four keys unset and the walk
+# missing or unreadable is recorded as all five keys unset and the walk
 # continues — one broken runner must not shrink the sample the verdict is
 # computed over.
 #
-# This is an OPERATOR-run tool, not a job hook. Do not wire it into
-# ACTIONS_RUNNER_HOOK_JOB_STARTED: that hook runs inside the job's clock, where
+# This is an OPERATOR-run tool, not a job hook. Do not wire it into either job
+# hook: both run inside the job's clock, where
 # every read is billed to `Set up runner` and counts against the job's
 # `timeout-minutes` (issue #343). A pool-wide walk belongs outside that clock.
 #
@@ -86,6 +93,7 @@ set -u
 
 MANDATED_KEYS=(
   ACTIONS_RUNNER_HOOK_JOB_STARTED
+  ACTIONS_RUNNER_HOOK_JOB_COMPLETED
   RUNNER_TOOL_CACHE
   AGENT_TOOLSDIRECTORY
   LANG