mandrel-platform 0.21.0 → 0.25.0

package/scripts/check-runner-health.test.mjs

@@ -0,0 +1,389 @@
+#!/usr/bin/env node
+/**
+ * check-runner-health.test.mjs — node:test suite for the scheduled
+ * runner-fleet health monitor (Story #258).
+ *
+ * The checker exposes pure helpers plus an injectable `runGh` seam, so the
+ * whole pipeline is exercised offline with canned GitHub responses — no
+ * network, no `gh` auth.
+ *
+ * Run: node scripts/check-runner-health.test.mjs   (or `node --test scripts/`)
+ */
+
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+
+import {
+  buildReport,
+  classifyRunners,
+  fetchQueuedRuns,
+  fetchRunners,
+  hasUnhealthy,
+  isRepoHealthy,
+  isStaleQueuedRun,
+  parseArgv,
+  renderReport,
+  runCli,
+  runnerMatchesLabels,
+} from "./check-runner-health.mjs";
+
+// ---------------------------------------------------------------------------
+// parseArgv
+// ---------------------------------------------------------------------------
+
+test("parseArgv defaults to the fleet consumer config", () => {
+  const opts = parseArgv([]);
+  assert.equal(opts.config, "scripts/runner-fleet-consumers.json");
+  assert.equal(opts.json, false);
+});
+
+test("parseArgv parses --config and --json", () => {
+  const opts = parseArgv(["--config", "custom.json", "--json"]);
+  assert.equal(opts.config, "custom.json");
+  assert.equal(opts.json, true);
+});
+
+// ---------------------------------------------------------------------------
+// runnerMatchesLabels
+// ---------------------------------------------------------------------------
+
+test("runnerMatchesLabels is case-insensitive and order-independent", () => {
+  assert.equal(
+    runnerMatchesLabels(["Self-Hosted", "macOS", "ARM64", "domio-runner"], [
+      "self-hosted",
+      "arm64",
+    ]),
+    true,
+  );
+});
+
+test("runnerMatchesLabels fails when an expected label is missing", () => {
+  assert.equal(
+    runnerMatchesLabels(["self-hosted", "macOS"], ["self-hosted", "ARM64"]),
+    false,
+  );
+});
+
+// ---------------------------------------------------------------------------
+// classifyRunners
+// ---------------------------------------------------------------------------
+
+const EXPECTED = { expectedCount: 3, labels: ["self-hosted", "macOS", "ARM64", "domio-runner"] };
+
+test("classifyRunners reports a fully healthy fleet", () => {
+  const runners = [1, 2, 3].map((n) => ({
+    id: n,
+    name: `domio-runner-${n}`,
+    status: "online",
+    labels: [{ name: "self-hosted" }, { name: "macOS" }, { name: "ARM64" }, { name: "domio-runner" }],
+  }));
+  const v = classifyRunners(runners, EXPECTED);
+  assert.equal(v.total, 3);
+  assert.equal(v.online, 3);
+  assert.equal(v.matchingOnline, 3);
+  assert.equal(v.shortfall, 0);
+  assert.equal(v.hasShortfall, false);
+  assert.equal(v.hasOffline, false);
+});
+
+test("classifyRunners flags an offline runner", () => {
+  const runners = [
+    { id: 1, name: "domio-runner-1", status: "online", labels: [{ name: "self-hosted" }, { name: "macOS" }, { name: "ARM64" }, { name: "domio-runner" }] },
+    { id: 2, name: "domio-runner-2", status: "offline", labels: [{ name: "self-hosted" }, { name: "macOS" }, { name: "ARM64" }, { name: "domio-runner" }] },
+  ];
+  const v = classifyRunners(runners, EXPECTED);
+  assert.equal(v.hasOffline, true);
+  assert.equal(v.offline.length, 1);
+  assert.equal(v.offline[0].name, "domio-runner-2");
+  assert.equal(v.matchingOnline, 1);
+  assert.equal(v.hasShortfall, true);
+  assert.equal(v.shortfall, 2);
+});
+
+test("classifyRunners flags a count shortfall even with all-online but wrong labels", () => {
+  const runners = [
+    { id: 1, name: "other-runner", status: "online", labels: [{ name: "self-hosted" }, { name: "linux" }] },
+  ];
+  const v = classifyRunners(runners, EXPECTED);
+  assert.equal(v.hasOffline, false);
+  assert.equal(v.matchingOnline, 0);
+  assert.equal(v.hasShortfall, true);
+  assert.equal(v.shortfall, 3);
+});
+
+test("classifyRunners handles an empty runner list", () => {
+  const v = classifyRunners([], EXPECTED);
+  assert.equal(v.total, 0);
+  assert.equal(v.hasShortfall, true);
+  assert.equal(v.shortfall, 3);
+});
+
+// ---------------------------------------------------------------------------
+// isStaleQueuedRun
+// ---------------------------------------------------------------------------
+
+const NOW = Date.parse("2026-07-03T12:00:00Z");
+
+test("isStaleQueuedRun flags an old queued run with no matching online runner", () => {
+  const run = {
+    status: "queued",
+    created_at: "2026-07-03T11:00:00Z", // 60 min ago
+    labels: ["self-hosted", "macOS", "ARM64", "domio-runner"],
+  };
+  assert.equal(isStaleQueuedRun(run, [], 20, NOW), true);
+});
+
+test("isStaleQueuedRun does not flag a recently queued run", () => {
+  const run = {
+    status: "queued",
+    created_at: "2026-07-03T11:55:00Z", // 5 min ago
+    labels: ["self-hosted", "macOS", "ARM64", "domio-runner"],
+  };
+  assert.equal(isStaleQueuedRun(run, [], 20, NOW), false);
+});
+
+test("isStaleQueuedRun does not flag when a matching online runner exists", () => {
+  const run = {
+    status: "queued",
+    created_at: "2026-07-03T11:00:00Z",
+    labels: ["self-hosted", "macOS", "ARM64", "domio-runner"],
+  };
+  const onlineLabelSets = [["self-hosted", "macOS", "ARM64", "domio-runner"]];
+  assert.equal(isStaleQueuedRun(run, onlineLabelSets, 20, NOW), false);
+});
+
+test("isStaleQueuedRun ignores non-queued/waiting runs", () => {
+  const run = { status: "completed", created_at: "2026-07-03T09:00:00Z" };
+  assert.equal(isStaleQueuedRun(run, [], 20, NOW), false);
+});
+
+test("isStaleQueuedRun does not flag when no label info is available", () => {
+  const run = { status: "queued", created_at: "2026-07-03T09:00:00Z", labels: [] };
+  assert.equal(isStaleQueuedRun(run, [], 20, NOW), false);
+});
+
+// ---------------------------------------------------------------------------
+// isRepoHealthy
+// ---------------------------------------------------------------------------
+
+test("isRepoHealthy is true only with no offline/shortfall/stale", () => {
+  const healthyVerdict = { hasOffline: false, hasShortfall: false };
+  assert.equal(isRepoHealthy(healthyVerdict, []), true);
+  assert.equal(isRepoHealthy({ ...healthyVerdict, hasOffline: true }, []), false);
+  assert.equal(isRepoHealthy({ ...healthyVerdict, hasShortfall: true }, []), false);
+  assert.equal(isRepoHealthy(healthyVerdict, [{ id: 1 }]), false);
+});
+
+// ---------------------------------------------------------------------------
+// renderReport
+// ---------------------------------------------------------------------------
+
+test("renderReport renders a healthy fleet with no Degraded section", () => {
+  const report = {
+    results: [
+      {
+        name: "domio",
+        repo: "dsj1984/domio",
+        verdict: { matchingOnline: 3, shortfall: 0, offline: [], hasOffline: false, hasShortfall: false },
+        staleRuns: [],
+        healthy: true,
+      },
+    ],
+  };
+  const text = renderReport(report);
+  assert.match(text, /✅ healthy/);
+  assert.match(text, /✅ Fleet healthy/);
+  assert.doesNotMatch(text, /### Degraded/);
+});
+
+test("renderReport surfaces offline / shortfall / stale-run detail lines", () => {
+  const report = {
+    results: [
+      {
+        name: "domio",
+        repo: "dsj1984/domio",
+        verdict: {
+          matchingOnline: 1,
+          shortfall: 2,
+          offline: [{ id: 2, name: "domio-runner-2", status: "offline" }],
+          hasOffline: true,
+          hasShortfall: true,
+        },
+        staleRuns: [{ id: 42, html_url: "https://github.com/dsj1984/domio/actions/runs/42", created_at: "x" }],
+        healthy: false,
+      },
+    ],
+  };
+  const text = renderReport(report);
+  assert.match(text, /❌ degraded/);
+  assert.match(text, /OFFLINE runner\(s\)/);
+  assert.match(text, /SHORTFALL/);
+  assert.match(text, /STALE QUEUED RUN/);
+  assert.match(text, /### Degraded/);
+  assert.match(text, /templates\/runbooks\/runner-fleet-health\.md/);
+});
+
+test("renderReport surfaces a fetch error row", () => {
+  const report = { results: [{ name: "domio", repo: "dsj1984/domio", error: "boom" }] };
+  const text = renderReport(report);
+  assert.match(text, /⚠️ error/);
+  assert.match(text, /error — boom/);
+});
+
+// ---------------------------------------------------------------------------
+// fetchRunners / fetchQueuedRuns (injectable runGh)
+// ---------------------------------------------------------------------------
+
+test("fetchRunners returns the runners array from the API response", () => {
+  const runGh = () =>
+    JSON.stringify({ total_count: 1, runners: [{ id: 1, name: "r1", status: "online", labels: [] }] });
+  const runners = fetchRunners("dsj1984/domio", runGh);
+  assert.equal(runners.length, 1);
+  assert.equal(runners[0].name, "r1");
+});
+
+test("fetchRunners returns [] on a 404", () => {
+  const runGh = () => {
+    const err = new Error("gh: Not Found (HTTP 404)");
+    throw err;
+  };
+  assert.deepEqual(fetchRunners("dsj1984/domio", runGh), []);
+});
+
+test("fetchRunners propagates a non-404 error", () => {
+  const runGh = () => {
+    throw new Error("gh: Forbidden (HTTP 403)");
+  };
+  assert.throws(() => fetchRunners("dsj1984/domio", runGh));
+});
+
+test("fetchQueuedRuns merges queued and waiting runs", () => {
+  const runGh = (args) => {
+    const path = args[1];
+    if (path.includes("status=queued")) {
+      return JSON.stringify({ workflow_runs: [{ id: 1, status: "queued", created_at: "x" }] });
+    }
+    if (path.includes("status=waiting")) {
+      return JSON.stringify({ workflow_runs: [{ id: 2, status: "waiting", created_at: "y" }] });
+    }
+    throw new Error(`unexpected path ${path}`);
+  };
+  const runs = fetchQueuedRuns("dsj1984/domio", runGh);
+  assert.equal(runs.length, 2);
+  assert.deepEqual(runs.map((r) => r.id).sort(), [1, 2]);
+});
+
+test("fetchQueuedRuns returns [] on a 404", () => {
+  const runGh = () => {
+    throw new Error("gh: Not Found (HTTP 404)");
+  };
+  assert.deepEqual(fetchQueuedRuns("dsj1984/domio", runGh), []);
+});
+
+// ---------------------------------------------------------------------------
+// buildReport / hasUnhealthy
+// ---------------------------------------------------------------------------
+
+const CONFIG = {
+  defaultStaleQueuedMinutes: 20,
+  repos: [
+    { name: "domio", repo: "dsj1984/domio", expectedCount: 2, labels: ["self-hosted", "domio-runner"] },
+  ],
+};
+
+test("buildReport marks a repo healthy when runners are all online and matching", () => {
+  const runGh = (args) => {
+    const path = args[1];
+    if (path.includes("actions/runners")) {
+      return JSON.stringify({
+        runners: [
+          { id: 1, name: "r1", status: "online", labels: [{ name: "self-hosted" }, { name: "domio-runner" }] },
+          { id: 2, name: "r2", status: "online", labels: [{ name: "self-hosted" }, { name: "domio-runner" }] },
+        ],
+      });
+    }
+    if (path.includes("actions/runs")) {
+      return JSON.stringify({ workflow_runs: [] });
+    }
+    throw new Error(`unexpected ${path}`);
+  };
+  const report = buildReport(CONFIG, runGh, NOW);
+  assert.equal(report.results[0].healthy, true);
+  assert.equal(hasUnhealthy(report), false);
+});
+
+test("buildReport marks a repo unhealthy and records a fetch error as unhealthy", () => {
+  const runGh = (args) => {
+    const path = args[1];
+    if (path.includes("actions/runners")) throw new Error("gh: Service Unavailable (HTTP 503)");
+    return "{}";
+  };
+  const report = buildReport(CONFIG, runGh, NOW);
+  assert.equal(report.results[0].error !== undefined, true);
+  assert.equal(hasUnhealthy(report), true);
+});
+
+// ---------------------------------------------------------------------------
+// runCli (end-to-end against a temp config + injected runGh)
+// ---------------------------------------------------------------------------
+
+test("runCli exits 0 for a healthy fleet and 1 for a degraded one", () => {
+  const dir = mkdtempSync(join(tmpdir(), "runner-health-cli-"));
+  try {
+    const configPath = join(dir, "runner-fleet-consumers.json");
+    writeFileSync(
+      configPath,
+      JSON.stringify({
+        defaultStaleQueuedMinutes: 20,
+        repos: [
+          { name: "domio", repo: "dsj1984/domio", expectedCount: 2, labels: ["self-hosted", "domio-runner"] },
+        ],
+      }),
+    );
+    const stdout = { buf: "", write(s) { this.buf += s; } };
+    const stderr = { buf: "", write(s) { this.buf += s; } };
+    const healthyRunGh = (args) => {
+      const path = args[1];
+      if (path.includes("actions/runners")) {
+        return JSON.stringify({
+          runners: [
+            { id: 1, name: "r1", status: "online", labels: [{ name: "self-hosted" }, { name: "domio-runner" }] },
+            { id: 2, name: "r2", status: "online", labels: [{ name: "self-hosted" }, { name: "domio-runner" }] },
+          ],
+        });
+      }
+      if (path.includes("actions/runs")) return JSON.stringify({ workflow_runs: [] });
+      throw new Error(`unexpected ${path}`);
+    };
+    const code = runCli({
+      argv: ["--config", configPath],
+      cwd: process.cwd(),
+      stdout,
+      stderr,
+      runGh: healthyRunGh,
+      summaryPath: undefined,
+      nowMs: NOW,
+    });
+    assert.equal(code, 0);
+    assert.match(stdout.buf, /Fleet healthy|Runner-fleet health dashboard/);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+test("runCli returns 1 on a bad config path", () => {
+  const stdout = { buf: "", write(s) { this.buf += s; } };
+  const stderr = { buf: "", write(s) { this.buf += s; } };
+  const code = runCli({
+    argv: ["--config", "scripts/does-not-exist.json"],
+    cwd: process.cwd(),
+    stdout,
+    stderr,
+    runGh: () => "{}",
+  });
+  assert.equal(code, 1);
+  assert.match(stderr.buf, /failed to read config/);
+});

package/scripts/deploy-boot-smoke.mjs

@@ -40,10 +40,15 @@
  * appended to $GITHUB_ENV so the workflow's `Rollback failed workers` step
  * fires. Exit code 1 on any smoke failure.
  *
- * Wrangler: the workers.dev subdomain derivation shells out to the
- * consumer's lockfile-pinned `pnpm exec wrangler whoami` (installed by
- * setup-toolchain, preflighted via its `require-wrangler` input). This
- * script never fetches a registry-latest wrangler.
+ * Subdomain derivation (M14): the workers.dev account subdomain is derived
+ * via the Cloudflare REST endpoint
+ * `GET /accounts/{account_id}/workers/subdomain` using the in-scope
+ * CLOUDFLARE_API_TOKEN + CLOUDFLARE_ACCOUNT_ID — NOT `wrangler whoami`. This
+ * removes the root-wrangler dependency for probe-only consumers: every real
+ * consumer is a pnpm workspace with wrangler in an app sub-package (not the
+ * root), so the former `pnpm exec wrangler whoami` derivation failed the
+ * smoke preflight AFTER the worker already deployed. The REST call needs no
+ * wrangler at all.
  *
  * Environment contract (all read from process.env):
  *   DEPLOYED_WORKERS       csv of deployed worker names (required)
@@ -53,6 +58,8 @@
  *   WORKERS_DEV_SUBDOMAIN  explicit workers.dev slug (optional)
  *   VERIFY_COMMIT_SHA      'true' to assert the health JSON version field
  *   EXPECTED_SHA           the SHA verify-commit-sha asserts (github.sha)
+ *   CLOUDFLARE_API_TOKEN   CF API token for the subdomain REST derivation
+ *   CLOUDFLARE_ACCOUNT_ID  CF account id for the subdomain REST derivation
  *   SMOKE_FAILED_FILE      rollback-list path (default: a freshly-created
  *                          private temp dir via mkdtemp — never a predictable
  *                          world-writable path; CI sets this explicitly)
@@ -61,13 +68,16 @@
  * Exit codes:
  *   0 — every probe passed.
  *   1 — a probe failed (rollback list written), or the probe target could
- *       not be resolved (no subdomain derivable — no rollback list).
+ *       not be resolved (no subdomain derivable — no rollback list), or an
+ *       unhandled error crashed the run (terminal-write-before-exit — the
+ *       rollback file + smoke_failed=true flag are still written so the
+ *       workflow's rollback step fires, M2).
  */
 
 import { writeFileSync, appendFileSync, mkdtempSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { execFileSync, spawnSync } from "node:child_process";
+import { spawnSync } from "node:child_process";
 
 // ---------------------------------------------------------------------------
 // Pure helpers (unit-tested)
@@ -87,12 +97,28 @@ export function parseSmokePaths(csv) {
 }
 
 /**
- * Extract the workers.dev account subdomain slug from `wrangler whoami`
- * output (the first `<slug>.workers.dev` token). Returns null when absent.
+ * Extract the workers.dev account subdomain slug from a Cloudflare REST
+ * `GET /accounts/{account_id}/workers/subdomain` response body (M14). The API
+ * returns `{ success, result: { name: "<slug>" } }`; this reads the top-level
+ * `result.name`. Returns null when the body is not JSON, not the expected
+ * shape, unsuccessful, or the name is missing / not a non-empty string.
+ * Deliberately tolerant of a full `<slug>.workers.dev` value (some responses
+ * echo the host) by stripping a trailing `.workers.dev`.
  */
-export function extractSubdomain(whoamiOutput) {
-  const m = String(whoamiOutput ?? "").match(/([A-Za-z0-9-]+)\.workers\.dev/);
-  return m ? m[1] : null;
+export function extractSubdomain(responseBody) {
+  let parsed;
+  try {
+    parsed = JSON.parse(responseBody);
+  } catch {
+    return null;
+  }
+  if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return null;
+  if (parsed.success === false) return null;
+  const result = parsed.result;
+  if (result === null || typeof result !== "object" || Array.isArray(result)) return null;
+  const name = result.name;
+  if (typeof name !== "string" || name.length === 0) return null;
+  return name.replace(/\.workers\.dev$/, "");
 }
 
 /**
@@ -199,7 +225,7 @@ export async function runSmoke(env, deps = {}) {
     log = (line) => process.stdout.write(`${line}\n`),
     probe = probeUrl,
     runShell = defaultRunShell,
-    whoami = defaultWhoami,
+    deriveSubdomain = defaultDeriveSubdomain,
   } = deps;
 
   const workers = parseCsv(env.DEPLOYED_WORKERS);
@@ -228,15 +254,20 @@ export async function runSmoke(env, deps = {}) {
   // ----- Built-in probe -----
   // Resolve the workers.dev subdomain slug. NEVER the account ID (a UUID) —
   // workers.dev subdomains are keyed by the account name/slug. Prefer the
-  // explicit input, else derive from `wrangler whoami`.
+  // explicit input, else derive from the Cloudflare REST endpoint
+  // GET /accounts/{account_id}/workers/subdomain (M14 — no wrangler needed).
   let subdomain = (env.WORKERS_DEV_SUBDOMAIN ?? "").trim();
   if (!subdomain && !smokeBaseUrl) {
-    log("::group::Deriving workers.dev subdomain from wrangler whoami");
-    subdomain = extractSubdomain(whoami());
+    log("::group::Deriving workers.dev subdomain from the Cloudflare REST API");
+    subdomain = await deriveSubdomain({
+      accountId: (env.CLOUDFLARE_ACCOUNT_ID ?? "").trim(),
+      apiToken: (env.CLOUDFLARE_API_TOKEN ?? "").trim(),
+    });
     if (!subdomain) {
       log(
-        "::error::Could not derive workers.dev subdomain from 'wrangler whoami'. " +
-          "Pass workers_dev_subdomain or smoke_base_url explicitly."
+        "::error::Could not derive workers.dev subdomain from the Cloudflare REST API " +
+          "(GET /accounts/{account_id}/workers/subdomain). Check CLOUDFLARE_ACCOUNT_ID / " +
+          "CLOUDFLARE_API_TOKEN, or pass workers_dev_subdomain / smoke_base_url explicitly."
       );
       log("::endgroup::");
       // No rollback list: the target could not be resolved, so nothing was
@@ -306,18 +337,30 @@ function defaultRunShell(command, extraEnv) {
   return res.status ?? 1;
 }
 
-function defaultWhoami() {
-  // Consumer lockfile-pinned wrangler (`pnpm exec wrangler`), installed and
-  // preflighted by setup-toolchain (require-wrangler). Failure tolerated —
-  // an empty output falls through to the "could not derive" error above.
+/**
+ * Derive the workers.dev subdomain slug via the Cloudflare REST endpoint
+ * `GET /accounts/{account_id}/workers/subdomain` (M14). Uses the in-scope
+ * CLOUDFLARE_* creds — no wrangler. Any failure (missing creds, network
+ * error, non-2xx, unparsable body) is tolerated and returns null, which the
+ * caller surfaces as the "could not derive" error above. fetchImpl is
+ * injectable for the test suite.
+ */
+export async function defaultDeriveSubdomain({ accountId, apiToken }, fetchImpl = fetch) {
+  if (!accountId || !apiToken) return null;
+  const url = `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(accountId)}/workers/subdomain`;
   try {
-    return execFileSync("pnpm", ["exec", "wrangler", "whoami"], {
-      encoding: "utf8",
-      stdio: ["ignore", "pipe", "ignore"],
-      maxBuffer: 8 * 1024 * 1024,
+    const res = await fetchImpl(url, {
+      method: "GET",
+      headers: {
+        Authorization: `Bearer ${apiToken}`,
+        Accept: "application/json",
+      },
+      signal: AbortSignal.timeout(15000),
     });
+    if (!res.ok) return null;
+    return extractSubdomain(await res.text());
   } catch {
-    return "";
+    return null;
   }
 }
 
@@ -341,18 +384,55 @@ export function resolveFailedFile(env, mkdtempImpl = mkdtempSync) {
   return join(mkdtempImpl(join(tmpdir(), "deploy-boot-smoke-")), "smoke-failed-workers.txt");
 }
 
+/**
+ * Write the rollback terminal state: the failed-worker list to `failedFile`
+ * (overwrite, never append) and `smoke_failed=true` to $GITHUB_ENV so the
+ * workflow's rollback step fires. No-op when `failedWorkers` is empty.
+ * Extracted so both the normal smoke-failure path AND the crash path (M2 —
+ * terminal-write-before-exit) share one writer, and so the write is
+ * unit-testable via injected fs seams.
+ *
+ * Overwrite rationale: a reused self-hosted runner may carry a stale list
+ * from a previous run, and rolling back workers this run never deployed would
+ * widen the blast radius.
+ */
+export function writeRollbackState(
+  failedFile,
+  failedWorkers,
+  { githubEnv, writeFile = writeFileSync, appendFile = appendFileSync } = {}
+) {
+  if (!failedWorkers || failedWorkers.length === 0) return;
+  writeFile(failedFile, `${failedWorkers.join("\n")}\n`);
+  if (githubEnv) {
+    appendFile(githubEnv, "smoke_failed=true\n");
+  }
+}
+
 async function main() {
   const failedFile = resolveFailedFile(process.env);
-  const { exitCode, failedWorkers } = await runSmoke(process.env);
-
-  if (failedWorkers.length > 0) {
-    // Overwrite (never append): a reused self-hosted runner may carry a
-    // stale list from a previous run, and rolling back workers this run
-    // never deployed would widen the blast radius.
-    writeFileSync(failedFile, `${failedWorkers.join("\n")}\n`);
-    if (process.env.GITHUB_ENV) {
-      appendFileSync(process.env.GITHUB_ENV, "smoke_failed=true\n");
-    }
+  let exitCode;
+  try {
+    const result = await runSmoke(process.env);
+    exitCode = result.exitCode;
+    writeRollbackState(failedFile, result.failedWorkers, {
+      githubEnv: process.env.GITHUB_ENV,
+    });
+  } catch (err) {
+    // Crash path (M2): an unhandled error must NOT leave a deployed worker
+    // serving unverified code with no rollback. Write the terminal rollback
+    // state — rolling back EVERY deployed worker, since the crash gives no
+    // per-worker attribution — BEFORE exiting non-zero, so the workflow's
+    // rollback step still fires. This is the terminal-write-before-exit
+    // guarantee the former (write-only-on-a-clean-return) shape lacked.
+    process.stdout.write(
+      `::error::deploy-boot-smoke crashed: ${err?.message ?? err}. ` +
+        "Marking every deployed worker for rollback.\n"
+    );
+    const deployed = uniqueSorted(parseCsv(process.env.DEPLOYED_WORKERS));
+    writeRollbackState(failedFile, deployed, {
+      githubEnv: process.env.GITHUB_ENV,
+    });
+    exitCode = 1;
   }
   process.exit(exitCode);
 }