mandrel-platform 1.2.0 → 1.3.0

package/scripts/check-environments-isolation-audit.test.mjs

@@ -0,0 +1,212 @@
+#!/usr/bin/env node
+/**
+ * check-environments-isolation-audit.test.mjs — behavioural guard for the
+ * environments isolation audit's verdicts (Story #367).
+ *
+ * The bug this pins: reading an environment's `deployment_branch_policy`
+ * requires repo Administration: read, which the default GITHUB_TOKEN cannot be
+ * granted. Without it GitHub omits the field from the response entirely — and
+ * the audit read that absence as "this environment has NO deployment branch
+ * policy" and failed with a security finding it had never observed. A check
+ * that reports the insecure conclusion when the truth is that it could not look
+ * is a check operators learn to wave through, which is why no consumer in the
+ * fleet had it enabled.
+ *
+ * Absent-because-unreadable and absent-because-unset are different states and
+ * must produce different outcomes. Reading the YAML cannot prove that: what
+ * decides the verdict is a shell branch over a `jq` probe. So this extracts the
+ * real `run:` body and executes it against a stub `gh` serving fixture
+ * responses — the same read-then-execute approach as
+ * check-setup-toolchain-store.test.mjs / check-osv-scan-mode.test.mjs.
+ *
+ * Run: node --test scripts/check-environments-isolation-audit.test.mjs
+ */
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { execFileSync } from "node:child_process";
+import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+import { stepByName, runScript } from "./lib/yaml-step.mjs";
+
+const ACTION = ".github/actions/environments-isolation-audit/action.yml";
+const REPO = "acme/widgets";
+
+const auditScript = runScript(stepByName(readFileSync(ACTION, "utf8"), "Audit deployment branch policies"));
+
+/**
+ * Run the extracted audit body against a stub `gh` that serves `responses`
+ * keyed by API path. A path with no fixture makes the stub exit non-zero with
+ * no output — exactly how `gh api` behaves on a 404 or a permission failure.
+ *
+ * @param {object} opts
+ * @param {Record<string, unknown>} opts.responses  API path → JSON body.
+ * @param {string} [opts.environments]              ENVIRONMENTS_CSV.
+ * @param {string} [opts.allowedBranch]             ALLOWED_BRANCH.
+ * @returns {{ code: number, output: string }}  `output` is stdout+stderr;
+ *   GitHub's `::error::` annotations are written to stdout.
+ */
+function runAudit({ responses, environments = "staging", allowedBranch = "main" }) {
+  const dir = mkdtempSync(path.join(tmpdir(), "env-isolation-audit-"));
+  try {
+    const fixtures = path.join(dir, "fixtures");
+    mkdirSync(fixtures);
+    for (const [apiPath, body] of Object.entries(responses)) {
+      writeFileSync(path.join(fixtures, apiPath.replace(/\//g, "_") + ".json"), JSON.stringify(body));
+    }
+    const stub = path.join(dir, "gh");
+    writeFileSync(
+      stub,
+      "#!/bin/sh\n" +
+        'f="$FIXTURE_DIR/$(printf %s "$2" | tr / _).json"\n' +
+        'if [ -f "$f" ]; then cat "$f"; exit 0; fi\n' +
+        'echo "gh: Not Found (HTTP 404)" >&2\n' +
+        "exit 1\n"
+    );
+    chmodSync(stub, 0o755);
+    const script = path.join(dir, "audit.sh");
+    writeFileSync(script, auditScript);
+    const env = {
+      PATH: `${dir}${path.delimiter}${process.env.PATH}`,
+      FIXTURE_DIR: fixtures,
+      GITHUB_REPOSITORY: REPO,
+      AUDIT_REPO: "",
+      ENVIRONMENTS_CSV: environments,
+      ALLOWED_BRANCH: allowedBranch,
+    };
+    try {
+      const stdout = execFileSync("bash", [script], { cwd: dir, encoding: "utf8", env });
+      return { code: 0, output: stdout };
+    } catch (err) {
+      return { code: err.status, output: `${err.stdout ?? ""}${err.stderr ?? ""}` };
+    }
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+const ENV_API = `repos/${REPO}/environments/staging`;
+const POLICIES_API = `${ENV_API}/deployment-branch-policies`;
+
+// ── the canonical posture still passes ─────────────────────────────────────
+
+test("an environment restricted to the allowed branch passes", () => {
+  const res = runAudit({
+    responses: {
+      [ENV_API]: {
+        name: "staging",
+        deployment_branch_policy: { protected_branches: false, custom_branch_policies: true },
+      },
+      [POLICIES_API]: { total_count: 1, branch_policies: [{ name: "main" }] },
+    },
+  });
+  assert.equal(res.code, 0, res.output);
+  assert.match(res.output, /restricts deploys to 'main'/);
+});
+
+// ── unreadable ≠ unset (the Story #367 split) ──────────────────────────────
+
+test("a policy field the token cannot read is reported as unreadable, not as absent", () => {
+  // What GitHub actually returns without Administration: read — the key is
+  // simply not in the response.
+  const res = runAudit({ responses: { [ENV_API]: { name: "staging", id: 1 } } });
+  assert.equal(res.code, 1);
+  assert.match(res.output, /UNREADABLE/);
+  assert.match(res.output, /Administration: read/);
+  assert.doesNotMatch(
+    res.output,
+    /has NO deployment branch policy/,
+    "the audit must not assert a conclusion it never observed"
+  );
+});
+
+test("a genuinely unset policy still fails as a real finding", () => {
+  const res = runAudit({
+    responses: { [ENV_API]: { name: "staging", deployment_branch_policy: null } },
+  });
+  assert.equal(res.code, 1);
+  assert.match(res.output, /has NO deployment branch policy/);
+  assert.doesNotMatch(res.output, /UNREADABLE/, "a null field is observed, not unreadable");
+});
+
+test("the two absent-policy states produce different messages", () => {
+  const unreadable = runAudit({ responses: { [ENV_API]: { name: "staging" } } }).output;
+  const unset = runAudit({
+    responses: { [ENV_API]: { name: "staging", deployment_branch_policy: null } },
+  }).output;
+  assert.notEqual(unreadable, unset);
+});
+
+test("an unreadable read is summarised as not-a-verdict at the end of the run", () => {
+  const res = runAudit({ responses: { [ENV_API]: { name: "staging" } } });
+  assert.match(res.output, /are NOT policy verdicts/);
+});
+
+test("unreadable named branch policies are not reported as ZERO policies", () => {
+  // The policy field reads fine; the follow-up policies call fails.
+  const res = runAudit({
+    responses: {
+      [ENV_API]: {
+        name: "staging",
+        deployment_branch_policy: { protected_branches: false, custom_branch_policies: true },
+      },
+    },
+  });
+  assert.equal(res.code, 1);
+  assert.match(res.output, /named branch policies UNREADABLE/);
+  assert.doesNotMatch(res.output, /ZERO named policies/);
+});
+
+// ── every other finding is unchanged ───────────────────────────────────────
+
+test("protected-branches-only still fails with its own message", () => {
+  const res = runAudit({
+    responses: {
+      [ENV_API]: {
+        name: "staging",
+        deployment_branch_policy: { protected_branches: true, custom_branch_policies: false },
+      },
+    },
+  });
+  assert.equal(res.code, 1);
+  assert.match(res.output, /protected branches only/);
+  assert.doesNotMatch(res.output, /UNREADABLE/);
+});
+
+test("zero named policies, a wildcard, and a wrong branch each still fail", () => {
+  const custom = {
+    name: "staging",
+    deployment_branch_policy: { protected_branches: false, custom_branch_policies: true },
+  };
+  const zero = runAudit({
+    responses: { [ENV_API]: custom, [POLICIES_API]: { total_count: 0, branch_policies: [] } },
+  });
+  assert.equal(zero.code, 1);
+  assert.match(zero.output, /ZERO named policies/);
+
+  const wildcard = runAudit({
+    responses: {
+      [ENV_API]: custom,
+      [POLICIES_API]: { total_count: 1, branch_policies: [{ name: "release/*" }] },
+    },
+  });
+  assert.equal(wildcard.code, 1);
+  assert.match(wildcard.output, /wildcard branch policy/);
+
+  const wrongBranch = runAudit({
+    responses: {
+      [ENV_API]: custom,
+      [POLICIES_API]: { total_count: 1, branch_policies: [{ name: "develop" }] },
+    },
+  });
+  assert.equal(wrongBranch.code, 1);
+  assert.match(wrongBranch.output, /allows branch 'develop', not 'main'/);
+});
+
+test("a missing environment still fails with the does-not-exist message", () => {
+  const res = runAudit({ responses: {} });
+  assert.equal(res.code, 1);
+  assert.match(res.output, /does not exist on/);
+});

package/scripts/check-fail-fast-attribution.test.mjs

@@ -136,12 +136,29 @@ const jobs = extractJobs();
 //    are wired into every tier job that participates in fail-fast.
 // ---------------------------------------------------------------------------
 
-test("the collateral explainer fires only on cancellation under fail-fast", () => {
+test("the collateral explainer fires on cancellation and nothing else", () => {
+  // Story #364 widened this from `cancelled() && inputs.fail-fast` to
+  // `cancelled()`: a job killed for exceeding its own `timeout-minutes` is also
+  // recorded as `cancelled`, and with fail-fast off nothing explained it at all.
+  // `cancelled()` remains the ceiling — an `always()` gate would fire the step
+  // on the green path, and `success()`/bare truthiness would fire it on a real
+  // failure, where the tier's own logs are the evidence.
   assert.match(
     explainStep,
-    /^\s+if:\s+\$\{\{\s*cancelled\(\)\s*&&\s*inputs\.fail-fast\s*\}\}\s*$/m,
-    "`&explain-cancellation` must be gated `cancelled() && inputs.fail-fast` — " +
-      "an `always()` gate would fire it on the green path"
+    /^\s+if:\s+\$\{\{\s*cancelled\(\)\s*\}\}\s*$/m,
+    "`&explain-cancellation` must be gated `cancelled()` alone"
+  );
+  assert.doesNotMatch(
+    explainStep,
+    /^\s+if:\s+\$\{\{\s*always\(\)/m,
+    "an `always()` gate would fire the explainer on the green path"
+  );
+  // The fail-fast input still has to reach the script — it is what separates a
+  // collateral cancellation from a timeout kill.
+  assert.match(
+    explainStep,
+    /^\s+FAIL_FAST:\s+\$\{\{\s*inputs\.fail-fast\s*\}\}\s*$/m,
+    "the explainer must receive inputs.fail-fast to classify the cancellation"
   );
 });
 
@@ -288,6 +305,9 @@ function runStep(script, { ghExit = 0, ghStdout = "", env = {} } = {}) {
         RUN_ID: "30168441137",
         REPO: "Beestera/swarm-os",
         TIER_ID: "typecheck",
+        // The collateral branch is the fail-fast-on case; a case that is about
+        // the timeout branch overrides this explicitly.
+        FAIL_FAST: "true",
         ...env,
       },
     });
@@ -369,6 +389,7 @@ test("collateral side: still explains itself when gh is absent entirely", () =>
         REPO: "o/r",
         TIER_ID: "lint",
         GH_TOKEN: "stub-token",
+        FAIL_FAST: "true",
       },
     });
     assert.equal(r.status, 0, r.stderr);
@@ -379,6 +400,71 @@ test("collateral side: still explains itself when gh is absent entirely", () =>
   }
 });
 
+// ---------------------------------------------------------------------------
+// Timeout provenance (Story #364) — the other half of the same step. A job
+// killed for exceeding `timeout-minutes` is recorded as `cancelled`, not
+// `failure`, and `ci-required` reads `cancelled` as a red gate. With every work
+// step green, the operator previously had only timings to go on.
+// ---------------------------------------------------------------------------
+
+test("timeout side: a cancellation with no failing sibling is not reported as collateral", () => {
+  // Lookup SUCCEEDS and returns no failing tier — evidence, not an unknown.
+  const r = runStep(explainScript, { ghExit: 0, ghStdout: "" });
+  assert.equal(r.status, 0, r.stderr);
+  assert.doesNotMatch(
+    r.stdout,
+    /fail-fast collateral/,
+    "blaming a sibling that did not fail is the same misattribution in reverse"
+  );
+  assert.match(r.stdout, /::notice title=cancelled without a failing sibling::/);
+});
+
+test("timeout side: names the job's own ceiling and the post-job teardown that eats it", () => {
+  const r = runStep(explainScript, { ghExit: 0, ghStdout: "" });
+  assert.match(r.summary, /timeout-minutes/, "the summary must name the ceiling");
+  assert.match(
+    r.summary,
+    /cache save/,
+    "the summary must name the teardown that runs after the last work step"
+  );
+  assert.match(
+    r.summary,
+    /toolchain-cache/,
+    "the summary must point at the input that turns the save off"
+  );
+  assert.match(r.summary, /run-wide cancel/i, "the other live cause must stay named");
+});
+
+test("timeout side: an unknown lookup under fail-fast stays collateral, not a timeout claim", () => {
+  // Absence of evidence is not evidence of absence: a failed lookup must not be
+  // read as "no sibling failed", or a genuine collateral cancel gets a timeout
+  // diagnosis and the operator triages the wrong job.
+  const r = runStep(explainScript, { ghExit: 1 });
+  assert.match(r.stdout, /fail-fast collateral/);
+  assert.doesNotMatch(r.stdout, /cancelled without a failing sibling/);
+});
+
+test("timeout side: with fail-fast disabled the cancellation is never collateral", () => {
+  // fail-fast off means no sibling could have cancelled this job, whatever the
+  // lookup returns.
+  const r = runStep(explainScript, {
+    ghExit: 0,
+    ghStdout: "e2e",
+    env: { FAIL_FAST: "false" },
+  });
+  assert.equal(r.status, 0, r.stderr);
+  assert.doesNotMatch(r.stdout, /fail-fast collateral/);
+  assert.match(r.summary, /fail-fast is disabled on this run/);
+});
+
+test("timeout side: stays non-fatal and emits no failure annotation", () => {
+  // Same terms as the collateral branch — an explainer that can fail a job
+  // would turn a diagnostic into a second failure mode.
+  const r = runStep(explainScript, { ghExit: 0, ghStdout: "", env: { FAIL_FAST: "false" } });
+  assert.equal(r.status, 0, r.stderr);
+  assert.doesNotMatch(r.stdout, /::error/);
+});
+
 test("collateral side: never claims this tier failed", () => {
   const r = runStep(explainScript, { ghStdout: "Accessibility (2/3)" });
   assert.doesNotMatch(

package/scripts/check-first-party-pin-freshness.mjs

@@ -26,13 +26,26 @@
  * consumer on a self-hosted fleet kept leaking ~164 MB per run into the
  * host-shared temp root on the latest release.
  *
+ * ## Why the comparison is the DIRECTORY, not the manifest (Story #379)
+ *
+ * The first cut of this checker compared only `action.yml`, which is
+ * structurally unable to protect a composite action whose behaviour lives in a
+ * sibling script — the majority of this repo's action surface. Story #365
+ * rewrote `.github/actions/osv-scan/osv-report-gate.mjs` (+189/-12) without
+ * touching `action.yml`, so this guard reported `osv-scan` fresh while both
+ * call sites ran a 689-line gate against 866 lines on `main`. The comparison
+ * is therefore the whole subpath tree — every file `git ls-tree -r <sha> --
+ * <subpath>` names, plus every tracked working-tree file under it, so an added
+ * or removed sibling is drift too.
+ *
  * This checker closes it by classifying every first-party SHA pin into one of
  * two failure classes — deliberately kept distinct, because their remedies
  * differ:
  *
- *   • `stale`       — the manifest AT THE PINNED SHA differs from the
- *                     working-tree manifest at the same subpath. The fix is to
- *                     BUMP the pin to a commit carrying the current manifest.
+ *   • `stale`       — the SUBPATH TREE at the pinned SHA differs from the
+ *                     working-tree copy — any file under it, not just the
+ *                     manifest. The fix is to BUMP the pin to a commit
+ *                     carrying the current tree.
  *   • `unreachable` — the pinned SHA is not an ancestor of the checked-out
  *                     ref. Typically a pre-squash branch commit: content-
  *                     identical to `main` today, resolvable only until GitHub
@@ -279,9 +292,108 @@ export function createGit(repoRoot) {
         return null;
       }
     },
+    /**
+     * Repo-relative paths of every blob a subpath covers AT `sha`. A directory
+     * subpath yields its whole tree; a file subpath yields just itself. `-z`
+     * so a path with a space or a quote survives intact.
+     */
+    lsTree(sha, subpath) {
+      try {
+        return run(["ls-tree", "-r", "--name-only", "-z", sha, "--", subpath])
+          .split("\0")
+          .filter(Boolean);
+      } catch {
+        return [];
+      }
+    },
+    /**
+     * Repo-relative paths of every TRACKED working-tree file under a subpath.
+     * Tracked, not on-disk: an ignored build artefact or a stray `.DS_Store`
+     * inside an action directory is not something a consumer ever runs.
+     */
+    lsFiles(subpath) {
+      try {
+        return run(["ls-files", "-z", "--", subpath]).split("\0").filter(Boolean);
+      } catch {
+        return [];
+      }
+    },
   };
 }
 
+// ---------------------------------------------------------------------------
+// Subpath tree comparison
+// ---------------------------------------------------------------------------
+
+/** How each drift kind reads in the report. */
+const DRIFT_PHRASE = {
+  differs: "differs from the working-tree copy",
+  added: "is absent at the pinned SHA (added since)",
+  removed: "is gone from the working tree (removed since)",
+  unreadable: "is tracked but unreadable in the working tree",
+};
+
+/**
+ * Compare every file a `uses:` subpath covers at `sha` against the working
+ * tree, and return one record per drifting path (empty when the tree matches).
+ *
+ * The union of both sides is walked, so a sibling script ADDED or REMOVED
+ * since the pinned revision is drift just as much as one whose bytes changed —
+ * all three change what the pinned revision actually executes.
+ *
+ * @param {{lsTree: Function, lsFiles: Function, show: Function}} git
+ * @param {string} repoRoot
+ * @param {string} sha
+ * @param {string} subpath
+ * @returns {Array<{path: string, kind: "differs" | "added" | "removed" | "unreadable"}>}
+ */
+export function diffSubpathAtSha(git, repoRoot, sha, subpath) {
+  const pinned = new Set(git.lsTree(sha, subpath));
+  const working = new Set(git.lsFiles(subpath));
+  const drift = [];
+
+  for (const path of [...new Set([...pinned, ...working])].sort()) {
+    if (!working.has(path)) {
+      drift.push({ path, kind: "removed" });
+      continue;
+    }
+    if (!pinned.has(path)) {
+      drift.push({ path, kind: "added" });
+      continue;
+    }
+    let workingBody;
+    try {
+      workingBody = readFileSync(join(repoRoot, path), "utf8");
+    } catch {
+      drift.push({ path, kind: "unreadable" });
+      continue;
+    }
+    const pinnedBody = git.show(sha, path);
+    if (pinnedBody === null || !manifestsMatch(pinnedBody, workingBody)) {
+      drift.push({ path, kind: "differs" });
+    }
+  }
+
+  return drift;
+}
+
+/**
+ * Render a drift list as the one-line `reason` a finding carries. Action
+ * directories hold a handful of files, so every drifting path is named rather
+ * than summarised — the operator needs to know WHICH file is inert.
+ *
+ * @param {string} subpath
+ * @param {ReturnType<typeof diffSubpathAtSha>} drift
+ * @returns {string}
+ */
+export function describeDrift(subpath, drift) {
+  const detail = drift.map((d) => `${d.path} ${DRIFT_PHRASE[d.kind]}`).join("; ");
+  return (
+    `${drift.length} file(s) under ${subpath} lag the pinned SHA — the pinned ` +
+    `revision is what actually runs: ${detail}`
+  );
+}
+
 // ---------------------------------------------------------------------------
 // Check
 // ---------------------------------------------------------------------------
@@ -343,6 +455,18 @@ export function runCheck(opts = {}, git = createGit(resolve(opts.cwd || process.
   const unpinnedRefs = [];
   let scanned = 0;
 
+  // Every call site for a subpath must move together, so the same
+  // (sha, subpath) pair is compared repeatedly — `setup-toolchain` alone has
+  // five. Resolve each tree once.
+  const driftCache = new Map();
+  const driftFor = (sha, subpath) => {
+    const key = `${sha}:${subpath}`;
+    if (!driftCache.has(key)) {
+      driftCache.set(key, diffSubpathAtSha(git, repoRoot, sha, subpath));
+    }
+    return driftCache.get(key);
+  };
+
   for (const file of files) {
     let content;
     try {
@@ -379,8 +503,7 @@ export function runCheck(opts = {}, git = createGit(resolve(opts.cwd || process.
         continue;
       }
 
-      const pinnedBody = git.show(pin.sha, manifest.path);
-      if (pinnedBody === null) {
+      if (git.show(pin.sha, manifest.path) === null) {
         stale.push({
           ...pin,
           reason: `${manifest.path} does not exist at the pinned SHA — the pin predates the manifest`,
@@ -388,19 +511,12 @@ export function runCheck(opts = {}, git = createGit(resolve(opts.cwd || process.
         continue;
       }
 
-      let workingBody;
-      try {
-        workingBody = readFileSync(join(repoRoot, manifest.path), "utf8");
-      } catch {
-        stale.push({ ...pin, reason: `cannot read the working-tree ${manifest.path}` });
-        continue;
-      }
-
-      if (!manifestsMatch(pinnedBody, workingBody)) {
+      const drift = driftFor(pin.sha, pin.subpath);
+      if (drift.length > 0) {
         stale.push({
           ...pin,
           manifest: manifest.path,
-          reason: `the manifest at the pinned SHA differs from the working-tree ${manifest.path} — the pinned revision is what actually runs`,
+          reason: describeDrift(pin.subpath, drift),
         });
       }
     }
@@ -485,13 +601,13 @@ export function runCli(argv, { log = console.log, err = console.error } = {}) {
   if (result.stale.length > 0) {
     err(
       `[pin-freshness] ❌ ${result.stale.length} stale first-party pin(s) — ` +
-        `the pinned manifest lags the working tree:`
+        `the pinned revision lags the working tree:`
     );
     for (const f of result.stale) err(formatFinding(f, "stale"));
     err(
       "[pin-freshness] Bump each pin to a commit on the default branch whose " +
-        "manifest matches the working-tree copy. Every call site for a given " +
-        "subpath must move together (check-action-pins.mjs enforces the " +
+        "action directory matches the working-tree copy. Every call site for a " +
+        "given subpath must move together (check-action-pins.mjs enforces the " +
         "single-pin invariant per subpath)."
     );
   }
@@ -516,9 +632,9 @@ export function runCli(argv, { log = console.log, err = console.error } = {}) {
   }
 
   log(
-    `[pin-freshness] ✅ all ${result.scanned} first-party pin(s) resolve to a manifest ` +
-      `matching the working tree and reachable from ${opts.ref} (${result.headSha.slice(0, 7)}); ` +
-      `${result.files.length} file(s) scanned.`
+    `[pin-freshness] ✅ all ${result.scanned} first-party pin(s) resolve to an action ` +
+      `directory matching the working tree and reachable from ${opts.ref} ` +
+      `(${result.headSha.slice(0, 7)}); ${result.files.length} file(s) scanned.`
   );
   return 0;
 }