mandrel-platform 1.1.0 → 1.3.0

package/scripts/check-environments-isolation-audit.test.mjs

@@ -0,0 +1,212 @@
+#!/usr/bin/env node
+/**
+ * check-environments-isolation-audit.test.mjs — behavioural guard for the
+ * environments isolation audit's verdicts (Story #367).
+ *
+ * The bug this pins: reading an environment's `deployment_branch_policy`
+ * requires repo Administration: read, which the default GITHUB_TOKEN cannot be
+ * granted. Without it GitHub omits the field from the response entirely — and
+ * the audit read that absence as "this environment has NO deployment branch
+ * policy" and failed with a security finding it had never observed. A check
+ * that reports the insecure conclusion when the truth is that it could not look
+ * is a check operators learn to wave through, which is why no consumer in the
+ * fleet had it enabled.
+ *
+ * Absent-because-unreadable and absent-because-unset are different states and
+ * must produce different outcomes. Reading the YAML cannot prove that: what
+ * decides the verdict is a shell branch over a `jq` probe. So this extracts the
+ * real `run:` body and executes it against a stub `gh` serving fixture
+ * responses — the same read-then-execute approach as
+ * check-setup-toolchain-store.test.mjs / check-osv-scan-mode.test.mjs.
+ *
+ * Run: node --test scripts/check-environments-isolation-audit.test.mjs
+ */
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { execFileSync } from "node:child_process";
+import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+import { stepByName, runScript } from "./lib/yaml-step.mjs";
+
+const ACTION = ".github/actions/environments-isolation-audit/action.yml";
+const REPO = "acme/widgets";
+
+const auditScript = runScript(stepByName(readFileSync(ACTION, "utf8"), "Audit deployment branch policies"));
+
+/**
+ * Run the extracted audit body against a stub `gh` that serves `responses`
+ * keyed by API path. A path with no fixture makes the stub exit non-zero with
+ * no output — exactly how `gh api` behaves on a 404 or a permission failure.
+ *
+ * @param {object} opts
+ * @param {Record<string, unknown>} opts.responses  API path → JSON body.
+ * @param {string} [opts.environments]              ENVIRONMENTS_CSV.
+ * @param {string} [opts.allowedBranch]             ALLOWED_BRANCH.
+ * @returns {{ code: number, output: string }}  `output` is stdout+stderr;
+ *   GitHub's `::error::` annotations are written to stdout.
+ */
+function runAudit({ responses, environments = "staging", allowedBranch = "main" }) {
+  const dir = mkdtempSync(path.join(tmpdir(), "env-isolation-audit-"));
+  try {
+    const fixtures = path.join(dir, "fixtures");
+    mkdirSync(fixtures);
+    for (const [apiPath, body] of Object.entries(responses)) {
+      writeFileSync(path.join(fixtures, apiPath.replace(/\//g, "_") + ".json"), JSON.stringify(body));
+    }
+    const stub = path.join(dir, "gh");
+    writeFileSync(
+      stub,
+      "#!/bin/sh\n" +
+        'f="$FIXTURE_DIR/$(printf %s "$2" | tr / _).json"\n' +
+        'if [ -f "$f" ]; then cat "$f"; exit 0; fi\n' +
+        'echo "gh: Not Found (HTTP 404)" >&2\n' +
+        "exit 1\n"
+    );
+    chmodSync(stub, 0o755);
+    const script = path.join(dir, "audit.sh");
+    writeFileSync(script, auditScript);
+    const env = {
+      PATH: `${dir}${path.delimiter}${process.env.PATH}`,
+      FIXTURE_DIR: fixtures,
+      GITHUB_REPOSITORY: REPO,
+      AUDIT_REPO: "",
+      ENVIRONMENTS_CSV: environments,
+      ALLOWED_BRANCH: allowedBranch,
+    };
+    try {
+      const stdout = execFileSync("bash", [script], { cwd: dir, encoding: "utf8", env });
+      return { code: 0, output: stdout };
+    } catch (err) {
+      return { code: err.status, output: `${err.stdout ?? ""}${err.stderr ?? ""}` };
+    }
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+const ENV_API = `repos/${REPO}/environments/staging`;
+const POLICIES_API = `${ENV_API}/deployment-branch-policies`;
+
+// ── the canonical posture still passes ─────────────────────────────────────
+
+test("an environment restricted to the allowed branch passes", () => {
+  const res = runAudit({
+    responses: {
+      [ENV_API]: {
+        name: "staging",
+        deployment_branch_policy: { protected_branches: false, custom_branch_policies: true },
+      },
+      [POLICIES_API]: { total_count: 1, branch_policies: [{ name: "main" }] },
+    },
+  });
+  assert.equal(res.code, 0, res.output);
+  assert.match(res.output, /restricts deploys to 'main'/);
+});
+
+// ── unreadable ≠ unset (the Story #367 split) ──────────────────────────────
+
+test("a policy field the token cannot read is reported as unreadable, not as absent", () => {
+  // What GitHub actually returns without Administration: read — the key is
+  // simply not in the response.
+  const res = runAudit({ responses: { [ENV_API]: { name: "staging", id: 1 } } });
+  assert.equal(res.code, 1);
+  assert.match(res.output, /UNREADABLE/);
+  assert.match(res.output, /Administration: read/);
+  assert.doesNotMatch(
+    res.output,
+    /has NO deployment branch policy/,
+    "the audit must not assert a conclusion it never observed"
+  );
+});
+
+test("a genuinely unset policy still fails as a real finding", () => {
+  const res = runAudit({
+    responses: { [ENV_API]: { name: "staging", deployment_branch_policy: null } },
+  });
+  assert.equal(res.code, 1);
+  assert.match(res.output, /has NO deployment branch policy/);
+  assert.doesNotMatch(res.output, /UNREADABLE/, "a null field is observed, not unreadable");
+});
+
+test("the two absent-policy states produce different messages", () => {
+  const unreadable = runAudit({ responses: { [ENV_API]: { name: "staging" } } }).output;
+  const unset = runAudit({
+    responses: { [ENV_API]: { name: "staging", deployment_branch_policy: null } },
+  }).output;
+  assert.notEqual(unreadable, unset);
+});
+
+test("an unreadable read is summarised as not-a-verdict at the end of the run", () => {
+  const res = runAudit({ responses: { [ENV_API]: { name: "staging" } } });
+  assert.match(res.output, /are NOT policy verdicts/);
+});
+
+test("unreadable named branch policies are not reported as ZERO policies", () => {
+  // The policy field reads fine; the follow-up policies call fails.
+  const res = runAudit({
+    responses: {
+      [ENV_API]: {
+        name: "staging",
+        deployment_branch_policy: { protected_branches: false, custom_branch_policies: true },
+      },
+    },
+  });
+  assert.equal(res.code, 1);
+  assert.match(res.output, /named branch policies UNREADABLE/);
+  assert.doesNotMatch(res.output, /ZERO named policies/);
+});
+
+// ── every other finding is unchanged ───────────────────────────────────────
+
+test("protected-branches-only still fails with its own message", () => {
+  const res = runAudit({
+    responses: {
+      [ENV_API]: {
+        name: "staging",
+        deployment_branch_policy: { protected_branches: true, custom_branch_policies: false },
+      },
+    },
+  });
+  assert.equal(res.code, 1);
+  assert.match(res.output, /protected branches only/);
+  assert.doesNotMatch(res.output, /UNREADABLE/);
+});
+
+test("zero named policies, a wildcard, and a wrong branch each still fail", () => {
+  const custom = {
+    name: "staging",
+    deployment_branch_policy: { protected_branches: false, custom_branch_policies: true },
+  };
+  const zero = runAudit({
+    responses: { [ENV_API]: custom, [POLICIES_API]: { total_count: 0, branch_policies: [] } },
+  });
+  assert.equal(zero.code, 1);
+  assert.match(zero.output, /ZERO named policies/);
+
+  const wildcard = runAudit({
+    responses: {
+      [ENV_API]: custom,
+      [POLICIES_API]: { total_count: 1, branch_policies: [{ name: "release/*" }] },
+    },
+  });
+  assert.equal(wildcard.code, 1);
+  assert.match(wildcard.output, /wildcard branch policy/);
+
+  const wrongBranch = runAudit({
+    responses: {
+      [ENV_API]: custom,
+      [POLICIES_API]: { total_count: 1, branch_policies: [{ name: "develop" }] },
+    },
+  });
+  assert.equal(wrongBranch.code, 1);
+  assert.match(wrongBranch.output, /allows branch 'develop', not 'main'/);
+});
+
+test("a missing environment still fails with the does-not-exist message", () => {
+  const res = runAudit({ responses: {} });
+  assert.equal(res.code, 1);
+  assert.match(res.output, /does not exist on/);
+});

package/scripts/check-fail-fast-attribution.test.mjs

@@ -136,12 +136,29 @@ const jobs = extractJobs();
 //    are wired into every tier job that participates in fail-fast.
 // ---------------------------------------------------------------------------
 
-test("the collateral explainer fires only on cancellation under fail-fast", () => {
+test("the collateral explainer fires on cancellation and nothing else", () => {
+  // Story #364 widened this from `cancelled() && inputs.fail-fast` to
+  // `cancelled()`: a job killed for exceeding its own `timeout-minutes` is also
+  // recorded as `cancelled`, and with fail-fast off nothing explained it at all.
+  // `cancelled()` remains the ceiling — an `always()` gate would fire the step
+  // on the green path, and `success()`/bare truthiness would fire it on a real
+  // failure, where the tier's own logs are the evidence.
   assert.match(
     explainStep,
-    /^\s+if:\s+\$\{\{\s*cancelled\(\)\s*&&\s*inputs\.fail-fast\s*\}\}\s*$/m,
-    "`&explain-cancellation` must be gated `cancelled() && inputs.fail-fast` — " +
-      "an `always()` gate would fire it on the green path"
+    /^\s+if:\s+\$\{\{\s*cancelled\(\)\s*\}\}\s*$/m,
+    "`&explain-cancellation` must be gated `cancelled()` alone"
+  );
+  assert.doesNotMatch(
+    explainStep,
+    /^\s+if:\s+\$\{\{\s*always\(\)/m,
+    "an `always()` gate would fire the explainer on the green path"
+  );
+  // The fail-fast input still has to reach the script — it is what separates a
+  // collateral cancellation from a timeout kill.
+  assert.match(
+    explainStep,
+    /^\s+FAIL_FAST:\s+\$\{\{\s*inputs\.fail-fast\s*\}\}\s*$/m,
+    "the explainer must receive inputs.fail-fast to classify the cancellation"
   );
 });
 
@@ -288,6 +305,9 @@ function runStep(script, { ghExit = 0, ghStdout = "", env = {} } = {}) {
         RUN_ID: "30168441137",
         REPO: "Beestera/swarm-os",
         TIER_ID: "typecheck",
+        // The collateral branch is the fail-fast-on case; a case that is about
+        // the timeout branch overrides this explicitly.
+        FAIL_FAST: "true",
         ...env,
       },
     });
@@ -369,6 +389,7 @@ test("collateral side: still explains itself when gh is absent entirely", () =>
         REPO: "o/r",
         TIER_ID: "lint",
         GH_TOKEN: "stub-token",
+        FAIL_FAST: "true",
       },
     });
     assert.equal(r.status, 0, r.stderr);
@@ -379,6 +400,71 @@ test("collateral side: still explains itself when gh is absent entirely", () =>
   }
 });
 
+// ---------------------------------------------------------------------------
+// Timeout provenance (Story #364) — the other half of the same step. A job
+// killed for exceeding `timeout-minutes` is recorded as `cancelled`, not
+// `failure`, and `ci-required` reads `cancelled` as a red gate. With every work
+// step green, the operator previously had only timings to go on.
+// ---------------------------------------------------------------------------
+
+test("timeout side: a cancellation with no failing sibling is not reported as collateral", () => {
+  // Lookup SUCCEEDS and returns no failing tier — evidence, not an unknown.
+  const r = runStep(explainScript, { ghExit: 0, ghStdout: "" });
+  assert.equal(r.status, 0, r.stderr);
+  assert.doesNotMatch(
+    r.stdout,
+    /fail-fast collateral/,
+    "blaming a sibling that did not fail is the same misattribution in reverse"
+  );
+  assert.match(r.stdout, /::notice title=cancelled without a failing sibling::/);
+});
+
+test("timeout side: names the job's own ceiling and the post-job teardown that eats it", () => {
+  const r = runStep(explainScript, { ghExit: 0, ghStdout: "" });
+  assert.match(r.summary, /timeout-minutes/, "the summary must name the ceiling");
+  assert.match(
+    r.summary,
+    /cache save/,
+    "the summary must name the teardown that runs after the last work step"
+  );
+  assert.match(
+    r.summary,
+    /toolchain-cache/,
+    "the summary must point at the input that turns the save off"
+  );
+  assert.match(r.summary, /run-wide cancel/i, "the other live cause must stay named");
+});
+
+test("timeout side: an unknown lookup under fail-fast stays collateral, not a timeout claim", () => {
+  // Absence of evidence is not evidence of absence: a failed lookup must not be
+  // read as "no sibling failed", or a genuine collateral cancel gets a timeout
+  // diagnosis and the operator triages the wrong job.
+  const r = runStep(explainScript, { ghExit: 1 });
+  assert.match(r.stdout, /fail-fast collateral/);
+  assert.doesNotMatch(r.stdout, /cancelled without a failing sibling/);
+});
+
+test("timeout side: with fail-fast disabled the cancellation is never collateral", () => {
+  // fail-fast off means no sibling could have cancelled this job, whatever the
+  // lookup returns.
+  const r = runStep(explainScript, {
+    ghExit: 0,
+    ghStdout: "e2e",
+    env: { FAIL_FAST: "false" },
+  });
+  assert.equal(r.status, 0, r.stderr);
+  assert.doesNotMatch(r.stdout, /fail-fast collateral/);
+  assert.match(r.summary, /fail-fast is disabled on this run/);
+});
+
+test("timeout side: stays non-fatal and emits no failure annotation", () => {
+  // Same terms as the collateral branch — an explainer that can fail a job
+  // would turn a diagnostic into a second failure mode.
+  const r = runStep(explainScript, { ghExit: 0, ghStdout: "", env: { FAIL_FAST: "false" } });
+  assert.equal(r.status, 0, r.stderr);
+  assert.doesNotMatch(r.stdout, /::error/);
+});
+
 test("collateral side: never claims this tier failed", () => {
   const r = runStep(explainScript, { ghStdout: "Accessibility (2/3)" });
   assert.doesNotMatch(