@intentius/chant 0.124.30 → 0.124.31

package/src/workspace/ci-run.test.ts

@@ -12,12 +12,13 @@
 
 import { execFileSync } from "node:child_process";
 import { createPublicKey, generateKeyPairSync } from "node:crypto";
-import { chmodSync, existsSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
+import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
-import { afterAll, describe, expect, test } from "vitest";
-import { cleanScratch, declaration, git, scratchDir, writeFiles } from "./__fixtures__/contract-repo";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { cleanScratch, declaration, git, repoFixture, scratchDir, writeFiles } from "./__fixtures__/contract-repo";
 import { checkRunEventLines } from "./__fixtures__/run-event-check";
-import { ciJobCommand, ciJobEnv, ciJobFailure, ciJobOp, ciRun, CiRunError, formatCiRun } from "./ci-run";
+import { ciJobCommand, ciJobEnv, ciJobFailure, ciJobOp, ciRun, CiRunError, ciStatusContext, ciStatusFromEnv, ciStatusUrl, formatCiRun } from "./ci-run";
+import type { CommitStatus } from "../pr-forge";
 import { ciTick, evaluateCommit, readOpRunSource, revokedAnnotation } from "./ci-green";
 import type { StepRecord } from "../op/local-executor";
 import { sshFromEd25519 } from "./trust/dsse";
@@ -25,6 +26,8 @@ import { verifyOpRunStatement } from "./trust/op-run-statement";
 import { readRunLedger } from "../lifecycle/run-ledger";
 import { prLoopJobs, PR_LOOP_SETUP } from "../components/pr-pipeline";
 import { parseArgs } from "../cli/parse-args";
+import { ciSecretsFor, readCiSecrets } from "./ci-secrets";
+import { runEventFileName, runEventsDir } from "../run-event-store";
 
 afterAll(cleanScratch);
 
@@ -51,11 +54,17 @@ function commitAt(root: string, hours: number, file: string): string {
   return git(root, "rev-parse", "HEAD");
 }
 
-/** A checkout on main with a bare origin: member `app`, `ci.green` from runs, and BOT as the runner `chant-runner`. */
-function workspace(): { root: string; app: string } {
-  const origin = scratchDir("chant-ci-run-origin-");
+/**
+ * A checkout on main with a bare origin: member `app`, `ci.green` from runs, and BOT as the runner `chant-runner`.
+ * The history is two commits, `base` and then `sha`, which changes app/a.txt, and both are pushed to origin.
+ */
+const workspaceTemplate = repoFixture(() => {
+  const top = scratchDir("chant-ci-run-template-");
+  const origin = join(top, "origin");
+  const root = join(top, "root");
+  mkdirSync(origin);
+  mkdirSync(root);
   git(origin, "init", "-q", "--bare");
-  const root = scratchDir("chant-ci-run-");
   git(root, "init", "-q");
   git(root, "symbolic-ref", "HEAD", "refs/heads/main");
   git(root, "config", "user.name", "t");
@@ -68,7 +77,17 @@ function workspace(): { root: string; app: string } {
   });
   git(root, "remote", "add", "origin", origin);
   commitAt(root, 23, "app/init.txt");
-  return { root, app: join(root, "app") };
+  commitAt(root, 2, "app/a.txt");
+  git(root, "push", "-q", "origin", "main");
+  return top;
+}, "chant-ci-run-");
+
+/** A copy of the template repository, its origin pointed at the copy of the bare one. */
+function workspace(): { root: string; app: string; base: string; sha: string } {
+  const top = workspaceTemplate();
+  const root = join(top, "root");
+  git(root, "remote", "set-url", "origin", join(top, "origin"));
+  return { root, app: join(root, "app"), base: git(root, "rev-parse", "HEAD~1"), sha: git(root, "rev-parse", "HEAD") };
 }
 
 /**
@@ -183,23 +202,29 @@ describe("the job as an Op (#4094)", () => {
 });
 
 describe("chant ci run, then ci tick from runs (#4094)", () => {
-  test("plan and apply run in a fresh worktree, are recorded signed, and the tick tags the commit green", async () => {
-    const before = { worktrees: undefined as string | undefined };
-    const { root, app } = workspace();
-    const base = git(root, "rev-parse", "HEAD");
-    const sha = commitAt(root, 2, "app/a.txt");
-    git(root, "push", "-q", "origin", "main");
-    before.worktrees = git(root, "worktree", "list", "--porcelain");
-    const chant = fakeChant();
-    const reports = scratchDir("chant-ci-run-reports-");
-
-    const events: string[] = [];
-    const plan = await ciRun({ image: JOB_IMAGE,
-      cwd: app,
+  // The plan and the apply run once, in beforeAll, and the tests below read what they left. A job
+  // takes a few hundred milliseconds here and several times that on a loaded runner, so two jobs and
+  // a tick in one test came close to the unit budget (#2817). A file's beforeAll is not counted in it.
+  let w: ReturnType<typeof workspace>;
+  let worktrees: string;
+  let chant: string;
+  let reports: string;
+  let events: string[];
+  let plan: Awaited<ReturnType<typeof ciRun>>;
+  let apply: Awaited<ReturnType<typeof ciRun>>;
+
+  beforeAll(async () => {
+    w = workspace();
+    worktrees = git(w.root, "worktree", "list", "--porcelain");
+    chant = fakeChant();
+    reports = scratchDir("chant-ci-run-reports-");
+    events = [];
+    plan = await ciRun({ image: JOB_IMAGE,
+      cwd: w.app,
       job: "plan",
       env: "prod",
       forge: "forgejo",
-      base,
+      base: w.base,
       pr: 7,
       chant,
       hostEnv: HOST,
@@ -209,6 +234,11 @@ describe("chant ci run, then ci tick from runs (#4094)", () => {
       runId: "chant-plan-1",
       runEvents: (line) => events.push(line),
     });
+    apply = await ciRun({ image: JOB_IMAGE, cwd: w.app, job: "apply", env: "prod", forge: "forgejo", base: w.base, chant, hostEnv: HOST, runnerKey: BOT.pem, output: join(reports, "apply") });
+  });
+
+  test("the plan runs in a fresh worktree, in the member's directory, with a clean environment", () => {
+    const { root, base, sha } = w;
     expect(plan.status).toBe("ok");
     expect(plan.warnings).toEqual([]);
     expect(plan.signer).toBe("chant-runner");
@@ -224,14 +254,18 @@ describe("chant ci run, then ci tick from runs (#4094)", () => {
     expect(pwd).toBe(join(plan.checkout, "app"));
     expect(pwd.startsWith(realpathSync(root))).toBe(false);
     expect(existsSync(plan.checkout)).toBe(false);
-    expect(git(root, "worktree", "list", "--porcelain")).toBe(before.worktrees);
+    expect(git(root, "worktree", "list", "--porcelain")).toBe(worktrees);
     expect(JSON.parse(readFileSync(join(out, "pr-plan.json"), "utf-8"))).toEqual({ stage: "pr-plan" });
 
-    // The environment is clean: the job's variables and the few the runner hands on.
+    // The environment is clean: the job's variables and the few the runner hands on. The copied
+    // report shows the values of the variables whose names look secret as *** (#4096).
     const env = envFile(out, "pr-plan");
-    expect(env).toMatchObject({ BASE_SHA: base, PR_NUMBER: "7", GITHUB_SHA: sha, CHANT_ENV: "prod", GITHUB_TOKEN: "forge-token", AWS_ACCESS_KEY_ID: "floci" });
+    expect(env).toMatchObject({ BASE_SHA: base, PR_NUMBER: "7", GITHUB_SHA: sha, CHANT_ENV: "prod", GITHUB_TOKEN: "***", AWS_ACCESS_KEY_ID: "***" });
     for (const name of ["NOT_FOR_JOBS", "CHANT_RUNNER_KEY", "GITHUB_RUN_ID"]) expect(env).not.toHaveProperty(name);
+  });
 
+  test("the plan and the apply are recorded signed, with a pipeline run's events", async () => {
+    const { app, base, sha } = w;
     // The record: Op pr-plan at the commit, run by chant, with a statement that verifies.
     expect(plan.record).toMatchObject({ op: "pr-plan", env: "prod", status: "ok", gitSha: sha, runOrigin: { forge: "chant" }, id: "chant-plan-1" });
     const stored = (await readRunLedger("prod", "pr-plan", { cwd: app })).records.at(-1)!;
@@ -244,13 +278,14 @@ describe("chant ci run, then ci tick from runs (#4094)", () => {
     expect(stream[0]).toMatchObject({ kind: "pipeline", runtime: "ci", name: "pr-plan", runId: "chant-plan-1", gitSha: sha, runOrigin: { forge: "chant" } });
     expect(stream.at(-1)).toMatchObject({ type: "run-settle", status: "ok" });
 
-    const apply = await ciRun({ image: JOB_IMAGE, cwd: app, job: "apply", env: "prod", forge: "forgejo", base, chant, hostEnv: HOST, runnerKey: BOT.pem, output: join(reports, "apply") });
     expect(apply.status).toBe("ok");
     const applyArgs = readFileSync(join(reports, "apply", "pr-apply.args"), "utf-8").trimEnd().split("\n");
     expect(applyArgs).toEqual(["components", "pr-apply", "--base", base, "--env", "prod", "--gate", "pr-apply", "--output", ".chant/pr", "--forge", "forgejo", "--member", "app", "--require-review", "--resume", ".chant/pr-resume/pr-apply.json"]);
     expect(formatCiRun(apply)).toContain("signed by chant-runner");
+  });
 
-    // The records reached origin, and a tick from runs tags the commit green with no forge.
+  test("the records reached origin, and a tick from runs tags the commit green with no forge", async () => {
+    const { root, app, sha } = w;
     expect(git(root, "ls-remote", "origin", "chant/lifecycle")).toContain(git(root, "rev-parse", "chant/lifecycle"));
     const tick = await ciTick({ cwd: app, now: NOW, runnerKey: BOT.pem });
     expect(tick.source).toBe("runs");
@@ -262,9 +297,7 @@ describe("chant ci run, then ci tick from runs (#4094)", () => {
   });
 
   test("a failing job is recorded failed, and a key from the environment that trust.json does not list leaves it unsigned", async () => {
-    const { root, app } = workspace();
-    const base = git(root, "rev-parse", "HEAD");
-    commitAt(root, 2, "app/a.txt");
+    const { app, base } = workspace();
     const stranger = runnerKey();
     const lines: string[] = [];
     const r = await ciRun({ image: JOB_IMAGE,
@@ -300,9 +333,7 @@ describe("chant ci run, then ci tick from runs (#4094)", () => {
   });
 
   test("--pr-build runs the job's build and lint lines after --before-script and before the command, and a failing build fails the job", async () => {
-    const { root, app } = workspace();
-    const base = git(root, "rev-parse", "HEAD");
-    const sha = commitAt(root, 2, "app/a.txt");
+    const { root, app, base, sha } = workspace();
     const chant = fakeChant();
     const out = join(scratchDir("chant-ci-run-build-"), "pr");
     const ok = await ciRun({ image: JOB_IMAGE,
@@ -349,9 +380,7 @@ describe("chant ci run, then ci tick from runs (#4094)", () => {
   });
 
   test("a failed job's record and settle event say why from its pr report, signed, and a revoke note names the code", async () => {
-    const { root, app } = workspace();
-    const base = git(root, "rev-parse", "HEAD");
-    const sha = commitAt(root, 2, "app/a.txt");
+    const { root, app, base, sha } = workspace();
     const refusedReport = { stage: "apply", status: "refused", refusal: "not-a-reviewer", message: "ana approved the plan, and ana has not approved the pull request on the forge" };
     const extra = { FAKE_EXIT: "3", FAKE_REPORT: JSON.stringify(refusedReport) };
     const events: string[] = [];
@@ -414,3 +443,274 @@ describe("chant ci run, then ci tick from runs (#4094)", () => {
     expect(at([ok("build"), fail("apply", 1)])!.code).toBe("command-failed");
   });
 });
+
+describe("a commit status per job (#4097)", () => {
+  /** A fake forge client: it records each status, and fails on the states in `refuse`. */
+  function fakeForge(refuse: string[] = []): { client: { setCommitStatus(sha: string, status: CommitStatus): Promise<void> }; posts: { sha: string; status: CommitStatus }[] } {
+    const posts: { sha: string; status: CommitStatus }[] = [];
+    return {
+      posts,
+      client: {
+        async setCommitStatus(sha, status) {
+          if (refuse.includes(status.state)) throw new Error(`POST /statuses/${sha} answered 500`);
+          posts.push({ sha, status });
+        },
+      },
+    };
+  }
+
+  async function job(over: Record<string, unknown>) {
+    const { root, app, base, sha } = workspace();
+    const r = await ciRun({ image: JOB_IMAGE, cwd: app, job: "plan", env: "prod", forge: "forgejo", base, pr: 7, chant: fakeChant(), hostEnv: HOST, push: false, output: join(scratchDir("chant-ci-status-"), "pr"), ...over } as Parameters<typeof ciRun>[0]);
+    return { r, sha };
+  }
+
+  test("a good job posts pending, then success, at its commit", async () => {
+    const forge = fakeForge();
+    const { r, sha } = await job({ status: { client: forge.client } });
+    expect(r.status).toBe("ok");
+    expect(r.warnings).toEqual([]);
+    expect(forge.posts.map((p) => [p.sha, p.status.context, p.status.state])).toEqual([[sha, "chant/pr-plan", "pending"], [sha, "chant/pr-plan", "success"]]);
+    expect(forge.posts[0]!.status.description).toBe("pr-plan running");
+    expect(forge.posts[1]!.status.description).toMatch(/^pr-plan passed in \d+s$/);
+    expect(forge.posts[1]!.status).not.toHaveProperty("targetUrl");
+  });
+
+  test("a failed job posts pending, then failure with the run record's code and reason", async () => {
+    const forge = fakeForge();
+    const report = JSON.stringify({ stage: "apply", status: "refused", refusal: "not-approved", message: "no reviewer approved this plan" });
+    const { r } = await job({ job: "apply", pr: undefined, hostEnv: { ...HOST, FAKE_EXIT: "1", FAKE_REPORT: report }, passEnv: ["FAKE_EXIT", "FAKE_REPORT"], status: { client: forge.client } });
+    expect(r.status).toBe("fail");
+    expect(forge.posts.map((p) => [p.status.context, p.status.state])).toEqual([["chant/pr-apply", "pending"], ["chant/pr-apply", "failure"]]);
+    expect(r.record.failure!.code).toBe("not-approved");
+    expect(forge.posts[1]!.status.description).toBe("not-approved: no reviewer approved this plan");
+  });
+
+  test("a long failure reason is cut to the forge's 140 characters", async () => {
+    const forge = fakeForge();
+    const report = JSON.stringify({ stage: "apply", status: "refused", refusal: "plan-changed", message: "y".repeat(400) });
+    await job({ job: "apply", pr: undefined, hostEnv: { ...HOST, FAKE_EXIT: "1", FAKE_REPORT: report }, passEnv: ["FAKE_EXIT", "FAKE_REPORT"], status: { client: forge.client } });
+    const d = forge.posts[1]!.status.description;
+    expect(d).toHaveLength(140);
+    expect(d.startsWith("plan-changed: yyy")).toBe(true);
+    expect(d.endsWith("...")).toBe(true);
+  });
+
+  test("a renamed gate names the apply's context", async () => {
+    const forge = fakeForge();
+    await job({ job: "apply", pr: undefined, loop: { gate: "prod-approval" }, status: { client: forge.client } });
+    expect(forge.posts.map((p) => p.status.context)).toEqual(["chant/prod-approval", "chant/prod-approval"]);
+    expect(ciStatusContext("plan", "prod-approval")).toBe("chant/pr-plan");
+  });
+
+  // One job per test: each runs a whole Op in a worktree, and three in one test went over the unit budget.
+  test("target_url is an https --report-url with the key", async () => {
+    const https = fakeForge();
+    const { sha } = await job({ loop: { reportUrl: "https://host/reports/" }, status: { client: https.client } });
+    const key = `7/${sha}/app`;
+    expect(https.posts.map((p) => p.status.targetUrl)).toEqual([`https://host/reports/${key}/`, `https://host/reports/${key}/`]);
+  });
+
+  /**
+   * An `aws` stand-in first on PATH for the test. The run index of an s3 --report-url is read and
+   * written with the AWS CLI, and the real one reaches for credentials and the network: seconds on a
+   * runner that has it. The stub takes a put and answers a get with a missing object.
+   */
+  function withStubAws(): () => void {
+    const dir = scratchDir("chant-ci-run-aws-");
+    const file = join(dir, "aws");
+    writeFileSync(file, ['#!/bin/sh', 'if [ "$3" = "-" ]; then cat > /dev/null; exit 0; fi', 'echo "An error occurred (404) when calling the HeadObject operation: Key not found" >&2', "exit 1", ""].join("\n"));
+    chmodSync(file, 0o755);
+    const saved = process.env.PATH;
+    process.env.PATH = `${dir}:${saved}`;
+    return () => {
+      process.env.PATH = saved;
+    };
+  }
+
+  test("target_url is the --status-url template for an s3 --report-url", async () => {
+    const s3 = fakeForge();
+    const restore = withStubAws();
+    try {
+      const mapped = await job({ loop: { reportUrl: "s3://bucket/prefix" }, status: { client: s3.client, urlTemplate: "https://host/reports/{key}" } });
+      expect(s3.posts.map((p) => p.status.targetUrl)).toEqual([`https://host/reports/7/${mapped.sha}/app`, `https://host/reports/7/${mapped.sha}/app`]);
+    } finally {
+      restore();
+    }
+  });
+
+  test("an s3 or file --report-url with no --status-url gives no target_url", async () => {
+    const bare = fakeForge();
+    const restore = withStubAws();
+    try {
+      await job({ loop: { reportUrl: "s3://bucket/prefix" }, status: { client: bare.client } });
+    } finally {
+      restore();
+    }
+    expect(bare.posts.every((p) => p.status.targetUrl === undefined)).toBe(true);
+    expect(ciStatusUrl({ reportUrl: "file:///tmp/r", commit: "abc" })).toBeUndefined();
+  });
+
+  test("--no-status, a missing token and GitLab give no client; a bad --status-url is refused", () => {
+    const env = { GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "acme/estate", GITHUB_SERVER_URL: "http://forgejo.localhost", GITHUB_API_URL: "http://forgejo.localhost/api/v1" };
+    expect(ciStatusFromEnv({ forge: "forgejo", hostEnv: env, noStatus: true })).toEqual({});
+    expect(ciStatusFromEnv({ forge: "forgejo", hostEnv: { GITHUB_REPOSITORY: "acme/estate" } })).toEqual({});
+    expect(ciStatusFromEnv({ forge: "gitlab", hostEnv: { ...env, GITLAB_TOKEN: "t" } })).toEqual({});
+    expect(ciStatusFromEnv({ forge: "forgejo", hostEnv: env }).status?.client.setCommitStatus).toBeTypeOf("function");
+    expect(ciStatusFromEnv({ forge: "forgejo", hostEnv: env, statusUrl: "https://h/{key}" }).status?.urlTemplate).toBe("https://h/{key}");
+    expect(ciStatusFromEnv({ forge: "forgejo", hostEnv: { GITHUB_TOKEN: "t" } }).warning).toMatch(/no commit status/);
+    expect(() => ciStatusFromEnv({ forge: "forgejo", hostEnv: env, statusUrl: "s3://b/{key}" })).toThrow(/--status-url takes an http or https URL/);
+    const flags = parseArgs(["ci", "run", "plan", "--no-status", "--status-url", "https://h/{key}"]);
+    expect(flags).toMatchObject({ noStatus: true, statusUrl: "https://h/{key}" });
+  });
+
+  test("a status the forge refuses is a warning, and the job stays ok", async () => {
+    const forge = fakeForge(["pending", "success"]);
+    const { r } = await job({ status: { client: forge.client } });
+    expect(r.status).toBe("ok");
+    expect(r.record.status).toBe("ok");
+    expect(r.warnings).toHaveLength(2);
+    expect(r.warnings[0]).toMatch(/could not set the chant\/pr-plan status: POST \/statuses\/[0-9a-f]+ answered 500/);
+  });
+});
+
+/** A secrets file only its owner can read, as `--secrets` takes it. */
+function secretsFile(text: string, name = "secrets.env"): string {
+  const file = join(scratchDir("chant-ci-run-secrets-"), name);
+  writeFileSync(file, text, { mode: 0o600 });
+  chmodSync(file, 0o600);
+  return file;
+}
+
+/** Every file under `dir`, with its text. */
+function filesUnder(dir: string): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const entry of readdirSync(dir, { recursive: true, withFileTypes: true })) {
+    if (!entry.isFile()) continue;
+    const path = join(entry.parentPath, entry.name);
+    out[path] = readFileSync(path, "utf-8");
+  }
+  return out;
+}
+
+describe("the job's secrets from the host (#4096)", () => {
+  test("a secret reaches the step, and the file's allowlist keeps one from the plan", async () => {
+    const { app, base } = workspace();
+    const secrets = readCiSecrets(
+      secretsFile(JSON.stringify({ secrets: { DB_PASSWORD: "hunter22-db", DEPLOY_KEY: "deploy-key-value" }, jobs: { plan: ["DB_PASSWORD"] } }), "secrets.json"),
+    );
+    const out = join(scratchDir("chant-ci-run-secret-"), "pr");
+    const r = await ciRun({
+      image: JOB_IMAGE,
+      cwd: app,
+      job: "plan",
+      env: "prod",
+      forge: "forgejo",
+      base,
+      pr: 5,
+      chant: fakeChant(),
+      hostEnv: HOST,
+      secrets,
+      // The step checks the value it got, and that the plan got no DEPLOY_KEY.
+      beforeScript: ['test "$DB_PASSWORD" = "hunter22-db" && test -z "${DEPLOY_KEY+set}" && mkdir -p .chant/pr && echo reached > .chant/pr/reached'],
+      push: false,
+      output: out,
+    });
+    expect(r.status).toBe("ok");
+    expect(readFileSync(join(out, "reached"), "utf-8").trim()).toBe("reached");
+    const env = envFile(out, "pr-plan");
+    // The value reached the command too, and the copied report shows it as ***.
+    expect(env.DB_PASSWORD).toBe("***");
+    expect(env).not.toHaveProperty("DEPLOY_KEY");
+    expect(env).not.toHaveProperty("CHANT_RUNNER_KEY");
+
+    // The apply is not in the allowlist, so it gets every secret.
+    const [, apply] = prLoopJobs("forgejo", "prod");
+    const applyEnv = ciJobEnv(apply!, { forge: "forgejo", env: "prod", values: { base, commit: base }, hostEnv: HOST, secrets: ciSecretsFor(secrets, "apply") });
+    expect(applyEnv).toMatchObject({ DB_PASSWORD: "hunter22-db", DEPLOY_KEY: "deploy-key-value" });
+    // A secret can replace a forge token, and never the variables the runner sets.
+    const over = ciJobEnv(apply!, { forge: "forgejo", env: "prod", values: { base, commit: base }, hostEnv: HOST, secrets: { GITHUB_TOKEN: "from-file", BASE_SHA: "x", CHANT_RUNNER_KEY: "k" } });
+    expect(over).toMatchObject({ GITHUB_TOKEN: "from-file", BASE_SHA: base });
+    expect(over).not.toHaveProperty("CHANT_RUNNER_KEY");
+    expect(() => ciJobEnv(apply!, { forge: "forgejo", env: "prod", values: { base, commit: base }, hostEnv: HOST, passEnv: ["CHANT_RUNNER_KEY"] })).toThrow(/never passed to a job/);
+  });
+
+  test("a secret the job prints, writes in its report or fails with is *** in the events, the signed record, the copied report and the output", async () => {
+    const { app, base } = workspace();
+    const SECRET = "s3cret pass#1";
+    const TOKEN = "tok-abc123";
+    const secrets = readCiSecrets(secretsFile(`# the host's secrets\nDB_PASSWORD="${SECRET}"\nexport PIN=abc\n`));
+    expect(secrets.values).toEqual({ DB_PASSWORD: SECRET, PIN: "abc" });
+
+    const leaky = join(scratchDir("chant-ci-run-bin-"), "chant");
+    writeFileSync(
+      leaky,
+      [
+        "#!/bin/sh",
+        "mkdir -p .chant/pr",
+        'env | sort > .chant/pr/pr-apply.env',
+        'printf \'{"stage":"apply","status":"refused","refusal":"not-approved","message":"the deploy failed with %s"}\' "$DB_PASSWORD" > .chant/pr/pr-apply.json',
+        'printf \'{\\n  "entries": [\\n    { "password": "%s" }\\n  ]\\n}\\n\' "$DB_PASSWORD" > .chant/pr/change-set.json',
+        'echo "connecting with $DB_PASSWORD and $MY_API_TOKEN, pin $PIN"',
+        'echo "login refused for $DB_PASSWORD" >&2',
+        "exit 3",
+        "",
+      ].join("\n"),
+    );
+    chmodSync(leaky, 0o755);
+
+    const events: string[] = [];
+    const lines: string[] = [];
+    const out = join(scratchDir("chant-ci-run-leak-"), "pr");
+    const r = await ciRun({
+      image: JOB_IMAGE,
+      cwd: app,
+      job: "apply",
+      env: "prod",
+      forge: "forgejo",
+      base,
+      chant: leaky,
+      hostEnv: { ...HOST, MY_API_TOKEN: TOKEN },
+      passEnv: ["MY_API_TOKEN"],
+      secrets,
+      runnerKey: BOT.pem,
+      push: false,
+      output: out,
+      runId: "chant-leak-1",
+      runEvents: (line) => events.push(line),
+      stepOutput: (line) => lines.push(line),
+    });
+    expect(r.status).toBe("fail");
+    const leaks = (text: string): boolean => text.includes(SECRET) || text.includes(TOKEN);
+
+    // The events, as written and as kept on this host, still valid lines.
+    expect(events.some(leaks)).toBe(false);
+    const stream = checkRunEventLines(events);
+    expect(stream.at(-1)).toMatchObject({ type: "run-settle", status: "failed", failure: { code: "not-approved", reason: "the deploy failed with ***" } });
+    const kept = readFileSync(join((await runEventsDir(app))!, runEventFileName("chant-leak-1")), "utf-8");
+    expect(leaks(kept)).toBe(false);
+    expect(kept).toContain("***");
+
+    // The record: the step's error and failure.reason redacted, and the statement still verifies.
+    expect(leaks(JSON.stringify(r.record))).toBe(false);
+    expect(r.record.failure).toMatchObject({ code: "not-approved", reason: "the deploy failed with ***", exitCode: 3 });
+    const stored = (await readRunLedger("prod", "pr-apply", { cwd: app })).records.at(-1)!;
+    expect(leaks(JSON.stringify(stored))).toBe(false);
+    expect(stored.phases.at(-1)!.steps[0]!.error).toContain("login refused for ***");
+    expect(verifyOpRunStatement(stored, RUNNERS, { runner: ["chant-runner"] })).toMatchObject({ status: "signed", signer: "chant-runner" });
+
+    // The copied report and change set: redacted, and still JSON in the layout the job wrote.
+    const files = filesUnder(out);
+    expect(Object.values(files).some(leaks)).toBe(false);
+    expect(JSON.parse(files[join(out, "pr-apply.json")]!)).toMatchObject({ message: "the deploy failed with ***" });
+    expect(files[join(out, "change-set.json")]).toBe('{\n  "entries": [\n    {\n      "password": "***"\n    }\n  ]\n}\n');
+    expect(envFile(out, "pr-apply")).toMatchObject({ DB_PASSWORD: "***", MY_API_TOKEN: "***" });
+
+    // What ci run prints: the step's lines, the result's failure and the summary. A value under 4 characters is kept.
+    expect(lines.some(leaks)).toBe(false);
+    expect(lines.some((l) => l.endsWith("| connecting with *** and ***, pin abc"))).toBe(true);
+    expect(leaks(r.failure!.error)).toBe(false);
+    expect(leaks(formatCiRun(r))).toBe(false);
+    expect(formatCiRun(r)).toContain("not-approved in apply, exit 3: the deploy failed with ***");
+  });
+});