@intentius/chant 0.124.32 → 0.124.34

package/src/workspace/checks/security.ts

@@ -0,0 +1,305 @@
+/**
+ * The security group checklists, WSP161 to WSP165 (#4168 step 7).
+ *
+ * A workspace opts into a security group with `security` in the
+ * declaration, at the top level or on a member (../security.ts). Each group
+ * has one check here, which reports what the group still lacks in the tree
+ * read. A group the declaration does not name is never checked, so a
+ * workspace that opts into nothing gets no finding from this file (#4168:
+ * security features are opt-in).
+ *
+ * | Id | Group | Codes |
+ * |---|---|---|
+ * | WSP161 | `verified-releases` | `security-signers-missing`, `security-release-gate-missing`, `security-publish-allows-unsigned` |
+ * | WSP162 | `supply-chain` | `security-sbom-disabled`, `security-sbom-unsourced`, `security-publish-no-provenance`, `security-component-no-sbom`, `security-component-no-verify` |
+ * | WSP163 | `who-ran-it` | `security-signed-since-unset`, `security-runner-untrusted`, `security-tick-no-key` |
+ * | WSP164 | `review-rules` | `security-review-not-required`, `security-attribution-any`, `security-gate-unsigned` |
+ * | WSP165 | `isolated-runners` | `security-runner-start-no-key`, `security-runner-start-no-role`, `security-trust-unreadable` |
+ *
+ * The checks read files, never run them: a member's `chant.config.ts` and
+ * `*.component.ts` are scanned as text, and the pipeline files are the
+ * forge files at the workspace root and in each member (`.github/workflows`,
+ * `.forgejo/workflows`, `.gitlab-ci.yml`). A command line is one line of a
+ * pipeline file that runs `chant release publish`, `chant ci tick` or
+ * `chant components pr-apply`.
+ */
+
+import type { WorkspaceCheck, WorkspaceCheckContext, WorkspaceDiagnostic } from "../checks";
+import type { Declaration, Member } from "../declaration";
+import type { ReasonCode } from "../reason-codes";
+import { securityGroupsOf, type SecurityGroup } from "../security";
+import { joinPath, type WorkspaceTree } from "../tree";
+import { runnerStartReadiness } from "../ci-runner-start";
+import type { RecordSource } from "../record-source";
+import { readTrustPolicy, type TrustPolicy } from "../trust/policy";
+import { DEFAULT_SIGNERS_PATH, TRUST_CONFIG_PATH } from "../trust/trust-files";
+
+/** The read contract's codes for the security findings, carried as `code` on each. */
+export const SECURITY_FINDING_CODES = [
+  "security-signers-missing",
+  "security-release-gate-missing",
+  "security-publish-allows-unsigned",
+  "security-sbom-disabled",
+  "security-sbom-unsourced",
+  "security-publish-no-provenance",
+  "security-component-no-sbom",
+  "security-component-no-verify",
+  "security-signed-since-unset",
+  "security-runner-untrusted",
+  "security-tick-no-key",
+  "security-review-not-required",
+  "security-attribution-any",
+  "security-gate-unsigned",
+  "security-runner-start-no-key",
+  "security-runner-start-no-role",
+  "security-trust-unreadable",
+] as const satisfies readonly ReasonCode[];
+
+type SecurityCode = (typeof SECURITY_FINDING_CODES)[number];
+
+export const WSP_SECURITY_VERIFIED_RELEASES = "WSP161";
+export const WSP_SECURITY_SUPPLY_CHAIN = "WSP162";
+export const WSP_SECURITY_WHO_RAN_IT = "WSP163";
+export const WSP_SECURITY_REVIEW_RULES = "WSP164";
+export const WSP_SECURITY_ISOLATED_RUNNERS = "WSP165";
+
+const memberDir = (dir: string) => (dir === "." ? "" : dir);
+
+/** The members under `group`: every member when the top level names it, else those that name it themselves. */
+function membersUnder(declaration: Declaration, group: SecurityGroup): Member[] {
+  return declaration.members.filter((m) => securityGroupsOf(declaration, m.name).has(group));
+}
+
+/** Whether the declaration names `group` anywhere. */
+function declares(declaration: Declaration, group: SecurityGroup): boolean {
+  return declaration.security.includes(group) || declaration.members.some((m) => m.security.includes(group));
+}
+
+/** Where the group is named, for the finding's pointer: the top level, else the first member naming it. */
+function groupPointer(declaration: Declaration, group: SecurityGroup): string {
+  const top = declaration.security.indexOf(group);
+  if (top >= 0) return `/security/${top}`;
+  const m = declaration.members.find((e) => e.security.includes(group));
+  return m ? `${m.pointer}/security/${m.security.indexOf(group)}` : "/security";
+}
+
+export function read(tree: WorkspaceTree, path: string): string | null {
+  if (tree.stat(path) !== "file") return null;
+  try {
+    return tree.read(path);
+  } catch {
+    return null;
+  }
+}
+
+/** The forge pipeline files under `dir`, each as its path and lines. */
+function pipelineFiles(tree: WorkspaceTree, dir: string): { path: string; lines: string[] }[] {
+  const out: { path: string; lines: string[] }[] = [];
+  for (const folder of [".github/workflows", ".forgejo/workflows", ".gitea/workflows"]) {
+    for (const e of tree.list(joinPath(dir, folder)) ?? []) {
+      if (e.type !== "file" || !/\.ya?ml$/.test(e.name)) continue;
+      const path = joinPath(dir, folder, e.name);
+      const text = read(tree, path);
+      if (text !== null) out.push({ path, lines: text.split("\n") });
+    }
+  }
+  const gitlab = read(tree, joinPath(dir, ".gitlab-ci.yml"));
+  if (gitlab !== null) out.push({ path: joinPath(dir, ".gitlab-ci.yml"), lines: gitlab.split("\n") });
+  return out;
+}
+
+/** Every pipeline file of the workspace: its root's, and each member's. */
+export function allPipelineFiles(ctx: WorkspaceCheckContext): { path: string; lines: string[] }[] {
+  const dirs = new Set(["", ...ctx.declaration.members.map((m) => memberDir(m.dir))]);
+  return [...dirs].flatMap((d) => pipelineFiles(ctx.tree, d));
+}
+
+/** The lines of the pipeline files that run a chant command, `command` naming it, such as `release publish`. */
+export function commandLines(ctx: WorkspaceCheckContext, command: RegExp): { path: string; line: number; text: string }[] {
+  return allPipelineFiles(ctx).flatMap((f) => f.lines.flatMap((text, i) => (command.test(text) ? [{ path: f.path, line: i + 1, text }] : [])));
+}
+
+/** The member's `*.component.ts` files, walked a few levels down, past node_modules and build output. */
+export function componentFiles(tree: WorkspaceTree, dir: string, depth = 4): string[] {
+  const out: string[] = [];
+  for (const e of tree.list(dir) ?? []) {
+    if (e.type === "dir") {
+      if (depth > 0 && !["node_modules", "dist", ".git", ".chant", "target"].includes(e.name)) out.push(...componentFiles(tree, joinPath(dir, e.name), depth - 1));
+    } else if (e.name.endsWith(".component.ts")) out.push(joinPath(dir, e.name));
+  }
+  return out;
+}
+
+/** Whether a lockfile chant writes an SBOM from is in `dir` or a directory above it in the tree. */
+function lockfileAbove(tree: WorkspaceTree, dir: string): boolean {
+  let at = dir;
+  for (;;) {
+    if (["package-lock.json", "pom.xml", "Cargo.lock"].some((f) => tree.stat(joinPath(at, f)) === "file")) return true;
+    if (at === "") return false;
+    at = at.includes("/") ? at.slice(0, at.lastIndexOf("/")) : "";
+  }
+}
+
+/** The trust policy in the tree read: `.chant/trust.json` and the signers file it names. */
+function treePolicy(tree: WorkspaceTree): TrustPolicy {
+  const source: RecordSource = {
+    label: tree.label,
+    list: (dir) => tree.list(dir)?.filter((e) => e.type === "file").map((e) => e.name),
+    read: (path) => tree.read(path),
+    bytes: (path) => (tree.bytes ? tree.bytes(path) : new Uint8Array(Buffer.from(tree.read(path), "utf-8"))),
+  };
+  return readTrustPolicy(source, null);
+}
+
+function finding(check: WorkspaceCheck, code: SecurityCode, message: string, at: { pointer: string; entity?: string; treeFile?: string; line?: number }): WorkspaceDiagnostic {
+  return {
+    checkId: check.id,
+    severity: check.severity,
+    code,
+    message: `${code}: ${message}`,
+    pointer: at.pointer,
+    ...(at.entity !== undefined ? { entity: at.entity } : {}),
+    ...(at.treeFile !== undefined ? { treeFile: at.treeFile, line: at.line ?? 1, column: 1 } : {}),
+  };
+}
+
+const RELEASE_PUBLISH = /\bchant\b.*\brelease\s+publish\b/;
+const CI_TICK = /\bchant\b.*\bci\s+tick\b|main\.ts\s+ci\s+tick\b/;
+const PR_APPLY = /\bcomponents\s+pr-apply\b|\bci\s+run\s+apply\b/;
+
+export const SECURITY_CHECKS: readonly WorkspaceCheck[] = [
+  {
+    id: WSP_SECURITY_VERIFIED_RELEASES,
+    name: "security-verified-releases",
+    description: "Under the verified-releases group: a signers file, a release gate naming a class, and no publish job that passes --allow-unsigned.",
+    severity: "warning",
+    configurable: true,
+    check(ctx) {
+      const d = ctx.declaration;
+      if (!declares(d, "verified-releases")) return [];
+      const pointer = groupPointer(d, "verified-releases");
+      const out: WorkspaceDiagnostic[] = [];
+      if (ctx.tree.stat(DEFAULT_SIGNERS_PATH) !== "file") {
+        out.push(finding(this, "security-signers-missing", `the workspace opts into verified-releases, and there is no ${DEFAULT_SIGNERS_PATH}${ctx.tree.label} to verify a release's signature against; add the keys of the people who cut releases to it, in ssh-keygen's allowed_signers format`, { pointer }));
+      }
+      const gate = d.identity?.gates.release;
+      if (!gate?.class) {
+        out.push(finding(this, "security-release-gate-missing", gate ? `identity.gates.release names no class, so any signer in ${DEFAULT_SIGNERS_PATH} may cut a release; name the class that may` : "the workspace opts into verified-releases, and identity.gates has no release entry, so a cut is not held to a class of signers", { pointer: gate?.pointer ?? pointer }));
+      }
+      for (const l of commandLines(ctx, RELEASE_PUBLISH)) {
+        if (!/--allow-unsigned\b/.test(l.text)) continue;
+        out.push(finding(this, "security-publish-allows-unsigned", `${l.path} runs release publish with --allow-unsigned, which publishes a release whose signature does not verify`, { pointer, treeFile: l.path, line: l.line }));
+      }
+      return out;
+    },
+  },
+  {
+    id: WSP_SECURITY_SUPPLY_CHAIN,
+    name: "security-supply-chain",
+    description: "Under the supply-chain group: every publish target attaches an SBOM it has a source for, publish jobs require OIDC provenance, and components generate an SBOM and verify what they deploy.",
+    severity: "warning",
+    configurable: true,
+    check(ctx) {
+      const d = ctx.declaration;
+      if (!declares(d, "supply-chain")) return [];
+      const pointer = groupPointer(d, "supply-chain");
+      const out: WorkspaceDiagnostic[] = [];
+      for (const m of membersUnder(d, "supply-chain")) {
+        const configPath = joinPath(memberDir(m.dir), "chant.config.ts");
+        const config = read(ctx.tree, configPath);
+        if (config !== null && /\bpublish\s*:/.test(config)) {
+          const lines = config.split("\n");
+          const off = lines.findIndex((l) => /\bsbom\s*:\s*false\b/.test(l));
+          if (off >= 0) out.push(finding(this, "security-sbom-disabled", `member ${m.name}'s chant.config.ts sets sbom: false on a publish target, so what it publishes carries no SBOM`, { pointer, entity: m.name, treeFile: configPath, line: off + 1 }));
+          else if (!/\bsbom\s*:\s*["'`]/.test(config) && !lockfileAbove(ctx.tree, memberDir(m.dir))) {
+            out.push(finding(this, "security-sbom-unsourced", `member ${m.name} publishes, names no sbom file and has no package-lock.json, pom.xml or Cargo.lock in its directory or above it, so publish would refuse it as publish-sbom-missing`, { pointer, entity: m.name, treeFile: configPath }));
+          }
+        }
+        const components = componentFiles(ctx.tree, memberDir(m.dir));
+        if (components.length === 0) continue;
+        const text = components.map((f) => read(ctx.tree, f) ?? "").join("\n");
+        if (!/\bgenerateSbom\s*\(|["']generate-sbom["']/.test(text)) {
+          out.push(finding(this, "security-component-no-sbom", `member ${m.name}'s components have no generate-sbom step, so what they build carries no SBOM`, { pointer, entity: m.name, treeFile: components[0] }));
+        }
+        if (!/\bverify\s*\(|["']verify["']/.test(text)) {
+          out.push(finding(this, "security-component-no-verify", `member ${m.name}'s components have no verify step, so a signature or attestation is never checked before a deploy`, { pointer, entity: m.name, treeFile: components[0] }));
+        }
+      }
+      for (const l of commandLines(ctx, RELEASE_PUBLISH)) {
+        if (/--require-provenance\b/.test(l.text)) continue;
+        out.push(finding(this, "security-publish-no-provenance", `${l.path} runs release publish without --require-provenance, so a run with no OIDC token publishes with only a warning`, { pointer, treeFile: l.path, line: l.line }));
+      }
+      return out;
+    },
+  },
+  {
+    id: WSP_SECURITY_WHO_RAN_IT,
+    name: "security-who-ran-it",
+    description: "Under the who-ran-it group: ci.green.signedSince is set, .chant/trust.json lists a runner key, and the tick signs its tags.",
+    severity: "warning",
+    configurable: true,
+    check(ctx) {
+      const d = ctx.declaration;
+      if (!declares(d, "who-ran-it")) return [];
+      const pointer = groupPointer(d, "who-ran-it");
+      const out: WorkspaceDiagnostic[] = [];
+      if (!d.ci?.green?.signedSince) {
+        out.push(finding(this, "security-signed-since-unset", d.ci?.green ? "ci.green sets no signedSince, so an unsigned green tag still counts" : "the workspace opts into who-ran-it, and has no ci.green block, so no commit's green tag is signed", { pointer: d.ci?.green?.pointer ?? pointer }));
+      }
+      const runners = treePolicy(ctx.tree).runners.filter((r) => r.class === "runner").length;
+      if (runners === 0) {
+        out.push(finding(this, "security-runner-untrusted", `${TRUST_CONFIG_PATH}${ctx.tree.label} lists no runner key, so no run or tag can be traced to a runner; add one with chant ci serve --setup-key --trust`, { pointer }));
+      }
+      for (const l of commandLines(ctx, CI_TICK)) {
+        const file = allPipelineFiles(ctx).find((f) => f.path === l.path);
+        if (/--key\b/.test(l.text) || file?.lines.some((t) => /CHANT_RUNNER_KEY/.test(t))) continue;
+        out.push(finding(this, "security-tick-no-key", `${l.path} runs ci tick with no runner key, so its tags are unsigned; regenerate it with chant ci workflow --runner-key-secret <NAME>`, { pointer, treeFile: l.path, line: l.line }));
+      }
+      return out;
+    },
+  },
+  {
+    id: WSP_SECURITY_REVIEW_RULES,
+    name: "security-review-rules",
+    description: "Under the review-rules group: no apply job turns forge review off, attribution is identified, and every gate an apply job names needs a signed approval.",
+    severity: "warning",
+    configurable: true,
+    check(ctx) {
+      const d = ctx.declaration;
+      if (!declares(d, "review-rules")) return [];
+      const pointer = groupPointer(d, "review-rules");
+      const out: WorkspaceDiagnostic[] = [];
+      const applies = commandLines(ctx, PR_APPLY);
+      for (const l of applies) {
+        if (!/--no-require-review\b/.test(l.text)) continue;
+        out.push(finding(this, "security-review-not-required", `${l.path} passes --no-require-review, which the review-rules group overrides; drop it so the file says what runs`, { pointer, treeFile: l.path, line: l.line }));
+      }
+      if (d.identity?.attribution !== "identified") {
+        out.push(finding(this, "security-attribution-any", "identity.attribution is not identified, so a person-attributed write may name anyone", { pointer: d.identity ? "/identity" : pointer }));
+      }
+      const signed = new Set(Object.keys(d.identity?.gates ?? {}));
+      const named = new Set(applies.flatMap((l) => [...l.text.matchAll(/--(?:wave-)?gate\s+["']?([A-Za-z0-9_.-]+)/g)].map((m) => m[1]!)));
+      const unsigned = [...named].filter((g) => !signed.has(g));
+      if (unsigned.length > 0) {
+        out.push(finding(this, "security-gate-unsigned", `the apply jobs name the gate${unsigned.length > 1 ? "s" : ""} ${unsigned.join(", ")}, which identity.gates does not list, so an unsigned approval passes ${unsigned.length > 1 ? "them" : "it"}`, { pointer }));
+      } else if (named.size === 0 && signed.size === 0) {
+        out.push(finding(this, "security-gate-unsigned", "identity.gates lists no gate, so every approval may be unsigned", { pointer }));
+      }
+      return out;
+    },
+  },
+  {
+    id: WSP_SECURITY_ISOLATED_RUNNERS,
+    name: "security-isolated-runners",
+    description: "Under the isolated-runners group: .chant/trust.json lists a runner key whose principal holds the runner role, so an apply job can require a runner-start statement (#4147).",
+    severity: "warning",
+    configurable: true,
+    check(ctx) {
+      const d = ctx.declaration;
+      if (!declares(d, "isolated-runners")) return [];
+      const pointer = groupPointer(d, "isolated-runners");
+      const codes = { "no-runner-key": "security-runner-start-no-key", "no-runner-role": "security-runner-start-no-role", "trust-unreadable": "security-trust-unreadable" } as const;
+      return runnerStartReadiness(treePolicy(ctx.tree)).gaps.map((g) => finding(this, codes[g.gap], `the workspace opts into isolated-runners, so every apply job requires a runner-start statement, and ${g.reason}`, { pointer }));
+    },
+  },
+];

package/src/workspace/checks.ts

@@ -37,6 +37,8 @@
  * | WSP126, WSP127 | box intent: the decision record a box names exists, and constrains a member or path of this workspace (#2850, #2857) |
  * | WSP141, WSP142 | member links resolved against a live read, with `--live` (#2549) |
  * | WSP131 to WSP133 | diagram artifacts: source and render exist, a recorded source hash still matches (#2764) |
+ * | WSP151 | what a member under `ci.workflow` still lacks (#4168 step 6) |
+ * | WSP161 to WSP165 | the security groups the declaration opts into: what each still lacks (#4168 step 7) |
  */
 
 import { realpathSync } from "node:fs";
@@ -58,6 +60,8 @@ import { gatherLiveLinkFacts, LIVE_CHECKS, type LiveLinkFacts } from "./checks/l
 import { RECORD_CHECKS, type RecordFacts } from "./checks/records";
 import { BOX_CHECKS } from "./checks/boxes";
 import { DIAGRAM_CHECKS } from "./checks/diagrams";
+import { SECURITY_CHECKS } from "./checks/security";
+import { CI_WORKFLOW_CHECKS } from "./checks/ci-workflow";
 import { loadDeclaredKinds, type DeclaredKind } from "./declared-kinds";
 import { resolveBoxIntents, type ResolvedBoxIntent } from "./box-intent";
 import { gatherLexiconOutputs, type MemberLexiconOutputs } from "./lexicon-outputs";
@@ -371,6 +375,10 @@ export const WORKSPACE_CHECKS: readonly WorkspaceCheck[] = [
   ...DIAGRAM_CHECKS,
   // Links resolved against a live read, with --live (#2549), WSP141 and WSP142.
   ...LIVE_CHECKS,
+  // What a member under ci.workflow still lacks (#4168 step 6), WSP151.
+  ...CI_WORKFLOW_CHECKS,
+  // The security groups the declaration opts into (#4168 step 7), WSP161 to WSP165.
+  ...SECURITY_CHECKS,
 ];
 
 const BY_ID = new Map(WORKSPACE_CHECKS.map((c) => [c.id, c]));

package/src/workspace/ci-builds.e2e.test.ts

@@ -0,0 +1,208 @@
+/**
+ * #4169 end to end: `chant ci tick` and `chant ci run --at`, as a process,
+ * against a bare git remote and a local server that answers GitHub's check
+ * run, re-run and workflow dispatch calls the way Actions would for a
+ * `suites` phase declared `build: "newest"`.
+ *
+ * Seven commits go to main at once: a green base, five commits the suites
+ * never ran on (GitHub dropped their waiting runs), and the newest, whose
+ * suites run fails. The fourth of the five broke the suites. The tick runs
+ * each failed job again once, then bisects: it halves the five, steps back
+ * from the red, and names the culprit, leaving the two commits below the
+ * green middle one unbuilt.
+ */
+
+import { spawn } from "node:child_process";
+import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
+import type { AddressInfo } from "node:net";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { cleanScratch, declaration, git, REPO, scratchDir, writeFiles } from "./__fixtures__/contract-repo";
+
+afterAll(cleanScratch);
+
+const GREEN = {
+  branch: "main",
+  phases: { check: { runs: ["check"], build: "every" }, suites: { runs: ["suites"], build: "newest" } },
+  require: ["check", "suites"],
+};
+const SUITES_WORKFLOW = "name: release-suites\non:\n  push:\n    branches: [main]\n  workflow_dispatch: {}\njobs:\n  suites:\n    runs-on: ubuntu-latest\n    steps:\n      - run: true\n";
+
+interface Run {
+  id: number;
+  name: string;
+  status: string;
+  conclusion: string | null;
+}
+
+/** The fake GitHub: runs per commit, and what dispatching the suites at a commit concludes. */
+class FakeGithub {
+  runs = new Map<string, Run[]>();
+  dispatched: string[] = [];
+  reruns: number[] = [];
+  private id = 1;
+  server!: Server;
+  constructor(
+    readonly origin: string,
+    readonly suitesPass: (sha: string) => boolean,
+  ) {}
+
+  add(sha: string, name: string, conclusion: string): void {
+    this.runs.set(sha, [...(this.runs.get(sha) ?? []), { id: this.id++, name, status: "completed", conclusion }]);
+  }
+
+  async listen(): Promise<string> {
+    this.server = createServer((req, res) => void this.handle(req, res));
+    await new Promise<void>((done) => this.server.listen(0, "127.0.0.1", done));
+    return `http://127.0.0.1:${(this.server.address() as AddressInfo).port}`;
+  }
+
+  private async handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
+    const body = await new Promise<string>((done) => {
+      let text = "";
+      req.on("data", (c) => (text += c));
+      req.on("end", () => done(text));
+    });
+    const url = new URL(req.url ?? "/", "http://x");
+    const send = (status: number, json?: unknown) => {
+      res.writeHead(status, { "Content-Type": "application/json" });
+      res.end(json === undefined ? "" : JSON.stringify(json));
+    };
+    let m = /^\/repos\/acme\/app\/commits\/([0-9a-f]+)\/check-runs$/.exec(url.pathname);
+    if (req.method === "GET" && m) {
+      const runs = this.runs.get(m[1]) ?? [];
+      return send(200, { total_count: runs.length, check_runs: runs.map((r) => ({ ...r, completed_at: null, html_url: null })) });
+    }
+    m = /^\/repos\/acme\/app\/actions\/jobs\/(\d+)\/rerun$/.exec(url.pathname);
+    if (req.method === "POST" && m) {
+      const id = Number(m[1]);
+      this.reruns.push(id);
+      // The job runs again and fails the same way.
+      for (const [sha, runs] of this.runs) {
+        const r = runs.find((x) => x.id === id);
+        if (r) this.add(sha, r.name, r.conclusion ?? "failure");
+      }
+      return send(201, {});
+    }
+    m = /^\/repos\/acme\/app\/actions\/workflows\/([^/]+)\/dispatches$/.exec(url.pathname);
+    if (req.method === "POST" && m) {
+      const ref = (JSON.parse(body) as { ref: string }).ref;
+      const sha = git(this.origin, "rev-parse", `refs/tags/${ref}^{commit}`);
+      this.dispatched.push(sha);
+      this.add(sha, "suites", this.suitesPass(sha) ? "success" : "failure");
+      return send(204);
+    }
+    send(404, { message: "not found" });
+  }
+}
+
+function chant(cwd: string, api: string, ...args: string[]): Promise<{ code: number; out: string; err: string }> {
+  return new Promise((done) => {
+    const child = spawn(process.execPath, ["--import", pathToFileURL(join(REPO, "node_modules/tsx/dist/loader.mjs")).href, join(REPO, "packages/core/src/cli/main.ts"), ...args], {
+      cwd,
+      env: { ...process.env, GITHUB_API_URL: api, GITHUB_REPOSITORY: "acme/app", GITHUB_TOKEN: "t", GIT_TERMINAL_PROMPT: "0", NO_COLOR: "1" },
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    let out = "";
+    let err = "";
+    child.stdout.on("data", (c) => (out += c));
+    child.stderr.on("data", (c) => (err += c));
+    child.on("close", (code) => done({ code: code ?? 1, out, err }));
+  });
+}
+
+describe("a newest phase, bisected end to end (#4169)", () => {
+  let root: string;
+  let origin: string;
+  let forge: FakeGithub;
+  let api: string;
+  const shas: Record<string, string> = {};
+
+  beforeAll(async () => {
+    origin = scratchDir("chant-builds-e2e-origin-");
+    git(origin, "init", "-q", "--bare");
+    root = scratchDir("chant-builds-e2e-work-");
+    git(root, "init", "-q");
+    git(root, "symbolic-ref", "HEAD", "refs/heads/main");
+    git(root, "config", "user.name", "t");
+    git(root, "config", "user.email", "t@example.com");
+    git(root, "remote", "add", "origin", origin);
+    writeFiles(root, {
+      "chant.workspace.json": declaration([{ name: "app", dir: "app", kind: "other", because: "the app" }], { ci: { green: GREEN } }),
+      ".github/workflows/release-suites.yml": SUITES_WORKFLOW,
+    });
+    for (const name of ["base", "u1", "u2", "u3", "u4", "u5", "top"]) {
+      writeFiles(root, { [`app/${name}.txt`]: `${name}\n` });
+      git(root, "add", "-A");
+      git(root, "commit", "-q", "-m", name);
+      shas[name] = git(root, "rev-parse", "HEAD");
+    }
+    git(root, "push", "-q", "origin", "main");
+    const order = ["base", "u1", "u2", "u3", "u4", "u5", "top"].map((n) => shas[n]);
+    // u4 broke the suites: they pass below it.
+    forge = new FakeGithub(origin, (sha) => order.indexOf(sha) < order.indexOf(shas.u4));
+    for (const sha of order) forge.add(sha, "check", "success");
+    forge.add(shas.base, "suites", "success");
+    forge.add(shas.top, "suites", "failure");
+    api = await forge.listen();
+  });
+
+  afterAll(() => forge?.server.close());
+
+  const tags = () => git(origin, "for-each-ref", "--format=%(refname:lstrip=2)", "refs/tags/").split("\n").filter(Boolean).sort();
+  const record = (name: string) => JSON.parse(git(origin, "show", `refs/heads/chant/lifecycle:_ci/${shas[name]}.json`));
+  const tick = async () => {
+    const r = await chant(root, api, "ci", "tick");
+    expect(r.code, r.err).toBe(0);
+    return r.out;
+  };
+
+  test("retries once, bisects to the culprit, and leaves the commits below the green one unbuilt", async () => {
+    // The failed suites run again once, and fail again.
+    expect(await tick()).toContain(`ran suites at ${shas.top.slice(0, 12)} again`);
+    expect(forge.reruns).toHaveLength(1);
+    expect(record("top")).toMatchObject({ state: "pending", phases: { suites: { reason: "retrying" } } });
+
+    // Red now: five unbuilt below it, so the middle one, u3.
+    await tick();
+    expect(forge.dispatched).toEqual([shas.u3]);
+    // u3 passes and goes green; then the step back from the red, u5, then u4,
+    // each red run of them run again once before the next step.
+    await tick();
+    expect(tags()).toContain(`ci/green/${shas.u3}`);
+    expect(forge.dispatched).toEqual([shas.u3, shas.u5]);
+    expect(await tick()).toContain(`ran suites at ${shas.u5.slice(0, 12)} again`);
+    await tick();
+    expect(forge.dispatched).toEqual([shas.u3, shas.u5, shas.u4]);
+    expect(await tick()).toContain(`ran suites at ${shas.u4.slice(0, 12)} again`);
+    await tick();
+    expect(forge.dispatched).toHaveLength(3);
+    expect(forge.reruns).toHaveLength(3);
+
+    for (const name of ["u4", "u5", "top"]) expect(record(name), name).toMatchObject({ state: "red", phases: { suites: { culprit: shas.u4 } } });
+    for (const name of ["u1", "u2"]) expect(record(name), name).toMatchObject({ state: "unbuilt", phases: { suites: { verdict: "unbuilt", reason: "superseded" } } });
+    expect(record("u3")).toMatchObject({ state: "green" });
+    expect(tags().filter((t) => t.startsWith("ci/build/"))).toEqual([]);
+    expect(tags()).toEqual([`ci/green/${shas.base}`, `ci/green/${shas.u3}`].sort());
+
+    const last = await chant(root, api, "ci", "last-green", "--json");
+    const doc = JSON.parse(last.out);
+    expect(doc.commit.sha).toBe(shas.u3);
+    expect(doc.newer.map((n: { sha: string; record: { state: string; culprit?: string } }) => [n.sha, n.record.state, n.record.culprit])).toEqual([
+      [shas.top, "red", shas.u4],
+      [shas.u5, "red", shas.u4],
+      [shas.u4, "red", shas.u4],
+    ]);
+  }, 180_000);
+
+  test("ci run --at asks for a chosen commit and starts the suites there", async () => {
+    const r = await chant(root, api, "ci", "run", "--at", shas.u1);
+    expect(r.code, r.err).toBe(0);
+    expect(r.out).toContain(`Asked for check, suites at ${shas.u1.slice(0, 12)}: ci/build/${shas.u1}. Started release-suites.yml.`);
+    expect(tags()).toContain(`ci/build/${shas.u1}`);
+    await tick();
+    expect(tags()).toContain(`ci/green/${shas.u1}`);
+    expect(tags()).not.toContain(`ci/build/${shas.u1}`);
+  }, 60_000);
+});