@intentius/chant 0.124.31 → 0.124.33

package/src/workspace/ci-builds.e2e.test.ts

@@ -0,0 +1,208 @@
+/**
+ * #4169 end to end: `chant ci tick` and `chant ci run --at`, as a process,
+ * against a bare git remote and a local server that answers GitHub's check
+ * run, re-run and workflow dispatch calls the way Actions would for a
+ * `suites` phase declared `build: "newest"`.
+ *
+ * Seven commits go to main at once: a green base, five commits the suites
+ * never ran on (GitHub dropped their waiting runs), and the newest, whose
+ * suites run fails. The fourth of the five broke the suites. The tick runs
+ * each failed job again once, then bisects: it halves the five, steps back
+ * from the red, and names the culprit, leaving the two commits below the
+ * green middle one unbuilt.
+ */
+
+import { spawn } from "node:child_process";
+import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
+import type { AddressInfo } from "node:net";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { cleanScratch, declaration, git, REPO, scratchDir, writeFiles } from "./__fixtures__/contract-repo";
+
+afterAll(cleanScratch);
+
+const GREEN = {
+  branch: "main",
+  phases: { check: { runs: ["check"], build: "every" }, suites: { runs: ["suites"], build: "newest" } },
+  require: ["check", "suites"],
+};
+const SUITES_WORKFLOW = "name: release-suites\non:\n  push:\n    branches: [main]\n  workflow_dispatch: {}\njobs:\n  suites:\n    runs-on: ubuntu-latest\n    steps:\n      - run: true\n";
+
+interface Run {
+  id: number;
+  name: string;
+  status: string;
+  conclusion: string | null;
+}
+
+/** The fake GitHub: runs per commit, and what dispatching the suites at a commit concludes. */
+class FakeGithub {
+  runs = new Map<string, Run[]>();
+  dispatched: string[] = [];
+  reruns: number[] = [];
+  private id = 1;
+  server!: Server;
+  constructor(
+    readonly origin: string,
+    readonly suitesPass: (sha: string) => boolean,
+  ) {}
+
+  add(sha: string, name: string, conclusion: string): void {
+    this.runs.set(sha, [...(this.runs.get(sha) ?? []), { id: this.id++, name, status: "completed", conclusion }]);
+  }
+
+  async listen(): Promise<string> {
+    this.server = createServer((req, res) => void this.handle(req, res));
+    await new Promise<void>((done) => this.server.listen(0, "127.0.0.1", done));
+    return `http://127.0.0.1:${(this.server.address() as AddressInfo).port}`;
+  }
+
+  private async handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
+    const body = await new Promise<string>((done) => {
+      let text = "";
+      req.on("data", (c) => (text += c));
+      req.on("end", () => done(text));
+    });
+    const url = new URL(req.url ?? "/", "http://x");
+    const send = (status: number, json?: unknown) => {
+      res.writeHead(status, { "Content-Type": "application/json" });
+      res.end(json === undefined ? "" : JSON.stringify(json));
+    };
+    let m = /^\/repos\/acme\/app\/commits\/([0-9a-f]+)\/check-runs$/.exec(url.pathname);
+    if (req.method === "GET" && m) {
+      const runs = this.runs.get(m[1]) ?? [];
+      return send(200, { total_count: runs.length, check_runs: runs.map((r) => ({ ...r, completed_at: null, html_url: null })) });
+    }
+    m = /^\/repos\/acme\/app\/actions\/jobs\/(\d+)\/rerun$/.exec(url.pathname);
+    if (req.method === "POST" && m) {
+      const id = Number(m[1]);
+      this.reruns.push(id);
+      // The job runs again and fails the same way.
+      for (const [sha, runs] of this.runs) {
+        const r = runs.find((x) => x.id === id);
+        if (r) this.add(sha, r.name, r.conclusion ?? "failure");
+      }
+      return send(201, {});
+    }
+    m = /^\/repos\/acme\/app\/actions\/workflows\/([^/]+)\/dispatches$/.exec(url.pathname);
+    if (req.method === "POST" && m) {
+      const ref = (JSON.parse(body) as { ref: string }).ref;
+      const sha = git(this.origin, "rev-parse", `refs/tags/${ref}^{commit}`);
+      this.dispatched.push(sha);
+      this.add(sha, "suites", this.suitesPass(sha) ? "success" : "failure");
+      return send(204);
+    }
+    send(404, { message: "not found" });
+  }
+}
+
+function chant(cwd: string, api: string, ...args: string[]): Promise<{ code: number; out: string; err: string }> {
+  return new Promise((done) => {
+    const child = spawn(process.execPath, ["--import", pathToFileURL(join(REPO, "node_modules/tsx/dist/loader.mjs")).href, join(REPO, "packages/core/src/cli/main.ts"), ...args], {
+      cwd,
+      env: { ...process.env, GITHUB_API_URL: api, GITHUB_REPOSITORY: "acme/app", GITHUB_TOKEN: "t", GIT_TERMINAL_PROMPT: "0", NO_COLOR: "1" },
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    let out = "";
+    let err = "";
+    child.stdout.on("data", (c) => (out += c));
+    child.stderr.on("data", (c) => (err += c));
+    child.on("close", (code) => done({ code: code ?? 1, out, err }));
+  });
+}
+
+describe("a newest phase, bisected end to end (#4169)", () => {
+  let root: string;
+  let origin: string;
+  let forge: FakeGithub;
+  let api: string;
+  const shas: Record<string, string> = {};
+
+  beforeAll(async () => {
+    origin = scratchDir("chant-builds-e2e-origin-");
+    git(origin, "init", "-q", "--bare");
+    root = scratchDir("chant-builds-e2e-work-");
+    git(root, "init", "-q");
+    git(root, "symbolic-ref", "HEAD", "refs/heads/main");
+    git(root, "config", "user.name", "t");
+    git(root, "config", "user.email", "t@example.com");
+    git(root, "remote", "add", "origin", origin);
+    writeFiles(root, {
+      "chant.workspace.json": declaration([{ name: "app", dir: "app", kind: "other", because: "the app" }], { ci: { green: GREEN } }),
+      ".github/workflows/release-suites.yml": SUITES_WORKFLOW,
+    });
+    for (const name of ["base", "u1", "u2", "u3", "u4", "u5", "top"]) {
+      writeFiles(root, { [`app/${name}.txt`]: `${name}\n` });
+      git(root, "add", "-A");
+      git(root, "commit", "-q", "-m", name);
+      shas[name] = git(root, "rev-parse", "HEAD");
+    }
+    git(root, "push", "-q", "origin", "main");
+    const order = ["base", "u1", "u2", "u3", "u4", "u5", "top"].map((n) => shas[n]);
+    // u4 broke the suites: they pass below it.
+    forge = new FakeGithub(origin, (sha) => order.indexOf(sha) < order.indexOf(shas.u4));
+    for (const sha of order) forge.add(sha, "check", "success");
+    forge.add(shas.base, "suites", "success");
+    forge.add(shas.top, "suites", "failure");
+    api = await forge.listen();
+  });
+
+  afterAll(() => forge?.server.close());
+
+  const tags = () => git(origin, "for-each-ref", "--format=%(refname:lstrip=2)", "refs/tags/").split("\n").filter(Boolean).sort();
+  const record = (name: string) => JSON.parse(git(origin, "show", `refs/heads/chant/lifecycle:_ci/${shas[name]}.json`));
+  const tick = async () => {
+    const r = await chant(root, api, "ci", "tick");
+    expect(r.code, r.err).toBe(0);
+    return r.out;
+  };
+
+  test("retries once, bisects to the culprit, and leaves the commits below the green one unbuilt", async () => {
+    // The failed suites run again once, and fail again.
+    expect(await tick()).toContain(`ran suites at ${shas.top.slice(0, 12)} again`);
+    expect(forge.reruns).toHaveLength(1);
+    expect(record("top")).toMatchObject({ state: "pending", phases: { suites: { reason: "retrying" } } });
+
+    // Red now: five unbuilt below it, so the middle one, u3.
+    await tick();
+    expect(forge.dispatched).toEqual([shas.u3]);
+    // u3 passes and goes green; then the step back from the red, u5, then u4,
+    // each red run of them run again once before the next step.
+    await tick();
+    expect(tags()).toContain(`ci/green/${shas.u3}`);
+    expect(forge.dispatched).toEqual([shas.u3, shas.u5]);
+    expect(await tick()).toContain(`ran suites at ${shas.u5.slice(0, 12)} again`);
+    await tick();
+    expect(forge.dispatched).toEqual([shas.u3, shas.u5, shas.u4]);
+    expect(await tick()).toContain(`ran suites at ${shas.u4.slice(0, 12)} again`);
+    await tick();
+    expect(forge.dispatched).toHaveLength(3);
+    expect(forge.reruns).toHaveLength(3);
+
+    for (const name of ["u4", "u5", "top"]) expect(record(name), name).toMatchObject({ state: "red", phases: { suites: { culprit: shas.u4 } } });
+    for (const name of ["u1", "u2"]) expect(record(name), name).toMatchObject({ state: "unbuilt", phases: { suites: { verdict: "unbuilt", reason: "superseded" } } });
+    expect(record("u3")).toMatchObject({ state: "green" });
+    expect(tags().filter((t) => t.startsWith("ci/build/"))).toEqual([]);
+    expect(tags()).toEqual([`ci/green/${shas.base}`, `ci/green/${shas.u3}`].sort());
+
+    const last = await chant(root, api, "ci", "last-green", "--json");
+    const doc = JSON.parse(last.out);
+    expect(doc.commit.sha).toBe(shas.u3);
+    expect(doc.newer.map((n: { sha: string; record: { state: string; culprit?: string } }) => [n.sha, n.record.state, n.record.culprit])).toEqual([
+      [shas.top, "red", shas.u4],
+      [shas.u5, "red", shas.u4],
+      [shas.u4, "red", shas.u4],
+    ]);
+  }, 180_000);
+
+  test("ci run --at asks for a chosen commit and starts the suites there", async () => {
+    const r = await chant(root, api, "ci", "run", "--at", shas.u1);
+    expect(r.code, r.err).toBe(0);
+    expect(r.out).toContain(`Asked for check, suites at ${shas.u1.slice(0, 12)}: ci/build/${shas.u1}. Started release-suites.yml.`);
+    expect(tags()).toContain(`ci/build/${shas.u1}`);
+    await tick();
+    expect(tags()).toContain(`ci/green/${shas.u1}`);
+    expect(tags()).not.toContain(`ci/build/${shas.u1}`);
+  }, 60_000);
+});

package/src/workspace/ci-builds.test.ts

@@ -0,0 +1,174 @@
+/**
+ * #4169 — which commits get built: supersession, retries, requests and
+ * bisection over judged commits, and the records each commit gets.
+ */
+
+import { describe, expect, test } from "vitest";
+import { agedOut, BUILD_REQUEST_TTL_MS, planBuilds, recordOf, RETRY_TTL_MS, sameRecord, type BuildRequest, type CiRecord } from "./ci-builds";
+import { evaluateCommit, type TickCommit } from "./ci-green";
+import type { CheckRun } from "./ci-green-forge";
+import type { CiGreen } from "./declaration";
+
+const NOW = new Date("2026-10-20T12:00:00Z");
+const ago = (ms: number) => new Date(NOW.getTime() - ms).toISOString().replace(/\.\d{3}Z$/, "Z");
+
+const GREEN: CiGreen = {
+  branch: "main",
+  window: "24h",
+  phases: [
+    { name: "check", runs: ["check"], skipped: "fail", build: "every", pointer: "/ci/green/phases/check" },
+    { name: "suites", runs: ["suites"], skipped: "fail", build: "newest", pointer: "/ci/green/phases/suites" },
+  ],
+  require: ["check", "suites"],
+  signedSince: null,
+  source: "checks",
+  pointer: "/ci/green",
+};
+
+let nextId = 1;
+const run = (name: string, conclusion: string | null, status = "completed"): CheckRun => ({ id: nextId++, name, status, conclusion: status === "completed" ? conclusion : null, completedAt: null, url: null });
+const sha = (n: number) => String(n).padStart(40, "0");
+
+/** Commits newest first, each with its runs, judged as the tick judges them. */
+function judged(runsPerCommit: CheckRun[][], before: TickCommit["before"][] = []): { commits: TickCommit[]; runs: Map<string, CheckRun[]> } {
+  const runs = new Map<string, CheckRun[]>();
+  const commits = runsPerCommit.map((r, i) => {
+    const s = sha(runsPerCommit.length - i);
+    runs.set(s, r);
+    return { sha: s, committed: NOW.toISOString(), before: before[i] ?? "none", verdict: evaluateCommit(GREEN, r), made: null } as TickCommit;
+  });
+  return { commits, runs };
+}
+
+const plan = (commits: TickCommit[], runs: Map<string, CheckRun[]>, extra: Partial<Parameters<typeof planBuilds>[0]> = {}) =>
+  planBuilds({ green: GREEN, commits, runs, requests: new Map(), records: new Map(), now: NOW, retryVia: null, ...extra });
+
+const suites = (c: TickCommit) => c.verdict!.phases.find((p) => p.phase === "suites")!;
+
+describe("supersession (#4169)", () => {
+  test("a newest phase with no run below a commit that has one is unbuilt, covered by it; the newest commit stays pending", () => {
+    const { commits, runs } = judged([[run("check", "success")], [run("check", "success"), run("suites", "success")], [run("check", "success")]]);
+    plan(commits, runs);
+    expect(commits.map((c) => c.verdict!.verdict)).toEqual(["pending", "pass", "unbuilt"]);
+    expect(suites(commits[2])).toMatchObject({ verdict: "unbuilt", reason: "superseded", coveredBy: commits[1].sha });
+    expect(suites(commits[0])).toMatchObject({ verdict: "pending" });
+  });
+
+  test("an every phase is never superseded, and a cancelled run is unbuilt under either", () => {
+    const { commits, runs } = judged([[run("check", "success"), run("suites", "success")], [run("suites", "success")], [run("check", "cancelled"), run("suites", "success")]]);
+    plan(commits, runs);
+    expect(commits.map((c) => c.verdict!.verdict)).toEqual(["pass", "pending", "unbuilt"]);
+    expect(commits[2].verdict!.phases[0]).toMatchObject({ phase: "check", verdict: "unbuilt", reason: "cancelled" });
+  });
+});
+
+describe("bisect (#4169)", () => {
+  const green = () => [run("check", "success"), run("suites", "success")];
+  const red = () => [run("check", "success"), run("suites", "failure")];
+  const none = () => [run("check", "success")];
+
+  test("a red newest commit asks for the one just below it when four or fewer are unbuilt", () => {
+    const { commits, runs } = judged([red(), none(), none(), green()]);
+    const p = plan(commits, runs);
+    expect(p.requests).toEqual([expect.objectContaining({ commit: commits[1].sha, phases: ["suites"], reason: "bisect", tag: `ci/build/${commits[1].sha}` })]);
+    expect(suites(commits[1])).toMatchObject({ verdict: "pending", reason: "requested" });
+  });
+
+  test("halves the range above four, and asks for nothing while a request in it waits", () => {
+    const { commits, runs } = judged([red(), none(), none(), none(), none(), none(), none(), green()]);
+    const p = plan(commits, runs);
+    expect(p.requests.map((r) => r.commit)).toEqual([commits[4].sha]);
+    const waiting: BuildRequest = { ...p.requests[0], at: ago(60_000) };
+    const again = judged([red(), none(), none(), none(), none(), none(), none(), green()]);
+    expect(plan(again.commits, again.runs, { requests: new Map([[waiting.commit, waiting]]) }).requests).toEqual([]);
+  });
+
+  test("names the culprit on it and every commit after it once the red sits on a green", () => {
+    const { commits, runs } = judged([red(), none(), red(), green()]);
+    // The commit below the newest red is unbuilt between two reds: the culprit is at or below the older red.
+    plan(commits, runs);
+    expect(commits.map((c) => suites(c).culprit)).toEqual([commits[2].sha, commits[2].sha, commits[2].sha, undefined]);
+  });
+
+  test("names no culprit without a green below, and an every phase gets one too", () => {
+    const noGreen = judged([red(), red()]);
+    plan(noGreen.commits, noGreen.runs);
+    expect(noGreen.commits.map((c) => suites(c).culprit)).toEqual([undefined, undefined]);
+    const every = judged([[run("check", "failure")], [run("check", "failure")], [run("check", "success")]]);
+    plan(every.commits, every.runs);
+    expect(every.commits.map((c) => c.verdict!.phases[0].culprit)).toEqual([every.commits[1].sha, every.commits[1].sha, undefined]);
+  });
+});
+
+describe("requests (#4169)", () => {
+  const req = (commit: string, at: string): BuildRequest => ({ schema: 1, tag: `ci/build/${commit}`, commit, branch: "main", phases: ["suites"], reason: "requested", at });
+
+  test("a request holds its phase pending; once decided, or too old, or out of the window, it is done", () => {
+    const { commits, runs } = judged([[run("check", "success"), run("suites", "success")], [run("check", "success")]]);
+    const waiting = req(commits[1].sha, ago(60_000));
+    const p = plan(commits, runs, { requests: new Map([[waiting.commit, waiting]]) });
+    expect(suites(commits[1])).toMatchObject({ verdict: "pending", reason: "requested" });
+    expect(p.resolved).toEqual([]);
+
+    const old = judged([[run("check", "success"), run("suites", "success")], [run("check", "success")]]);
+    const stale = req(old.commits[1].sha, ago(BUILD_REQUEST_TTL_MS + 1));
+    expect(plan(old.commits, old.runs, { requests: new Map([[stale.commit, stale]]) }).resolved).toEqual([stale.tag]);
+    expect(suites(old.commits[1])).toMatchObject({ verdict: "unbuilt", reason: "request-unserved" });
+
+    const done = judged([[run("check", "success"), run("suites", "success")]]);
+    const served = req(done.commits[0].sha, ago(60_000));
+    const gone = req(sha(99), ago(60_000));
+    expect(plan(done.commits, done.runs, { requests: new Map([[served.commit, served], [gone.commit, gone]]) }).resolved.sort()).toEqual([served.tag, gone.tag].sort());
+  });
+});
+
+describe("retry (#4169)", () => {
+  test("a failed run with one attempt is run again and holds the commit pending; a second attempt decides", () => {
+    const failed = run("check", "failure");
+    const { commits, runs } = judged([[failed, run("suites", "success")]]);
+    const p = plan(commits, runs, { retryVia: "forge" });
+    expect(p.retries).toEqual([{ sha: commits[0].sha, run: failed }]);
+    expect(commits[0].verdict!.verdict).toBe("pending");
+    expect(commits[0].verdict!.phases[0]).toMatchObject({ verdict: "pending", reason: "retrying" });
+
+    const twice = judged([[failed, run("check", "failure"), run("suites", "success")]]);
+    expect(plan(twice.commits, twice.runs, { retryVia: "forge" }).retries).toEqual([]);
+    expect(twice.commits[0].verdict!.verdict).toBe("fail");
+  });
+
+  test("a retry asked for waits an hour for its attempt, then the failure counts; chant runs ask by request; a green commit is never retried", () => {
+    const failed = run("check", "failure");
+    const asked = (at: string): CiRecord => ({ schema: 1, commit: sha(1), branch: "main", state: "pending", at, phases: {}, retried: { check: { run: failed.id, at } } });
+    const waiting = judged([[failed, run("suites", "success")]]);
+    expect(plan(waiting.commits, waiting.runs, { retryVia: "forge", records: new Map([[sha(1), asked(ago(60_000))]]) }).retries).toEqual([]);
+    expect(waiting.commits[0].verdict!.verdict).toBe("pending");
+    const late = judged([[failed, run("suites", "success")]]);
+    plan(late.commits, late.runs, { retryVia: "forge", records: new Map([[sha(1), asked(ago(RETRY_TTL_MS + 1))]]) });
+    expect(late.commits[0].verdict!.verdict).toBe("fail");
+
+    const viaRequest = judged([[failed, run("suites", "success")]]);
+    expect(plan(viaRequest.commits, viaRequest.runs, { retryVia: "request" }).requests).toEqual([expect.objectContaining({ commit: sha(1), phases: ["check"], reason: "retry" })]);
+
+    const tagged = judged([[failed, run("suites", "success")]], ["green"]);
+    expect(plan(tagged.commits, tagged.runs, { retryVia: "forge" }).retries).toEqual([]);
+  });
+});
+
+describe("records (#4169)", () => {
+  test("say green, red, unbuilt or pending with each phase's verdict, reason and covering commit, and skip the time when compared", () => {
+    const { commits, runs } = judged([[run("check", "success"), run("suites", "success")], [run("check", "success")]]);
+    plan(commits, runs);
+    commits[0].made = { tag: `ci/green/${commits[0].sha}`, message: "" };
+    const top = recordOf({ green: GREEN, commit: commits[0], retries: [], requests: new Map(), now: NOW });
+    expect(top).toMatchObject({ state: "green", commit: commits[0].sha, phases: { check: { verdict: "pass", build: "every" }, suites: { verdict: "pass", build: "newest" } } });
+    const below = recordOf({ green: GREEN, commit: commits[1], retries: [], requests: new Map(), now: NOW });
+    expect(below).toMatchObject({ state: "unbuilt", phases: { suites: { verdict: "unbuilt", reason: "superseded", coveredBy: commits[0].sha } } });
+    expect(sameRecord({ ...below, at: "2020-01-01T00:00:00Z" }, below)).toBe(true);
+    expect(sameRecord({ ...below, state: "pending" }, below)).toBe(false);
+  });
+
+  test("a pending record that left the window is rewritten unbuilt, aged out", () => {
+    const pending: CiRecord = { schema: 1, commit: sha(1), branch: "main", state: "pending", at: ago(1), phases: { suites: { verdict: "pending", build: "newest", runs: [] } } };
+    expect(agedOut(pending, NOW)).toMatchObject({ state: "unbuilt", phases: { suites: { verdict: "unbuilt", reason: "aged-out" } } });
+  });
+});