@intentius/chant 0.124.29 → 0.124.31

package/src/workspace/ci-secrets.ts

@@ -0,0 +1,213 @@
+/**
+ * The chant runner's secrets file (#4096, epic #4101): the values a job gets
+ * from the host, such as cloud credentials and the endpoint, which a forge
+ * job would get from the forge's secrets.
+ *
+ * `chant ci run` and `chant ci serve` read it from `--secrets <file>`, or
+ * the file `CHANT_CI_SECRETS` names. It is a dotenv file of `NAME=value`
+ * lines, or a JSON object:
+ *
+ * - a map of name to value, `{ "AWS_SECRET_ACCESS_KEY": "..." }`;
+ * - or `{ "secrets": { ... }, "jobs": { "plan": [...], "apply": [...] } }`,
+ *   where `jobs` lists, for a job, the names it gets. A job `jobs` doesn't
+ *   name gets every secret.
+ *
+ * The allowlist lives in this file on the host, not in the repository, so a
+ * pull request can't widen what its plan gets.
+ *
+ * The file must belong to the user reading it and give group and others no
+ * permission at all, as ssh asks of a private key (`chmod 600`). A file
+ * that does is refused before any job runs.
+ *
+ * Every value in the file, and every value of the job's environment whose
+ * name looks secret ({@link SECRET_NAME}), is redacted from what the run
+ * writes about itself (../op/redact.ts).
+ */
+
+import { readFileSync, statSync } from "node:fs";
+import { redactorOf, type Redact } from "../op/redact";
+import { RUNNER_KEY_ENV } from "./ci-green";
+import type { CiJobName } from "./ci-run";
+
+/** The variable that names the secrets file when `--secrets` is not given. */
+export const CI_SECRETS_ENV = "CHANT_CI_SECRETS";
+
+/** A variable whose name matches this is taken as secret, and its value redacted. */
+export const SECRET_NAME = /TOKEN|SECRET|KEY|PASSWORD/i;
+
+const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
+const JOBS: readonly CiJobName[] = ["plan", "apply"];
+
+/** The secrets file, read. */
+export interface CiSecrets {
+  /** Where it was read from. */
+  file: string;
+  /** Every secret, by name. */
+  values: Record<string, string>;
+  /** For a job the file names, the secrets it gets. A job missing here gets every one. */
+  jobs: Partial<Record<CiJobName, string[]>>;
+}
+
+export class CiSecretsError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "CiSecretsError";
+  }
+}
+
+/**
+ * Refuse a secrets file anyone but its owner can read or write, or that
+ * belongs to another user, as ssh refuses a loose key. Not checked on
+ * Windows, whose files carry no such mode.
+ */
+export function checkCiSecretsMode(file: string): void {
+  let st: ReturnType<typeof statSync>;
+  try {
+    st = statSync(file);
+  } catch (err) {
+    throw new CiSecretsError(`the secrets file ${file} can't be read: ${(err as NodeJS.ErrnoException).code ?? String(err)}`);
+  }
+  if (!st.isFile()) throw new CiSecretsError(`the secrets file ${file} is not a file`);
+  if (process.platform === "win32") return;
+  const loose = Number(st.mode) & 0o077;
+  if (loose !== 0) {
+    const mode = (Number(st.mode) & 0o777).toString(8).padStart(4, "0");
+    throw new CiSecretsError(
+      `the secrets file ${file} has mode ${mode}, so its group or other users can ${loose & 0o044 ? "read" : "change"} it; a secrets file must be readable only by its owner, as ssh asks of a key: chmod 600 ${file}`,
+    );
+  }
+  const uid = typeof process.getuid === "function" ? process.getuid() : undefined;
+  if (uid !== undefined && Number(st.uid) !== uid) {
+    throw new CiSecretsError(`the secrets file ${file} belongs to uid ${st.uid}, not to this user (uid ${uid}); a secrets file must be its reader's own`);
+  }
+}
+
+/** A dotenv text's `NAME=value` pairs. Errors name the line, never a value. */
+export function parseDotenv(text: string, file: string): Record<string, string> {
+  const out: Record<string, string> = {};
+  let i = 0;
+  let line = 1;
+  const fail = (why: string): never => {
+    throw new CiSecretsError(`${file}:${line}: ${why}`);
+  };
+  while (i < text.length) {
+    // One line: blank, a comment, or NAME=value.
+    const eol = text.indexOf("\n", i);
+    const rest = text.slice(i, eol === -1 ? text.length : eol);
+    const trimmed = rest.trim();
+    if (trimmed === "" || trimmed.startsWith("#")) {
+      i = eol === -1 ? text.length : eol + 1;
+      line++;
+      continue;
+    }
+    const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*/.exec(rest);
+    if (!m) fail("expected NAME=value");
+    const name = m![1]!;
+    let j = i + m![0].length;
+    const startLine = line;
+    let value = "";
+    const q = text[j];
+    if (q === '"' || q === "'") {
+      j++;
+      for (;;) {
+        if (j >= text.length) {
+          line = startLine;
+          fail(`the value of ${name} has no closing ${q}`);
+        }
+        const c = text[j]!;
+        if (c === q) break;
+        if (c === "\n") line++;
+        if (q === '"' && c === "\\" && j + 1 < text.length) {
+          const n = text[j + 1]!;
+          value += n === "n" ? "\n" : n === "r" ? "\r" : n === "t" ? "\t" : n;
+          j += 2;
+          continue;
+        }
+        value += c;
+        j++;
+      }
+      j++;
+      const end = text.indexOf("\n", j);
+      const after = text.slice(j, end === -1 ? text.length : end).trim();
+      if (after !== "" && !after.startsWith("#")) fail(`unexpected text after the quoted value of ${name}`);
+      i = end === -1 ? text.length : end + 1;
+    } else {
+      const end = text.indexOf("\n", j);
+      value = text.slice(j, end === -1 ? text.length : end).replace(/\s+#.*$/, "").trim();
+      i = end === -1 ? text.length : end + 1;
+    }
+    if (Object.prototype.hasOwnProperty.call(out, name)) fail(`${name} is set twice`);
+    out[name] = value;
+    line++;
+  }
+  return out;
+}
+
+function checkName(name: string, file: string): void {
+  if (!ENV_NAME.test(name)) throw new CiSecretsError(`${file} names a secret ${JSON.stringify(name)}, which is not a variable name`);
+  if (name === RUNNER_KEY_ENV) throw new CiSecretsError(`${file} sets ${RUNNER_KEY_ENV}, and the runner key is never passed to a job; give it to ci run or ci serve with --key or ${RUNNER_KEY_ENV}`);
+}
+
+/** The secrets in a file's text: JSON when it starts with `{`, else dotenv. */
+export function parseCiSecrets(text: string, file: string): Omit<CiSecrets, "file"> {
+  if (!text.trimStart().startsWith("{")) {
+    const values = parseDotenv(text, file);
+    for (const name of Object.keys(values)) checkName(name, file);
+    return { values, jobs: {} };
+  }
+  let raw: unknown;
+  try {
+    raw = JSON.parse(text);
+  } catch {
+    // The parser's message quotes the text near the error, which may be a secret.
+    throw new CiSecretsError(`${file} starts with {, and is not valid JSON`);
+  }
+  const doc = raw as Record<string, unknown>;
+  const structured = doc.secrets !== null && typeof doc.secrets === "object" && !Array.isArray(doc.secrets);
+  const map = (structured ? doc.secrets : doc) as Record<string, unknown>;
+  const values: Record<string, string> = {};
+  for (const [name, value] of Object.entries(map)) {
+    checkName(name, file);
+    if (typeof value !== "string") throw new CiSecretsError(`${file}: the secret ${name} must be a string`);
+    values[name] = value;
+  }
+  const jobs: CiSecrets["jobs"] = {};
+  if (structured) {
+    for (const key of Object.keys(doc)) if (key !== "secrets" && key !== "jobs") throw new CiSecretsError(`${file} has ${JSON.stringify(key)} beside secrets and jobs`);
+    const given = doc.jobs ?? {};
+    if (given === null || typeof given !== "object" || Array.isArray(given)) throw new CiSecretsError(`${file}: jobs must map plan and apply to lists of secret names`);
+    for (const [job, names] of Object.entries(given as Record<string, unknown>)) {
+      if (!(JOBS as readonly string[]).includes(job)) throw new CiSecretsError(`${file}: jobs names ${JSON.stringify(job)}, and the jobs are plan and apply`);
+      if (!Array.isArray(names) || names.some((n) => typeof n !== "string")) throw new CiSecretsError(`${file}: jobs.${job} must be a list of secret names`);
+      for (const n of names as string[]) if (!Object.prototype.hasOwnProperty.call(values, n)) throw new CiSecretsError(`${file}: jobs.${job} names ${n}, which the file does not set`);
+      jobs[job as CiJobName] = [...(names as string[])];
+    }
+  }
+  return { values, jobs };
+}
+
+/** Read the secrets file, refusing one others can read. */
+export function readCiSecrets(file: string): CiSecrets {
+  checkCiSecretsMode(file);
+  return { file, ...parseCiSecrets(readFileSync(file, "utf-8"), file) };
+}
+
+/** The secrets `job` gets: those `jobs` lists for it, else every one. */
+export function ciSecretsFor(secrets: CiSecrets | undefined, job: CiJobName): Record<string, string> {
+  if (!secrets) return {};
+  const names = secrets.jobs[job] ?? Object.keys(secrets.values);
+  return Object.fromEntries(names.map((n) => [n, secrets.values[n]!]));
+}
+
+/**
+ * The redactor for a job: every value in the secrets file, whether the job
+ * gets it or not, and the value of each variable in the job's environment
+ * whose name looks secret, such as `GITHUB_TOKEN` or a `--pass-env
+ * AWS_SECRET_ACCESS_KEY`. Undefined when there is nothing to redact.
+ */
+export function ciRedactor(jobEnv: Record<string, string>, secrets?: CiSecrets): Redact | undefined {
+  const values = Object.values(secrets?.values ?? {});
+  for (const [name, value] of Object.entries(jobEnv)) if (SECRET_NAME.test(name)) values.push(value);
+  return redactorOf(values);
+}

package/src/workspace/ci-serve.test.ts

@@ -0,0 +1,367 @@
+/**
+ * #4095: `chant ci serve` polls the forge and runs `ci run` plan for each new
+ * pull request head and apply for each new target-branch commit, deduped in
+ * a state file, one apply per group through a git-ref lease, with each
+ * apply's resume record kept on the host, and at most `maxJobs` at once.
+ *
+ * Every test works in its own temp repository with a bare origin. The forge
+ * is a fake client, and the jobs' `chant` is a stub script outside the
+ * repository. Each test runs single `--once` passes in-process.
+ */
+
+import { execFileSync } from "node:child_process";
+import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { cleanScratch, declaration, git, repoFixture, scratchDir, writeFiles } from "./__fixtures__/contract-repo";
+import { ciServeInterval, ciServeKey, ciServePass, formatCiServeEvent, readCiServeState, type CiServeEvent, type CiServeForge, type CiServeOptions } from "./ci-serve";
+import { ciRun } from "./ci-run";
+import { acquireLease, releaseLease } from "../lifecycle/lease";
+import { prApplyGroup, prApplyRecordKey } from "../components/pr-pipeline";
+import { parseArgs } from "../cli/parse-args";
+import { forgejoForge, githubForge, type ForgeFetch, type OpenPullRequest } from "../pr-forge";
+
+afterAll(cleanScratch);
+
+function commit(root: string, file: string): string {
+  writeFiles(root, { [file]: `${file} ${Math.random()}\n` });
+  for (const args of [["add", "-A"], ["commit", "-q", "-m", file]]) {
+    execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", "-c", "commit.gpgsign=false", ...args], { cwd: root, stdio: "ignore" });
+  }
+  return git(root, "rev-parse", "HEAD");
+}
+
+/** The repository every test starts from, built once: a checkout on main with a bare origin and member `app`. */
+const workspaceTemplate = repoFixture(() => {
+  const top = scratchDir("chant-ci-serve-template-");
+  const origin = join(top, "origin");
+  const root = join(top, "root");
+  mkdirSync(origin);
+  mkdirSync(root);
+  git(origin, "init", "-q", "--bare");
+  git(root, "init", "-q");
+  git(root, "symbolic-ref", "HEAD", "refs/heads/main");
+  writeFiles(root, {
+    "chant.workspace.json": declaration([{ name: "app", dir: "app", kind: "other", because: "the app" }]),
+    ".gitignore": ".chant/pr/\n.chant/pr-resume/\n",
+    "app/a.txt": "a\n",
+  });
+  git(root, "remote", "add", "origin", origin);
+  commit(root, "app/init.txt");
+  git(root, "push", "-q", "origin", "main");
+  return top;
+}, "chant-ci-serve-");
+
+/** A copy of the template, its origin pointed at the copy of the bare one. */
+function workspace(): { root: string; app: string; main: string } {
+  const top = workspaceTemplate();
+  const root = join(top, "root");
+  git(root, "remote", "set-url", "origin", join(top, "origin"));
+  return { root, app: join(root, "app"), main: git(root, "rev-parse", "HEAD") };
+}
+
+/** A pull request's head commit, on a branch off main, pushed to the forge's refs/pull/<n>/head. */
+function pullHead(root: string, n: number): string {
+  git(root, "checkout", "-q", "-b", `pr-${n}`);
+  const sha = commit(root, `app/pr-${n}.txt`);
+  git(root, "push", "-q", "origin", `${sha}:refs/pull/${n}/head`);
+  git(root, "checkout", "-q", "main");
+  return sha;
+}
+
+/**
+ * A stand-in for `chant` that keeps its arguments under `.chant/pr` and exits
+ * `$FAKE_EXIT`. As `pr-apply` it copies the resume record it was given to
+ * `.chant/pr/resume-in.json` and writes a new one naming `$FAKE_ATTEMPT`.
+ */
+function fakeChant(): string {
+  const file = join(scratchDir("chant-ci-serve-bin-"), "chant");
+  writeFileSync(
+    file,
+    [
+      "#!/bin/sh",
+      'stage="$2"',
+      "mkdir -p .chant/pr",
+      'printf "%s\\n" "$@" > ".chant/pr/$stage.args"',
+      'if [ "$stage" = pr-apply ]; then',
+      "  if [ -f .chant/pr-resume/pr-apply.json ]; then cp .chant/pr-resume/pr-apply.json .chant/pr/resume-in.json; fi",
+      "  mkdir -p .chant/pr-resume",
+      '  echo "{\\"attempt\\": \\"${FAKE_ATTEMPT:-1}\\"}" > .chant/pr-resume/pr-apply.json',
+      "fi",
+      'if [ -n "$FAKE_SLEEP" ]; then sleep "$FAKE_SLEEP"; fi',
+      'exit "${FAKE_EXIT:-0}"',
+      "",
+    ].join("\n"),
+  );
+  chmodSync(file, 0o755);
+  return file;
+}
+
+/** A forge whose open pull requests and branch head the test sets. */
+function fakeForge(head: string): CiServeForge & { head: string; prs: OpenPullRequest[]; polls: number; merges: Map<string, number | "throw">; asked: string[] } {
+  const f = {
+    head,
+    merges: new Map<string, number | "throw">(),
+    asked: [] as string[],
+    async pullRequestFor(sha: string) {
+      f.asked.push(sha);
+      const n = f.merges.get(sha);
+      if (n === "throw") throw new Error("forge unavailable");
+      return n ?? null;
+    },
+    prs: [] as OpenPullRequest[],
+    polls: 0,
+    async openPullRequests(base: string) {
+      f.polls++;
+      return f.prs.filter((p) => p.base === base);
+    },
+    async branchHead(branch: string) {
+      return branch === "main" ? f.head : undefined;
+    },
+  };
+  return f;
+}
+
+const pr = (number: number, headSha: string, baseSha: string, extra: Partial<OpenPullRequest> = {}): OpenPullRequest => ({ number, head: `pr-${number}`, headSha, base: "main", baseSha, author: "driver", fork: false, ...extra });
+
+const HOST = { PATH: process.env.PATH, HOME: process.env.HOME, GITHUB_TOKEN: "forge-token", GITHUB_SERVER_URL: "http://forgejo.localhost", GITHUB_API_URL: "http://forgejo.localhost/api/v1" };
+
+function serve(app: string, client: CiServeForge, extra: Partial<CiServeOptions> & { hostEnv?: Record<string, string | undefined> } = {}): { opts: CiServeOptions; events: CiServeEvent[]; state: string } {
+  const state = extra.stateDir ?? scratchDir("chant-ci-serve-state-");
+  const events: CiServeEvent[] = [];
+  const { hostEnv, ...rest } = extra;
+  const env = { ...HOST, ...hostEnv };
+  return {
+    state,
+    events,
+    opts: {
+      cwd: app,
+      forge: "forgejo",
+      repo: "demo/estate",
+      env: "prod",
+      client,
+      stateDir: state,
+      holder: "serve-test",
+      // A job image of its own, so no setup line downloads OpenTofu.
+      job: { image: "chant-ci-serve-test:local", chant: fakeChant(), hostEnv: env, passEnv: Object.keys(hostEnv ?? {}) },
+      log: (e) => events.push(e),
+      ...rest,
+    },
+  };
+}
+
+const argsOf = (report: string, stage: string): string[] => readFileSync(join(report, `${stage}.args`), "utf-8").trimEnd().split("\n");
+const ofType = <T extends CiServeEvent["type"]>(events: CiServeEvent[], type: T) => events.filter((e): e is Extract<CiServeEvent, { type: T }> => e.type === type);
+
+describe("chant ci serve (#4095)", () => {
+  test("one poll plans a new pull request head once, and skips forks and authors not given", async () => {
+    const { root, app, main } = workspace();
+    const head = pullHead(root, 1);
+    const forge = fakeForge(main);
+    forge.prs = [pr(1, head, main), pr(2, head, main, { fork: true }), pr(3, head, main, { author: "stranger" })];
+    const { opts, events, state } = serve(app, forge, { authors: ["Driver"] });
+
+    const first = await ciServePass(opts);
+    expect(first.jobs).toHaveLength(1);
+    expect(first.jobs[0]).toMatchObject({ job: "plan", pr: 1, commit: head, base: main, status: "ok", attempts: 1 });
+    expect(ofType(events, "baseline")).toEqual([{ type: "baseline", commit: main }]);
+    expect(ofType(events, "skipped").map((e) => [e.pr, e.reason])).toEqual([[2, "fork"], [3, "author"]]);
+    const argv = argsOf(first.jobs[0]!.report!, "pr-plan");
+    expect(argv.slice(0, 5)).toEqual(["components", "pr-plan", "--base", main, "--pr"]);
+    expect(argv[5]).toBe("1");
+    expect(argv).toContain("--member");
+
+    const group = prApplyGroup("prod", "app");
+    const file = join(state, `${group}.json`);
+    const saved = readCiServeState(file, { forge: "forgejo", repo: "demo/estate", env: "prod", branch: "main" });
+    expect(saved.target).toBe(main);
+    expect(saved.jobs[ciServeKey({ job: "plan", pr: 1, commit: head })]).toMatchObject({ status: "ok" });
+
+    // The same head again: nothing runs.
+    const second = await ciServePass(opts);
+    expect(second.jobs).toEqual([]);
+    expect(ofType(second.events, "started")).toEqual([]);
+    expect(forge.polls).toBe(2);
+  });
+
+  test("a merge to the target branch applies the new commit, measured from the head before it", async () => {
+    const { root, app, main } = workspace();
+    const forge = fakeForge(main);
+    const { opts, events } = serve(app, forge);
+    expect((await ciServePass(opts)).jobs).toEqual([]);
+    expect(ofType(events, "baseline")).toHaveLength(1);
+
+    const merged = commit(root, "app/merged.txt");
+    git(root, "push", "-q", "origin", "main");
+    forge.head = merged;
+    const pass = await ciServePass(opts);
+    expect(pass.jobs).toEqual([expect.objectContaining({ job: "apply", commit: merged, base: main, status: "ok" })]);
+    const argv = argsOf(pass.jobs[0]!.report!, "pr-apply");
+    expect(argv.slice(0, 4)).toEqual(["components", "pr-apply", "--base", main]);
+    expect(argv.slice(-2)).toEqual(["--resume", ".chant/pr-resume/pr-apply.json"]);
+    // The run was recorded on chant/lifecycle and pushed.
+    expect(git(root, "ls-remote", "origin", "chant/lifecycle")).toContain(git(root, "rev-parse", "chant/lifecycle"));
+
+    expect((await ciServePass(opts)).jobs).toEqual([]);
+  });
+
+  test("an apply gets the number of the pull request whose merge commit it is (#4112)", async () => {
+    const { root, app, main } = workspace();
+    const forge = fakeForge(main);
+    const seen: Array<number | undefined> = [];
+    const { opts } = serve(app, forge, { run: (req) => (seen.push(req.pr), ciRun(req)) });
+    await ciServePass(opts);
+    const merged = commit(root, "app/merged.txt");
+    git(root, "push", "-q", "origin", "main");
+    forge.head = merged;
+    forge.merges.set(merged, 17);
+
+    const pass = await ciServePass(opts);
+    expect(forge.asked).toEqual([merged]);
+    expect(pass.jobs).toEqual([expect.objectContaining({ job: "apply", commit: merged, pr: 17, status: "ok" })]);
+    expect(seen).toEqual([17]);
+  });
+
+  /** A serve over a repository whose baseline pass is done, with the `pr` each run was asked for in `seen`. */
+  async function mergeWithoutPr() {
+    const { root, app, main } = workspace();
+    const forge = fakeForge(main);
+    const seen: Array<number | undefined> = [];
+    const { opts } = serve(app, forge, { run: (req) => (seen.push(req.pr), ciRun(req)) });
+    await ciServePass(opts);
+    return { root, forge, seen, opts };
+  }
+
+  test("an apply with no pull request found runs without --pr (#4112)", async () => {
+    const { root, forge, seen, opts } = await mergeWithoutPr();
+    const direct = commit(root, "app/direct.txt");
+    git(root, "push", "-q", "origin", "main");
+    forge.head = direct;
+    const pass = await ciServePass(opts);
+    expect(pass.jobs).toEqual([expect.objectContaining({ job: "apply", commit: direct, status: "ok" })]);
+    expect(pass.jobs[0]!.pr).toBeUndefined();
+    expect(seen).toEqual([undefined]);
+  });
+
+  test("an apply whose pull request lookup fails runs without --pr (#4112)", async () => {
+    const { root, forge, seen, opts } = await mergeWithoutPr();
+    const broken = commit(root, "app/broken.txt");
+    git(root, "push", "-q", "origin", "main");
+    forge.head = broken;
+    forge.merges.set(broken, "throw");
+    const pass = await ciServePass(opts);
+    expect(pass.jobs).toEqual([expect.objectContaining({ job: "apply", commit: broken, status: "ok" })]);
+    expect(pass.jobs[0]!.pr).toBeUndefined();
+    expect(seen).toEqual([undefined]);
+  });
+
+  test("the forge clients look a merge commit up with one GET each (#4112)", async () => {
+    const sha = "c".repeat(40);
+    const calls: string[] = [];
+    const fetch = (body: unknown): ForgeFetch => async (url, init) => {
+      calls.push(`${init.method} ${new URL(url).pathname}`);
+      return { ok: true, status: 200, text: async () => JSON.stringify(body) };
+    };
+    const gh: CiServeForge = githubForge({ apiBase: "https://api.github.com", repo: "demo/estate", token: "t", fetch: fetch([{ number: 5, merged_at: "2026-10-01T00:00:00Z", merge_commit_sha: sha }]) }) as unknown as CiServeForge;
+    expect(await gh.pullRequestFor!(sha)).toBe(5);
+    const fj: CiServeForge = forgejoForge({ apiBase: "http://forgejo.localhost/api/v1", host: "forgejo.localhost", repo: "demo/estate", token: "t", fetch: fetch({ number: 6 }) }) as unknown as CiServeForge;
+    expect(await fj.pullRequestFor!(sha)).toBe(6);
+    expect(calls).toEqual([`GET /repos/demo/estate/commits/${sha}/pulls`, `GET /api/v1/repos/demo/estate/commits/${sha}/pull`]);
+  });
+
+  test("the group's lease held elsewhere holds the apply until it is released", async () => {
+    const { root, app, main } = workspace();
+    const forge = fakeForge(main);
+    const { opts, events } = serve(app, forge);
+    await ciServePass(opts);
+    const merged = commit(root, "app/merged.txt");
+    git(root, "push", "-q", "origin", "main");
+    forge.head = merged;
+
+    const group = prApplyGroup("prod", "app");
+    const other = await acquireLease(group, "other-host", { cwd: app, mode: "claim", requirePush: true });
+    expect(other.acquired).toBe(true);
+    const held = await ciServePass(opts);
+    expect(held.jobs).toEqual([]);
+    expect(ofType(events, "held")).toEqual([{ type: "held", job: "apply", commit: merged, reason: "lease", heldBy: "other-host" }]);
+    expect(formatCiServeEvent(ofType(events, "held")[0]!)).toContain("the apply lease is held by other-host");
+
+    expect(await releaseLease(group, "other-host", other.lease!.token, { cwd: app })).toBe(true);
+    const ran = await ciServePass(opts);
+    expect(ran.jobs).toEqual([expect.objectContaining({ job: "apply", commit: merged, base: main, status: "ok" })]);
+    // The serve released its own lease after the apply.
+    expect((await acquireLease(group, "after", { cwd: app, mode: "claim" })).acquired).toBe(true);
+  });
+
+  describe("a failed apply", () => {
+    // The failing apply runs once in beforeAll, and the two tests read what it left, so no test
+    // runs two jobs. A file's beforeAll is not counted in the unit budget (#2817).
+    let w: ReturnType<typeof workspace>;
+    let forge: ReturnType<typeof fakeForge>;
+    let failing: ReturnType<typeof serve>;
+    let merged: string;
+    let failed: Awaited<ReturnType<typeof ciServePass>>;
+    let record: string;
+
+    beforeAll(async () => {
+      w = workspace();
+      forge = fakeForge(w.main);
+      failing = serve(w.app, forge, { hostEnv: { FAKE_EXIT: "3", FAKE_ATTEMPT: "1" } });
+      await ciServePass(failing.opts);
+      merged = commit(w.root, "app/merged.txt");
+      git(w.root, "push", "-q", "origin", "main");
+      forge.head = merged;
+      failed = await ciServePass(failing.opts);
+      record = join(failing.state, "resume", `${prApplyRecordKey("prod", merged, "app")}.json`);
+    });
+
+    test("stays failed, and keeps its resume record", async () => {
+      expect(failed.jobs).toEqual([expect.objectContaining({ job: "apply", status: "fail", attempts: 1, failure: expect.stringContaining("command-failed") })]);
+      expect(JSON.parse(readFileSync(record, "utf-8"))).toEqual({ attempt: "1" });
+      expect(existsSync(join(failed.jobs[0]!.report!, "resume-in.json"))).toBe(false);
+
+      // Without --rerun-failed the failed apply is not run again.
+      expect((await ciServePass(failing.opts)).jobs).toEqual([]);
+    });
+
+    test("--rerun-failed runs it again with its resume record", async () => {
+      const retry = serve(w.app, forge, { stateDir: failing.state, rerunFailed: true, hostEnv: { FAKE_EXIT: "0", FAKE_ATTEMPT: "2" } });
+      const again = await ciServePass(retry.opts);
+      expect(again.jobs).toEqual([expect.objectContaining({ job: "apply", commit: merged, base: w.main, status: "ok", attempts: 2 })]);
+      expect(ofType(retry.events, "started")[0]).toMatchObject({ attempt: 2, resume: record });
+      // The job read the first attempt's record, and left its own.
+      expect(JSON.parse(readFileSync(join(again.jobs[0]!.report!, "resume-in.json"), "utf-8"))).toEqual({ attempt: "1" });
+      expect(JSON.parse(readFileSync(record, "utf-8"))).toEqual({ attempt: "2" });
+      // A passed apply is not run again, --rerun-failed or not.
+      expect((await ciServePass(retry.opts)).jobs).toEqual([]);
+    });
+  });
+
+  test("--max-jobs holds the jobs over the limit for the next poll", async () => {
+    const { root, app, main } = workspace();
+    const one = pullHead(root, 1);
+    const two = pullHead(root, 2);
+    const forge = fakeForge(main);
+    forge.prs = [pr(1, one, main), pr(2, two, main)];
+    const { opts, events } = serve(app, forge, { maxJobs: 1 });
+
+    const first = await ciServePass(opts);
+    expect(first.jobs.map((j) => [j.pr, j.status])).toEqual([[1, "ok"]]);
+    expect(ofType(events, "held")).toEqual([{ type: "held", job: "plan", commit: two, pr: 2, reason: "capacity" }]);
+
+    const second = await ciServePass(opts);
+    expect(second.jobs.map((j) => [j.pr, j.status])).toEqual([[2, "ok"]]);
+    expect((await ciServePass(opts)).jobs).toEqual([]);
+  });
+
+  test("the command line takes the repository, the limit, the authors and the interval", () => {
+    const args = parseArgs(["ci", "serve", "--forge", "forgejo", "--repo", "demo/estate", "--env", "local", "--max-jobs", "2", "--author", "a", "--author", "b", "--state", "/tmp/s", "--once", "--rerun-failed", "--interval", "45"]);
+    expect(args).toMatchObject({ command: "ci", path: "serve", forge: "forgejo", ciRepo: "demo/estate", env: "local", maxJobs: 2, ciAuthors: ["a", "b"], statePath: "/tmp/s", once: true, rerunFailed: true, interval: "45" });
+    expect(() => parseArgs(["ci", "serve", "--repo", "estate"])).toThrow(/owner\/name/);
+    expect(() => parseArgs(["ci", "serve", "--max-jobs", "0"])).toThrow(/1 or more/);
+    expect(ciServeInterval(undefined)).toBe(30_000);
+    expect(ciServeInterval("45")).toBe(45_000);
+    expect(ciServeInterval("2m")).toBe(120_000);
+    expect(() => ciServeInterval("0")).toThrow(/a second or more/);
+  });
+});