@intentius/chant 0.124.31 → 0.124.33

package/src/workspace/ci-serve.ts

@@ -3,11 +3,15 @@
  * host. It polls the forge for the open pull requests into the target branch
  * and for the branch's head, and runs the pull-request jobs with
  * {@link ciRun} (./ci-run.ts): `plan` for each new pull request head, and
- * `apply` for each new commit on the target branch, measured from the head
- * the previous poll saw (the push's `before`).
+ * `record` for each new commit on the target branch (#4168), measured from
+ * the head the previous poll saw (the push's `before`). The record applies
+ * nothing; with `applyOnMerge` (`--apply-on-merge`) the commit gets `apply`
+ * in its place, as before #4168. After each job settles it runs `ci tick`
+ * from the runs the jobs recorded, when the declaration has `ci.green`.
+ * What the list below says of the apply holds for it under `applyOnMerge`.
  *
  * - Dedupe. A state file in the state directory keeps every job it started,
- *   keyed by job and commit (`plan:<pr>:<sha>`, `apply:<sha>`), with how it
+ *   keyed by job and commit (`plan:<pr>:<sha>`, `record:<sha>`, `apply:<sha>`), with how it
  *   ended, and the target head the last apply was measured to. A job already
  *   in it is not started again, unless it failed and `rerunFailed` asks for
  *   it: the forge's "re-run failed jobs".
@@ -23,8 +27,34 @@
  *   as its `--resume` record, so a re-run of a failed apply at the same
  *   commit finishes under the approval it has. This replaces the forge's CI
  *   cache.
+ * - A gate per wave (#4003). With `--wave-gate` the apply job runs
+ *   `pr-apply --wave-gate`, which applies the waves in order and stops at
+ *   the first whose moved plan waits on its own gate. The job fails, and the
+ *   resume record keeps the waves that applied; once the wave's plan is
+ *   approved, `rerunFailed` runs the apply again and it carries on from
+ *   that wave.
  * - Capacity. At most `maxJobs` jobs run at once. A job over the limit is not
  *   recorded, so a later poll starts it.
+ * - Fast-forwards only (#4122). An apply runs only when the target's new head
+ *   descends from the head seen before it. Any other move, such as a force
+ *   push back to an older commit, is recorded as seen and not applied.
+ * - `[skip ci]` (#4122). A target head or pull request head whose commit
+ *   message says `[skip ci]` or `[ci skip]` is recorded as seen and not run,
+ *   as Actions does.
+ * - Build requests (#4169). With `buildCommand`, each `ci/build/<sha>`
+ *   tag the tick or `ci run --at` pushed is served once: the command runs
+ *   in a fresh detached worktree of the commit, with `CHANT_BUILD_COMMIT`,
+ *   `CHANT_BUILD_PHASES` (comma-separated) and `CHANT_BUILD_REASON` set,
+ *   and is expected to run the phases' Ops (`chant run <op>`), whose
+ *   records the tick reads. A build counts against `maxJobs`, and the state
+ *   file keeps it under `build:<sha>`. Without `buildCommand` the tags are
+ *   left alone.
+ * - The forge's ledger (#4121). Before each job, when no other job of this
+ *   serve runs, the clone's `chant/lifecycle` follows the forge's: it is
+ *   deleted when the forge has none, and reset to the forge's when neither
+ *   contains the other. Each is a warning. A local branch ahead of the
+ *   forge's keeps its facts, and the job's push sends them. Nothing here
+ *   pushes the ledger.
  *
  * Host mode only: each job runs in a fresh worktree on this host as this
  * user, with the clean environment `ciRun` gives it. A pull request from a
@@ -36,17 +66,20 @@
  * turn a checked payload into the same triggers and schedule them.
  */
 
-import { execFileSync } from "node:child_process";
-import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { execFileSync, spawn, spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
+import { CI_BUILD_TAG_PREFIX, readBuildRequests, type BuildRequest } from "./ci-builds";
 import { prApplyGroup, prApplyRecordKey } from "../components/pr-pipeline";
 import { forgeFromEnv, type OpenPullRequest } from "../pr-forge";
 import type { CommandContext } from "../cli/registry";
 import { formatError, formatWarning } from "../cli/format";
 import { acquireLease, currentHolderId, releaseLease, DEFAULT_LEASE_TTL_MS, type LeaseRecord } from "../lifecycle/lease";
 import { parseDuration } from "../op/duration";
-import { CI_JOB_OPS, ciJobPlace, ciSecretsOf, ciRun, ciStatusFromEnv, CiRunError, type CiJobName, type CiRunRequest, type CiRunResult } from "./ci-run";
-import { RUNNER_KEY_ENV } from "./ci-green";
+import { reportUrlOf } from "./ci-report-url";
+import { CI_JOB_OPS, ciJobPlace, ciLoopOptions, ciSecretsOf, ciRun, ciStatusFromEnv, CiRunError, type CiJobName, type CiRunRequest, type CiRunResult } from "./ci-run";
+import { CiGreenError, ciTick, RUNNER_KEY_ENV, signedRunsRequired } from "./ci-green";
 import { gitTop } from "./tree";
 import { ciServeRunnerKey, formatRunnerKeySetup, setupRunnerKey } from "./ci-runner-key";
 
@@ -61,10 +94,14 @@ export interface CiServeForge {
   pullRequestFor?(sha: string): Promise<number | null>;
 }
 
-/** A job the forge's state calls for: the plan of a pull request's head, or the apply of a target-branch commit. */
+/**
+ * A job the forge's state calls for: the plan of a pull request's head, or
+ * for a target-branch commit its record (#4168) or, with `applyOnMerge`,
+ * its apply.
+ */
 export type CiServeTrigger =
   | { job: "plan"; commit: string; base: string; pr: number; rerun?: boolean }
-  | { job: "apply"; commit: string; base: string; rerun?: boolean };
+  | { job: "apply" | "record"; commit: string; base: string; rerun?: boolean };
 
 /** One job in the state file. */
 export interface CiServeJobState {
@@ -92,9 +129,13 @@ export interface CiServeState {
   env: string;
   branch: string;
   member?: string;
-  /** The target branch's head the last apply was measured to, or the head the first poll found. */
+  /** The target branch's head the last apply was measured to, or the last head seen and not applied. */
   target?: string;
   jobs: Record<string, CiServeJobState>;
+  /** Plans seen and not run, by state key, with why (#4122). */
+  skipped?: Record<string, "skip-ci">;
+  /** The build requests served (#4169), by `build:<sha>`, with how each ended. */
+  builds?: Record<string, { tag: string; status: "running" | "ok" | "fail"; startedAt: string; settledAt?: string }>;
 }
 
 /** What a pass did, in order, for the log. */
@@ -105,6 +146,11 @@ export type CiServeEvent =
   | { type: "held"; job: CiJobName; commit: string; pr?: number; reason: "capacity" | "lease"; heldBy?: string }
   | { type: "started"; job: CiJobName; commit: string; pr?: number; base: string; runId: string; attempt: number; resume?: string }
   | { type: "settled"; job: CiJobName; commit: string; pr?: number; status: "ok" | "fail"; failure?: string; report?: string }
+  | { type: "not-run"; job: CiJobName; commit: string; pr?: number; reason: "not-fast-forward" | "skip-ci"; branch?: string; from?: string }
+  | { type: "ledger"; action: "deleted" | "reset"; local: string; forge?: string }
+  | { type: "build"; commit: string; phases: string[]; status: "started" | "ok" | "fail" | "held"; exitCode?: number }
+  /** `ci tick` after the pass (#4168): the tags it made from the runs. */
+  | { type: "tick"; made: string[] }
   | { type: "error"; message: string };
 
 /** The `ciRun` request fields `ci serve` passes through to every job: the generator's options and the runner's. */
@@ -126,6 +172,18 @@ export interface CiServeOptions {
   authors?: readonly string[];
   /** Run again the failed apply at the target's head, and the failed plans of open heads. */
   rerunFailed?: boolean;
+  /**
+   * Apply each new target-branch commit (#4168), as `ci serve` did before
+   * #4168. Default false: each new commit gets the record job, which applies
+   * nothing.
+   */
+  applyOnMerge?: boolean;
+  /**
+   * Run after each pass that settled a job (#4168): `ci tick` from the runs
+   * the jobs recorded. Returns the tags it made, or undefined when the
+   * declaration has no `ci.green`. The command passes one; a test may not.
+   */
+  tick?: () => Promise<string[] | undefined>;
   /** Passed to each job. `job.hostEnv` gets `GITHUB_REPOSITORY` set to `repo`. */
   job?: CiServeJobOptions;
   /** This process's lease identity. Default `currentHolderId()`. */
@@ -135,18 +193,26 @@ export interface CiServeOptions {
   now?: () => Date;
   /** Runs a job. Default {@link ciRun}. */
   run?: (req: CiRunRequest) => Promise<CiRunResult>;
+  /**
+   * The shell line that builds a requested commit (#4169), run in a fresh
+   * worktree of it. Without it `ci/build/<sha>` tags are not served.
+   */
+  buildCommand?: string;
+  /** Runs a build. Default: `sh -c <buildCommand>` in a detached worktree of the commit. */
+  build?: (input: { top: string; request: BuildRequest; command: string }) => Promise<number>;
 }
 
 /** The jobs this process runs now, by state key, shared by every pass of one loop. */
 export interface CiServeRuntime {
-  inflight: Map<string, Promise<CiServeJobState>>;
+  /** Jobs and build requests (#4169) alike, by state key: they share `maxJobs`. */
+  inflight: Map<string, Promise<unknown>>;
 }
 
 export const newCiServeRuntime = (): CiServeRuntime => ({ inflight: new Map() });
 
 /** The state key of a job. */
 export function ciServeKey(t: { job: CiJobName; commit: string; pr?: number }): string {
-  return t.job === "plan" ? `plan:${t.pr}:${t.commit}` : `apply:${t.commit}`;
+  return t.job === "plan" ? `plan:${t.pr}:${t.commit}` : `${t.job}:${t.commit}`;
 }
 
 const message = (err: unknown): string => (err instanceof Error ? err.message : String(err));
@@ -159,6 +225,78 @@ function tryGit(cwd: string, args: string[]): string | undefined {
   }
 }
 
+const LEDGER = "chant/lifecycle";
+/** Where the forge's ledger tip is fetched to, a ref only this module writes. */
+const FORGE_LEDGER = "refs/chant/ci-serve/forge-ledger";
+
+/** `git` with its exit status, for the calls whose answer is the status. */
+function gitStatus(cwd: string, args: string[]): { code: number; out: string } {
+  const r = spawnSync("git", args, { cwd, encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, LC_ALL: "C" } });
+  return { code: r.status ?? -1, out: (r.stdout ?? "").trim() };
+}
+
+/**
+ * Make the clone's `chant/lifecycle` follow the forge's (#4121). The clone's
+ * branch outlives the forge's, so a ledger the forge deleted or rewrote would
+ * make the next pr-apply find the two diverged and stall.
+ *
+ * - The forge has none: the local branch and its remote-tracking ref go.
+ * - Neither contains the other: the local branch and its remote-tracking ref
+ *   are reset to the forge's tip.
+ * - The local branch is ahead of the forge's, or behind it, or the same: it
+ *   is left, and an apply's push sends what it holds that the forge lacks.
+ *
+ * Nothing is pushed, so a rewritten ledger never goes back to the forge. A
+ * forge that can't be reached leaves everything as it is; the job's own
+ * fetch says why.
+ */
+export function reconcileCiServeLedger(top: string): CiServeEvent | undefined {
+  const local = tryGit(top, ["rev-parse", "--verify", "--quiet", `refs/heads/${LEDGER}^{commit}`]);
+  const listed = gitStatus(top, ["ls-remote", "--exit-code", "origin", `refs/heads/${LEDGER}`]);
+  if (listed.code === 2) {
+    // A remote-tracking ref left from a deleted branch would be the next push's lease, and refuse it.
+    tryGit(top, ["update-ref", "-d", `refs/remotes/origin/${LEDGER}`]);
+    if (!local) return undefined;
+    tryGit(top, ["update-ref", "-d", `refs/heads/${LEDGER}`, local]);
+    return { type: "ledger", action: "deleted", local };
+  }
+  if (listed.code !== 0 || !local) return undefined;
+  const forge = listed.out.split(/\s+/)[0];
+  if (!forge || forge === local) return undefined;
+  if (gitStatus(top, ["fetch", "--quiet", "--no-tags", "origin", `+refs/heads/${LEDGER}:${FORGE_LEDGER}`]).code !== 0) return undefined;
+  try {
+    const ahead = gitStatus(top, ["merge-base", "--is-ancestor", forge, local]).code === 0;
+    const behind = gitStatus(top, ["merge-base", "--is-ancestor", local, forge]).code === 0;
+    if (ahead || behind) return undefined;
+    tryGit(top, ["update-ref", `refs/heads/${LEDGER}`, forge, local]);
+    tryGit(top, ["update-ref", `refs/remotes/origin/${LEDGER}`, forge]);
+    return { type: "ledger", action: "reset", local, forge };
+  } finally {
+    tryGit(top, ["update-ref", "-d", FORGE_LEDGER]);
+  }
+}
+
+/** A commit message that asks for no CI run, as Actions and GitLab read it. */
+const SKIP_CI = /\[(?:skip ci|ci skip)\]/i;
+
+/** Whether `commit`'s message asks for no CI run. A commit this clone can't read is not skipped. */
+function skipsCi(top: string, commit: string): boolean {
+  const body = tryGit(top, ["log", "-1", "--format=%B", commit]);
+  return body !== undefined && SKIP_CI.test(body);
+}
+
+/**
+ * How the target branch moved from `from` to `head` (#4122): `fast-forward`
+ * when `head` descends from `from`, `other` when it does not, and `unknown`
+ * when `head` could not be fetched. Once `head` is here its history is too,
+ * so a `from` this clone lacks is not in that history.
+ */
+function targetMove(top: string, branch: string, from: string, head: string): "fast-forward" | "other" | "unknown" {
+  tryGit(top, ["fetch", "--quiet", "--no-tags", "origin", branch]);
+  if (gitStatus(top, ["cat-file", "-e", `${head}^{commit}`]).code !== 0) return "unknown";
+  return gitStatus(top, ["merge-base", "--is-ancestor", from, head]).code === 0 ? "fast-forward" : "other";
+}
+
 /** Where a serve keeps its state when `--state` names nowhere: beside the run events, in the repository's common git directory. */
 export function defaultCiServeStateDir(cwd: string): string {
   const common = tryGit(cwd, ["rev-parse", "--path-format=absolute", "--git-common-dir"]);
@@ -209,34 +347,55 @@ const failedEntry = (entry: CiServeJobState | undefined, key: string, rt: CiServ
 
 /**
  * One poll of the forge, as triggers: an apply when the target's head moved
- * since the last apply (none on the first poll, which only notes the head),
+ * since the last apply by a fast-forward (none on the first poll, which only
+ * notes the head, and none for a head whose message says `[skip ci]`),
  * then a plan for each open pull request head not yet planned, oldest pull
  * request first. A job already in the state file is left out unless it
- * failed and `rerunFailed` is set. The first poll's head is written to the
- * state file.
+ * failed and `rerunFailed` is set. The first poll's head, and a head seen
+ * and not applied, are written to the state file.
  */
 export async function ciServeObserve(opts: CiServeOptions, rt: CiServeRuntime = newCiServeRuntime()): Promise<{ triggers: CiServeTrigger[]; events: CiServeEvent[] }> {
   const place = servePlace(opts);
-  const state = readCiServeState(place.stateFile, { forge: opts.forge, repo: opts.repo, env: opts.env, branch: place.branch, ...(place.member ? { member: place.member } : {}) });
+  const fresh = { forge: opts.forge, repo: opts.repo, env: opts.env, branch: place.branch, ...(place.member ? { member: place.member } : {}) };
+  const state = readCiServeState(place.stateFile, fresh);
+  // Read again at the write: a job of this serve may have settled while the forge answered.
+  const update = (fn: (s: CiServeState) => void): void => {
+    const s = readCiServeState(place.stateFile, fresh);
+    fn(s);
+    writeState(place.stateFile, s);
+  };
   const triggers: CiServeTrigger[] = [];
   const events: CiServeEvent[] = [];
   const [head, prs] = await Promise.all([opts.client.branchHead(place.branch), opts.client.openPullRequests(place.branch)]);
+  // #4168: a new target commit is recorded, and applied only with applyOnMerge.
+  const onMerge: "apply" | "record" = opts.applyOnMerge ? "apply" : "record";
 
   if (head === undefined) {
     events.push({ type: "no-branch", branch: place.branch });
   } else if (state.target === undefined) {
     // What the branch held before this runner watched it was applied by whatever ran then.
-    state.target = head;
-    writeState(place.stateFile, state);
+    update((s) => {
+      s.target = head;
+    });
     events.push({ type: "baseline", commit: head });
   } else if (head !== state.target) {
-    const entry = state.jobs[ciServeKey({ job: "apply", commit: head })];
-    if (!entry) triggers.push({ job: "apply", commit: head, base: state.target });
-    else if (opts.rerunFailed && failedEntry(entry, ciServeKey(entry), rt)) triggers.push({ job: "apply", commit: head, base: entry.base, rerun: true });
+    const from = state.target;
+    const move = targetMove(place.top, place.branch, from, head);
+    const entry = state.jobs[ciServeKey({ job: onMerge, commit: head })];
+    if (move === "unknown") {
+      events.push({ type: "error", message: `${place.branch} is at ${head.slice(0, 12)} on the forge, and that commit could not be fetched; the next poll tries again` });
+    } else if (move === "other" || (!entry && skipsCi(place.top, head))) {
+      // #4122: a move that is not a fast-forward, or a head that asks for no run, is seen and not applied.
+      update((s) => {
+        s.target = head;
+      });
+      events.push({ type: "not-run", job: onMerge, commit: head, reason: move === "other" ? "not-fast-forward" : "skip-ci", ...(move === "other" ? { branch: place.branch, from } : {}) });
+    } else if (!entry) triggers.push({ job: onMerge, commit: head, base: from });
+    else if (opts.rerunFailed && failedEntry(entry, ciServeKey(entry), rt)) triggers.push({ job: onMerge, commit: head, base: entry.base, rerun: true });
   } else {
-    const key = ciServeKey({ job: "apply", commit: head });
+    const key = ciServeKey({ job: onMerge, commit: head });
     const entry = state.jobs[key];
-    if (opts.rerunFailed && failedEntry(entry, key, rt)) triggers.push({ job: "apply", commit: head, base: entry.base, rerun: true });
+    if (opts.rerunFailed && failedEntry(entry, key, rt)) triggers.push({ job: onMerge, commit: head, base: entry.base, rerun: true });
   }
 
   const authors = opts.authors?.length ? new Set(opts.authors.map((a) => a.toLowerCase())) : undefined;
@@ -252,8 +411,17 @@ export async function ciServeObserve(opts: CiServeOptions, rt: CiServeRuntime =
     const key = ciServeKey({ job: "plan", commit: pr.headSha, pr: pr.number });
     const entry = state.jobs[key];
     const base = pr.baseSha ?? head;
-    if (!base) continue;
-    if (!entry) triggers.push({ job: "plan", commit: pr.headSha, base, pr: pr.number });
+    if (!base || state.skipped?.[key]) continue;
+    if (!entry) {
+      // #4122: a head whose message asks for no run is seen and not planned.
+      if (gitStatus(place.top, ["cat-file", "-e", `${pr.headSha}^{commit}`]).code !== 0) tryGit(place.top, ["fetch", "--quiet", "--no-tags", "origin", `refs/pull/${pr.number}/head`]);
+      if (skipsCi(place.top, pr.headSha)) {
+        update((s) => {
+          s.skipped = { ...s.skipped, [key]: "skip-ci" };
+        });
+        events.push({ type: "not-run", job: "plan", commit: pr.headSha, pr: pr.number, reason: "skip-ci" });
+      } else triggers.push({ job: "plan", commit: pr.headSha, base, pr: pr.number });
+    }
     else if (opts.rerunFailed && failedEntry(entry, key, rt)) triggers.push({ job: "plan", commit: pr.headSha, base: entry.base, pr: pr.number, rerun: true });
   }
   return { triggers, events };
@@ -316,6 +484,12 @@ export async function ciServeSchedule(opts: CiServeOptions, triggers: readonly C
       lease = got.lease;
     }
 
+    // #4121: with no other job of this serve writing it, the clone's ledger follows the forge's.
+    if (rt.inflight.size === 0) {
+      const ledger = reconcileCiServeLedger(place.top);
+      if (ledger) emit(ledger);
+    }
+
     const previous = readCiServeState(place.stateFile, fresh).jobs[key];
     const attempt = (previous?.attempts ?? 0) + 1;
     const runId = `chant-serve-${now().getTime()}-${t.job}-${t.commit.slice(0, 8)}`;
@@ -324,7 +498,7 @@ export async function ciServeSchedule(opts: CiServeOptions, triggers: readonly C
     const entry: CiServeJobState = { job: t.job, commit: t.commit, base: t.base, ...pr, status: "running", runId, attempts: attempt, startedAt: now().toISOString() };
     update((s) => {
       s.jobs[key] = entry;
-      if (t.job === "apply" && !t.rerun) s.target = t.commit;
+      if (t.job !== "plan" && !t.rerun) s.target = t.commit;
     });
     emit({ type: "started", job: t.job, commit: t.commit, ...pr, base: t.base, runId, attempt, ...(resume && existsSync(resume) ? { resume } : {}) });
 
@@ -339,7 +513,7 @@ export async function ciServeSchedule(opts: CiServeOptions, triggers: readonly C
         emit({ type: "error", message: `${key} settled, and the state file could not be written: ${message(err)}` });
       }
       emit({ type: "settled", job: t.job, commit: t.commit, ...pr, status: settled.status === "ok" ? "ok" : "fail", ...(settled.failure ? { failure: settled.failure } : {}), ...(settled.report ? { report: settled.report } : {}) });
-      return settled;
+      return serveTick(opts).then(() => settled);
     });
     rt.inflight.set(key, job);
     started.push(job);
@@ -367,7 +541,7 @@ async function runJob(
     tryGit(place.top, ["fetch", "--quiet", "--no-tags", "origin", t.job === "plan" ? `refs/pull/${t.pr}/head` : place.branch]);
     const hostEnv = { ...(opts.job?.hostEnv ?? process.env), GITHUB_REPOSITORY: opts.repo };
     // An apply is for a merge: pass the pull request whose merge commit it is, so the commit status link and the run index use `<pr>/<commit>`. None found, or a forge that errs, is today's behaviour.
-    if (t.job === "apply" && opts.client.pullRequestFor) merged = (await opts.client.pullRequestFor(t.commit).catch(() => null)) ?? undefined;
+    if (t.job !== "plan" && opts.client.pullRequestFor) merged = (await opts.client.pullRequestFor(t.commit).catch(() => null)) ?? undefined;
     const run = opts.run ?? ciRun;
     const r = await run({
       ...opts.job,
@@ -393,13 +567,101 @@ async function runJob(
   }
 }
 
-/** One pass: observe, schedule, and with `wait` wait for the jobs it started. */
+/** One pass: observe, schedule, serve build requests, and with `wait` wait for the jobs it started. */
 export async function ciServePass(opts: CiServeOptions, rt: CiServeRuntime = newCiServeRuntime(), wait = true): Promise<{ events: CiServeEvent[]; jobs: CiServeJobState[] }> {
   const observed = await ciServeObserve(opts, rt);
   for (const e of observed.events) opts.log?.(e);
   const scheduled = await ciServeSchedule(opts, observed.triggers, rt);
+  const builds = opts.buildCommand ? ciServeBuilds(opts, rt) : { started: [], events: [] };
   const jobs = wait ? await Promise.all(scheduled.started) : [];
-  return { events: [...observed.events, ...scheduled.events], jobs };
+  if (wait) await Promise.all(builds.started);
+  return { events: [...observed.events, ...scheduled.events, ...builds.events], jobs };
+}
+
+/**
+ * Serve the build requests (#4169): fetch the `ci/build/*` tags, and start
+ * `buildCommand` for each one not in the state file yet, oldest first, while
+ * fewer than `maxJobs` jobs run. A build that fails is recorded and not run
+ * again; the tick decides what its phases say.
+ */
+export function ciServeBuilds(opts: CiServeOptions, rt: CiServeRuntime): { started: Promise<void>[]; events: CiServeEvent[] } {
+  const place = servePlace(opts);
+  const fresh = { forge: opts.forge, repo: opts.repo, env: opts.env, branch: place.branch, ...(place.member ? { member: place.member } : {}) };
+  const update = (fn: (s: CiServeState) => void): void => {
+    const st = readCiServeState(place.stateFile, fresh);
+    fn(st);
+    writeState(place.stateFile, st);
+  };
+  const events: CiServeEvent[] = [];
+  const started: Promise<void>[] = [];
+  tryGit(place.top, ["fetch", "--quiet", "--prune", "origin", `+refs/tags/${CI_BUILD_TAG_PREFIX}*:refs/tags/${CI_BUILD_TAG_PREFIX}*`]);
+  const served = readCiServeState(place.stateFile, fresh).builds ?? {};
+  const max = opts.maxJobs ?? 1;
+  const now = opts.now ?? (() => new Date());
+  const requests = [...readBuildRequests(place.top).values()].sort((a, b) => a.at.localeCompare(b.at));
+  for (const request of requests) {
+    const key = `build:${request.commit}`;
+    if (served[key]?.tag === request.tag || rt.inflight.has(key)) continue;
+    const log = (e: CiServeEvent) => {
+      events.push(e);
+      opts.log?.(e);
+    };
+    if (rt.inflight.size >= max) {
+      log({ type: "build", commit: request.commit, phases: request.phases, status: "held" });
+      continue;
+    }
+    update((st) => {
+      st.builds = { ...st.builds, [key]: { tag: request.tag, status: "running", startedAt: now().toISOString() } };
+    });
+    log({ type: "build", commit: request.commit, phases: request.phases, status: "started" });
+    const build = opts.build ?? runBuild;
+    const job = build({ top: place.top, request, command: opts.buildCommand! })
+      .catch(() => 1)
+      .then((code) => {
+        update((st) => {
+          st.builds = { ...st.builds, [key]: { tag: request.tag, status: code === 0 ? "ok" : "fail", startedAt: st.builds?.[key]?.startedAt ?? now().toISOString(), settledAt: now().toISOString() } };
+        });
+        opts.log?.({ type: "build", commit: request.commit, phases: request.phases, status: code === 0 ? "ok" : "fail", exitCode: code });
+      });
+    const tracked = job.finally(() => rt.inflight.delete(key));
+    rt.inflight.set(key, tracked);
+    started.push(tracked);
+  }
+  return { started, events };
+}
+
+/** `sh -c <command>` in a fresh detached worktree of the request's commit, removed after. Resolves to the exit code. */
+async function runBuild(input: { top: string; request: BuildRequest; command: string }): Promise<number> {
+  const dir = mkdtempSync(join(tmpdir(), "chant-ci-build-"));
+  try {
+    execFileSync("git", ["worktree", "add", "--quiet", "--detach", dir, input.request.commit], { cwd: input.top, stdio: ["ignore", "pipe", "pipe"] });
+    return await new Promise<number>((done) => {
+      const child = spawn("sh", ["-c", input.command], {
+        cwd: dir,
+        stdio: ["ignore", "inherit", "inherit"],
+        env: { ...process.env, CHANT_BUILD_COMMIT: input.request.commit, CHANT_BUILD_PHASES: input.request.phases.join(","), CHANT_BUILD_REASON: input.request.reason },
+      });
+      child.on("error", () => done(127));
+      child.on("close", (code) => done(code ?? 1));
+    });
+  } finally {
+    tryGit(input.top, ["worktree", "remove", "--force", dir]);
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+/** `ci tick` after a job settled (#4168), logged; a tick that fails is an error event and never fails the job. */
+async function serveTick(opts: CiServeOptions): Promise<CiServeEvent | undefined> {
+  if (!opts.tick) return undefined;
+  let event: CiServeEvent | undefined;
+  try {
+    const made = await opts.tick();
+    event = made === undefined ? undefined : { type: "tick", made };
+  } catch (err) {
+    event = { type: "error", message: `ci tick after a job settled: ${message(err)}` };
+  }
+  if (event) opts.log?.(event);
+  return event;
 }
 
 /**
@@ -442,6 +704,21 @@ export function formatCiServeEvent(e: CiServeEvent): string {
       return `${at(e.job, e.commit, e.pr)} started, run ${e.runId}, attempt ${e.attempt}${e.resume ? ", resuming" : ""}`;
     case "settled":
       return `${at(e.job, e.commit, e.pr)} ${e.status === "ok" ? "passed" : `failed${e.failure ? `: ${e.failure}` : ""}`}${e.report ? `; report ${e.report}` : ""}`;
+    case "not-run":
+      return e.reason === "skip-ci"
+        ? `${at(e.job, e.commit, e.pr)} is not run: its commit message says [skip ci]`
+        : `${e.branch ?? "the target branch"} moved from ${e.from?.slice(0, 12) ?? "a commit"} to ${e.commit.slice(0, 12)}, which is not a fast-forward, so ${e.commit.slice(0, 12)} is recorded as seen and not applied`;
+    case "ledger":
+      return e.action === "deleted"
+        ? `the forge has no ${LEDGER}, so this clone's, at ${e.local.slice(0, 12)}, was deleted`
+        : `the forge's ${LEDGER} at ${e.forge!.slice(0, 12)} does not contain this clone's at ${e.local.slice(0, 12)}, so this clone's was reset to the forge's and not pushed`;
+    case "build": {
+      const what = `the build of ${e.phases.join(", ")} at ${e.commit.slice(0, 12)}`;
+      if (e.status === "held") return `${what} waits: --max-jobs jobs run`;
+      return e.status === "started" ? `${what} started` : e.status === "ok" ? `${what} passed` : `${what} failed${e.exitCode !== undefined ? ` with exit code ${e.exitCode}` : ""}`;
+    }
+    case "tick":
+      return e.made.length > 0 ? `ci tick made ${e.made.join(", ")}` : "ci tick: no tag due";
     case "error":
       return `error: ${e.message}`;
   }
@@ -478,7 +755,7 @@ export function runnerKeySetupCommand(ctx: CommandContext): number {
 const USAGE =
   "chant ci serve --setup-key [--trust [--principal <name>]] [--state <dir>] [--branch <name>] | " +
   "chant ci serve --forge forgejo|github --repo <owner/repo> --env <env> [--interval <s>] [--max-jobs <n>] [--state <dir>] " +
-  "[--author <login>]... [--once] [--rerun-failed] [--gate <name>] [--branch <name>] [--events-json] [--report-url <url>] [--no-status] [--status-url <url template>] " +
+  "[--author <login>]... [--once] [--rerun-failed] [--build-command <line>] [--gate <name>] [--wave-gate <name>] [--branch <name>] [--events-json] [--report-url <url>] [--no-status] [--status-url <url template>] " +
   "[--image <image>] [--before-script <line>]... [--pr-build] [--pr-build-command <line>] [--chant <command>] [--pass-env <NAME>]... [--secrets <file>] [--key <runner key.pem>] [--unsigned]";
 
 const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
@@ -501,6 +778,8 @@ export async function runCiServe(ctx: CommandContext): Promise<number> {
     if (!args.ciRepo) throw new CiRunError("--repo names the repository whose pull requests ci serve polls: --repo <owner/repo>");
     if (!args.env) throw new CiRunError("--env names the environment the pipeline is for: --env <env>");
     for (const name of args.passEnv ?? []) if (!ENV_NAME.test(name)) throw new CiRunError(`--pass-env takes a variable's name, not ${JSON.stringify(name)}`);
+    // #4168: a gate per wave gates the apply on merge, which a merge runs only when opted in.
+    if (args.waveGate && !args.applyOnMerge) throw new CiRunError("--wave-gate gates the apply on merge, so it needs --apply-on-merge (#4168); without it a merge gets the record job, which applies nothing");
     const intervalMs = ciServeInterval(args.interval);
     const cwd = process.cwd();
     const top = gitTop(cwd);
@@ -508,17 +787,33 @@ export async function runCiServe(ctx: CommandContext): Promise<number> {
     const stateDir = args.statePath ? resolve(args.statePath) : defaultCiServeStateDir(cwd);
     const branch = args.branch ?? "main";
     // #4099: the host's key, checked against .chant/trust.json at the target's head before the first poll.
-    const keyed = ciServeRunnerKey({ top, stateDir, branch, ...(args.key !== undefined ? { keyFile: resolve(args.key) } : {}), envKey: process.env[RUNNER_KEY_ENV] || undefined, unsigned: args.unsigned });
+    // #4156: signing is opt-in. Without ci.green.signedSince no key is quiet, and an untrusted one warns once and serves unsigned.
+    const keyed = ciServeRunnerKey({
+      top,
+      stateDir,
+      branch,
+      ...(args.key !== undefined ? { keyFile: resolve(args.key) } : {}),
+      envKey: process.env[RUNNER_KEY_ENV] || undefined,
+      unsigned: args.unsigned,
+      required: signedRunsRequired(cwd),
+    });
     const fromFlag = keyed.fromFlag;
     const runnerKey = keyed.pem;
-    console.error(`ci serve: ${runnerKey === undefined ? "runs are not signed (--unsigned)" : `signing runs with the runner key from ${keyed.source}`}`);
+    if (keyed.warning) console.error(formatWarning({ message: keyed.warning }));
+    if (runnerKey !== undefined) console.error(`ci serve: signing runs with the runner key from ${keyed.source}`);
+    else if (keyed.note && args.verbose) console.error(`ci serve: ${keyed.note}`);
     // #4096: read once, so a loose or broken secrets file stops the serve before its first poll.
     const secrets = ciSecretsOf(args.ciSecrets);
+    // #4160: without --report-url, the declaration's ci.reportUrl.
+    const reportUrl = await reportUrlOf(args.reportUrl, cwd);
+    if (reportUrl && !args.reportUrl) console.error(`ci serve: reports and the run index go to ${reportUrl}, the declaration's ci.reportUrl`);
     const hostEnv = { ...process.env, GITHUB_REPOSITORY: args.ciRepo };
     const client = forgeFromEnv(forge, hostEnv);
     const { status, warning } = ciStatusFromEnv({ forge, hostEnv, noStatus: args.noStatus, statusUrl: args.statusUrl });
     if (warning) console.error(formatWarning({ message: warning }));
     if (!client.openPullRequests || !client.branchHead) throw new CiRunError(`the ${forge} client cannot list pull requests`);
+    // #4168: a host that applied each merge before the upgrade hears that it no longer does.
+    if (!args.applyOnMerge) console.error("ci serve: each new commit on the target branch gets the record job, which applies nothing (#4168); --apply-on-merge applies it");
     const opts: CiServeOptions = {
       cwd,
       forge,
@@ -529,15 +824,19 @@ export async function runCiServe(ctx: CommandContext): Promise<number> {
       ...(args.maxJobs ? { maxJobs: args.maxJobs } : {}),
       ...(args.ciAuthors?.length ? { authors: args.ciAuthors } : {}),
       ...(args.rerunFailed ? { rerunFailed: true } : {}),
+      ...(args.ciBuildCommand ? { buildCommand: args.ciBuildCommand } : {}),
+      ...(args.applyOnMerge ? { applyOnMerge: true } : {}),
+      // #4168: the tick from the runs the jobs recorded, signed with the runs' key. No ci.green, no tick.
+      tick: async () => {
+        try {
+          return (await ciTick({ cwd, source: "runs", ...(runnerKey !== undefined ? { runnerKey } : {}) })).made;
+        } catch (err) {
+          if (err instanceof CiGreenError && err.code === "ci-green-undeclared") return undefined;
+          throw err;
+        }
+      },
       job: {
-        loop: {
-          ...(args.gate ? { gate: args.gate } : {}),
-          ...(args.branch ? { branch: args.branch } : {}),
-          ...(args.eventsJson ? { events: true } : {}),
-          ...(args.reportUrl ? { reportUrl: args.reportUrl } : {}),
-          ...(args.prBuild ? { build: true } : {}),
-          ...(args.prBuildCommand ? { buildCommand: args.prBuildCommand } : {}),
-        },
+        loop: ciLoopOptions(reportUrl ? { ...args, reportUrl } : args),
         ...(args.runIndexLimit ? { runIndexLimit: args.runIndexLimit } : {}),
         ...(args.image ? { image: args.image } : {}),
         ...(args.beforeScript?.length ? { beforeScript: args.beforeScript } : {}),
@@ -550,7 +849,7 @@ export async function runCiServe(ctx: CommandContext): Promise<number> {
         hostEnv,
         stepOutput: (line) => void process.stderr.write(`${line}\n`),
       },
-      log: (e) => console.error(e.type === "error" ? formatWarning({ message: formatCiServeEvent(e) }) : `ci serve: ${formatCiServeEvent(e)}`),
+      log: (e) => console.error(e.type === "error" || e.type === "ledger" ? formatWarning({ message: formatCiServeEvent(e) }) : `ci serve: ${formatCiServeEvent(e)}`),
     };
     if (args.once) {
       const pass = await ciServePass(opts);