omp-conductor 0.18.2 → 0.19.0

package/src/pause.ts

@@ -0,0 +1,233 @@
+/**
+ * The dispatch pause sentinel: one file per project (plus the legacy bare one),
+ * its provenance line, and the reads that judge it.
+ *
+ * A leaf module on purpose. `daemon.ts` owns dispatch and imports `doctor.ts`,
+ * so `doctor.ts` cannot import `daemon.ts` back — yet a fence is exactly the
+ * kind of state a health check has to read (#938). Both import this instead,
+ * and `daemon.ts` re-exports every name so no existing call site moves.
+ *
+ * Nothing here decides policy: it reads and writes the file honestly, and an
+ * unreadable or malformed sentinel is `undefined` so every caller keeps failing
+ * closed rather than inferring "no pause".
+ */
+
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { stateDir } from "./config.ts";
+
+/**
+ * The sentinel's provenance line, in one place.
+ *
+ * It was written twice — here and as a copy in the setup wizard's
+ * `isValidPauseBytes`, whose comment claimed to be "the exact grammar
+ * `pauseInstance` reads". Adding `owner=` (#938) proved the hazard rather than
+ * describing it: the copy did not know the new field, so a setup fence read as
+ * *malformed* to the transaction that had just written it, and the refusal
+ * path then declined to remove its own hold. One export, no copies.
+ */
+const PROVENANCE_LINE = /^source=(\S+)(?: reason="([^"]*)")?(?: owner=(\d+))?$/;
+
+/**
+ * Whether these bytes are a readable pause sentinel: a parseable ISO instant,
+ * then a provenance line. Exported so no caller has to restate the grammar to
+ * tell a hold it may restore from one it must never delete.
+ */
+export function pauseBytesValid(bytes: string): boolean {
+  const [line1, line2] = bytes.split("\n");
+  if (!Number.isFinite(Date.parse(line1?.trim() ?? ""))) return false;
+  if (line2 === undefined) return false;
+  return PROVENANCE_LINE.test(line2.trim());
+}
+
+/**
+ * Pause is a file rather than process state on purpose: `omp-conductor hold`
+ * runs in a different process from the daemon, and a flag
+ * on disk needs no IPC and survives a restart. A daemon that crashed while
+ * paused comes back paused.
+ */
+export function pausedPath(project?: string): string {
+  return join(stateDir(), project === undefined ? "paused" : `paused-${project}`);
+}
+
+function activePausePaths(project?: string): string[] {
+  const paths = project === undefined ? [pausedPath()] : [pausedPath(project), pausedPath()];
+  return paths.filter((path) => existsSync(path));
+}
+
+export function isPaused(project?: string): boolean {
+  return activePausePaths(project).length !== 0;
+}
+
+/**
+ * The epoch-ms timestamp at which the current pause began. A project pause also
+ * observes the legacy bare sentinel, which pauses every project. If any active
+ * sentinel is unreadable or unparseable, the timestamp is unknown so callers
+ * continue to fail closed.
+ */
+export function pausedAt(project?: string): number | undefined {
+  const paths = activePausePaths(project);
+  if (paths.length === 0) return undefined;
+  const times: number[] = [];
+  for (const path of paths) {
+    try {
+      const first = readFileSync(path, "utf8").split("\n")[0]?.trim();
+      if (first === undefined || first === "") return undefined;
+      const time = Date.parse(first);
+      if (Number.isNaN(time)) return undefined;
+      times.push(time);
+    } catch {
+      return undefined;
+    }
+  }
+  return Math.min(...times);
+}
+
+/**
+ * Who paused the project and why. Per-project provenance wins when both its
+ * sentinel and the legacy all-project sentinel are active.
+ */
+export function pauseProvenance(
+  project?: string,
+): { source: string; reason?: string; owner?: number } | undefined {
+  const paths =
+    project === undefined
+      ? [pausedPath()]
+      : [pausedPath(project), pausedPath()];
+  const path = paths.find((candidate) => existsSync(candidate));
+  if (path === undefined) return undefined;
+  try {
+    const second = readFileSync(path, "utf8").split("\n")[1];
+    if (second === undefined) return undefined;
+    const match = PROVENANCE_LINE.exec(second.trim());
+    if (match === null) return undefined;
+    const source = match[1]!;
+    const reason = match[2];
+    const owner = match[3] === undefined ? undefined : Number(match[3]);
+    return {
+      source,
+      ...(reason === undefined ? {} : { reason }),
+      ...(owner === undefined ? {} : { owner }),
+    };
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * One pause sentinel FILE read as an instance identity: who set it, why, and
+ * the creation instant, all from that exact file. An unreadable or malformed
+ * file is undefined — the caller may treat it as absence.
+ */
+export function pauseInstanceAt(
+  path: string,
+): { source: string; reason?: string; since: number; owner?: number } | undefined {
+  try {
+    const [line1, line2] = readFileSync(path, "utf8").split("\n");
+    const since = Date.parse(line1?.trim() ?? "");
+    if (!Number.isFinite(since)) return undefined;
+    if (line2 === undefined) return undefined;
+    const match = PROVENANCE_LINE.exec(line2.trim());
+    if (match === null) return undefined;
+    const source = match[1]!;
+    const reason = match[2];
+    const owner = match[3] === undefined ? undefined : Number(match[3]);
+    return {
+      source,
+      since,
+      ...(reason === undefined ? {} : { reason }),
+      ...(owner === undefined ? {} : { owner }),
+    };
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * One pause sentinel read as a single identity: who set it, why, and the
+ * creation instant, all from the SAME file that was selected. Unlike pairing
+ * {@link pauseProvenance} with {@link pausedAt} — which can describe
+ * different files when a project pause coexists with the legacy global
+ * sentinel, letting a stale global timestamp mask a recreated project pause —
+ * this reads provenance and timestamp from one sentinel, so a caller can prove
+ * "the pause I set still exists" instead of "some pause with the same labels
+ * still exists" (#377). Per-project sentinel wins, like {@link pauseProvenance}.
+ */
+export function pauseInstance(
+  project?: string,
+): { source: string; reason?: string; since: number; owner?: number } | undefined {
+  const paths =
+    project === undefined
+      ? [pausedPath()]
+      : [pausedPath(project), pausedPath()];
+  const path = paths.find((candidate) => existsSync(candidate));
+  if (path === undefined) return undefined;
+  return pauseInstanceAt(path);
+}
+
+/**
+ * Compare-and-clear one pause sentinel (#780 review): remove `path` only while
+ * it still holds exactly the `expected` instance — same source, same reason,
+ * same creation instant — as read by {@link pauseInstance}. A hold or pause
+ * that replaced or recreated the sentinel between the read and the clear is a
+ * newer instance (writes always re-stamp `since`), so it is never destroyed:
+ * returning false keeps the newer fence in force. Scoped strictly to one
+ * caller-provided path, so auto-expiry can clear the per-project spend-cap
+ * sentinel without ever touching the legacy global sentinel.
+ */
+export function clearPauseIfUnchanged(
+  path: string,
+  expected: { source: string; reason?: string; since: number },
+): boolean {
+  const current = pauseInstanceAt(path);
+  if (current === undefined) return false;
+  if (
+    current.source !== expected.source ||
+    current.since !== expected.since ||
+    current.reason !== expected.reason
+  ) {
+    return false;
+  }
+  rmSync(path, { force: true });
+  return true;
+}
+
+/**
+ * The pause sentinel's `source=` token, proven from a verb. The sentinel's
+ * source line is read back as a single `\S+` token (see {@link pauseInstance}
+ * and {@link pauseProvenance}), so a verb that contains a space (`setup host`)
+ * is unrepresentable verbatim and must be encoded before it reaches disk —
+ * otherwise the fence cannot prove its own pause and refuses forever (#552).
+ * Spaces become `-`; the human-readable verb is preserved in the sentinel's
+ * `reason=` instead.
+ */
+export function pauseSourceToken(verb: string): string {
+  return verb.trim().replace(/\s+/g, "-");
+}
+
+export function setPaused(
+  v: boolean,
+  why?: { source: string; reason?: string; owner?: number },
+  project?: string,
+): void {
+  const path = pausedPath(project);
+  if (v) {
+    mkdirSync(dirname(path), { recursive: true });
+    const line1 = `${new Date().toISOString()}\n`;
+    if (why === undefined) {
+      writeFileSync(path, line1);
+    } else {
+      const reason = why.reason === undefined ? "" : ` reason="${why.reason.replaceAll('"', "")}"`;
+      // `owner=` is written ONLY by a fence whose lifetime is a process's
+      // (#938): a `setup` apply holds dispatch across its own transaction, and
+      // when that process is killed nothing runs to lift it. An operator
+      // `hold` deliberately records none — it outlives every process by
+      // design, and a pid there would invite exactly the wrong inference.
+      const owner = why.owner === undefined ? "" : ` owner=${why.owner}`;
+      writeFileSync(path, `${line1}source=${why.source}${reason}${owner}\n`);
+    }
+  } else {
+    rmSync(path, { force: true });
+    if (project !== undefined) rmSync(pausedPath(), { force: true });
+  }
+}

package/src/settlement.ts

@@ -22,7 +22,7 @@ import {
   analyseSettlement,
   deriveChangedLine,
 } from "./diff-flags.ts";
-import { classifyRun, normalise, type ClassifyFacts } from "./failure-class.ts";
+import { SPINNING_CAP_CLASSES, classifyRun, normalise, type ClassifyFacts } from "./failure-class.ts";
 import { GhPrMissingError } from "./tracker/github.ts";
 import { formatModelsTried, modelsTried } from "./model-fallback.ts";
 import { PR_LOOKUP_WINDOW_MS } from "./decisions.ts";
@@ -785,6 +785,55 @@ export async function reconcileStaleLabels(d: Pick<SettlementDeps, "project" | "
   }
 }
 
+/**
+ * Retire grooming verdicts whose issue has closed (#964).
+ *
+ * The sibling above says it for labels: *never retain an `agent:*` label on a
+ * closed issue — the work is done by some route, and the label only makes the
+ * board lie about it.* A grooming verdict on a closed issue lies in exactly the
+ * same way, one surface over, and nothing retired one: `reconcileGrooming`
+ * clears only the two per-pass admission holds (`file-lane`, `depends-on`), so
+ * `promotable`, `considered`, `needs-product-decision` and in-flight batch
+ * markers accumulated for the life of the store.
+ *
+ * Measured on this fleet, 2026-08-23: 64 of 65 rows named a closed issue, all 26
+ * `promotable` rows did, and `status` therefore advertised 26 candidates ready
+ * to promote on a repo whose only open issue was one blocked on an operator.
+ *
+ * Deliberately **not** folded into `reconcileStaleLabels`, though the reasoning
+ * is its sibling's: that function iterates issues *carrying a label*, and a
+ * promotable issue carries none — not the queue label (promotion is what adds
+ * it) and no state label. It would have seen almost none of these rows.
+ *
+ * `listOpenIssues` is the required input and the reason this is safe: it is
+ * complete (paginated to the end), conditional (an unchanged repo costs a 304),
+ * and it *throws* rather than returning a short list. A label-filtered read here
+ * would condemn every verdict it failed to mention.
+ */
+export async function reconcileGroomingClosures(
+  d: Pick<SettlementDeps, "project" | "tracker" | "store">,
+): Promise<void> {
+  const { project, tracker, store } = d;
+  let open: readonly { number: number }[];
+  try {
+    open = await tracker.listOpenIssues();
+  } catch (err) {
+    // Fail closed, and loudly enough to notice: retiring on an unread tracker
+    // would delete the whole table the first time GitHub rate-limited us.
+    log(`grooming closure reconcile skipped (${errText(err)}) — retrying next tick`);
+    return;
+  }
+  const retired = store.retireGroomingNotOpen(
+    project.name,
+    open.map((issue) => issue.number),
+  );
+  if (retired.length === 0) return;
+  log(
+    `grooming reconciled: ${retired.length} verdict(s) retired for closed issues ` +
+      `(${retired.slice(0, 8).map((n) => `#${n}`).join(", ")}${retired.length > 8 ? ", …" : ""})`,
+  );
+}
+
 /**
  * Settles `claimed`/`running` rows left by a dead daemon process and, before
  * marking each one `orphaned`, salvages any dirty worktree.
@@ -1669,7 +1718,8 @@ async function recoverRun(
 
   if (recovery === "escalate") {
     const detail = [evidence];
-    if (cls === "turn-cap-spinning" || cls === "wall-clock-cap-spinning") {
+    const spinning = SPINNING_CAP_CLASSES.includes(cls);
+    if (spinning) {
       const calls = lastToolCalls(run.sessionFile);
       detail.push(
         calls.length === 0
@@ -1687,6 +1737,16 @@ async function recoverRun(
           ? `Session: ${run.sessionFile}`
           : `Session: ${run.sessionFile} (file was never written — the run died before its transcript was flushed)`,
     );
+    // #807: a chain's first artifact-free cap kill is retried once on the
+    // project's stronger tier instead of reaching a human — the capability
+    // recovery the operator asked for. Everything else (no target, a second
+    // cap, a spent continuation budget) falls through and escalates exactly as
+    // it always has, saying in the page why no retry happened.
+    if (spinning) {
+      const outcome = await escalateModelTier(d, run, cls, evidence, detail);
+      if (outcome.requeued) return;
+      detail.push(outcome.note);
+    }
     // The class and the run are in the summary, which is what the notifications
     // ledger dedupes on — so one class escalates once per run rather than every
     // five minutes.
@@ -1710,6 +1770,103 @@ async function recoverRun(
   // worktree holds the only copy of real work.
 }
 
+/**
+ * The one model escalation an issue chain may spend (#807).
+ *
+ * A run that reached a ceiling with nothing to show is usually a slice that was
+ * too big — which is why the cap classes are excluded from the provider
+ * failover chain and why a second cap stays a decomposition verdict. But the
+ * first one is worth one cheap experiment: the operator configures one stronger
+ * opaque selector, and this hands the chain back to the queue to spend it,
+ * instead of paging a human to do the same thing by hand.
+ *
+ * The marker, the queue hand-back and the row's recovery stamp are one store
+ * transaction (`escalateModel`), so a daemon that dies mid-recovery restarts
+ * into one of exactly two states: nothing happened and the next sweep escalates
+ * cleanly, or all of it happened and the continuation is already owed. There is
+ * no third state where the chain reads as escalated with the continuation it
+ * promised never queued. Every refusal returns a note instead of a silent
+ * fall-through: the page that follows says why no retry happened.
+ *
+ * Deliberately independent of `modelFallbacks`: nothing here reads or advances
+ * the provider chain, and nothing there reads this marker, so a chain that
+ * mixes provider aborts with a cap kill keeps both streaks intact.
+ */
+async function escalateModelTier(
+  d: SettlementDeps,
+  run: RunRecord,
+  cls: FailureClass,
+  evidence: string,
+  detail: readonly string[],
+): Promise<{ requeued: true } | { requeued: false; note: string }> {
+  const { project, caps, tracker, store } = d;
+  const target = project.workerEscalationModel;
+  if (target === undefined) {
+    return {
+      requeued: false,
+      note:
+        "No model escalation target exists: this project sets no `workerEscalationModel`, " +
+        "so the cap reaches a human exactly as it did before.",
+    };
+  }
+  const spent = store.modelEscalation(project.name, run.issue);
+  if (spent !== undefined) {
+    return {
+      requeued: false,
+      note:
+        `This chain already spent its one model escalation (${spent.model}, bought by ${spent.failureClass}). ` +
+        "A second cap is a decomposition verdict, not a third tier — re-scope the issue.",
+    };
+  }
+  // The same two guards the wall-clock continuation applies before handing a
+  // queue label back: a closed issue was resolved by another route, and a spent
+  // continuation budget would offer a candidate admission can never accept
+  // (#490, #348).
+  const state = await tracker.issueState(run.issue).catch(() => undefined);
+  if (state !== "open") {
+    return { requeued: false, note: `Not retried on ${target}: the issue is ${state ?? "unreadable"}.` };
+  }
+  const continuations = store.continuationsFor(project.name, run.issue);
+  if (!hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
+    return {
+      requeued: false,
+      note:
+        `Not retried on ${target}: ${continuations} continuation(s) are already charged ` +
+        `against a cap of ${caps.maxContinuationsPerIssue}.`,
+    };
+  }
+  const spending = store.escalateModel({
+    project: project.name,
+    issue: run.issue,
+    model: target,
+    failureClass: cls,
+    runId: run.id,
+    swapFrom: project.stateLabels.failed,
+    queueLabel: project.queueLabel,
+  });
+  if (!spending) {
+    return {
+      requeued: false,
+      note: `Not retried on ${target}: this chain's escalation was spent concurrently.`,
+    };
+  }
+  await safeEscalate(d, {
+    tier: 1,
+    project: project.name,
+    issue: run.issue,
+    runId: run.id,
+    summary: `[${cls}] #${run.issue} attempt ${run.attempt} — retrying once on ${target}: ${evidence}`,
+    detail: [
+      ...detail,
+      "",
+      `The queue label is back on and the next dispatch runs on ${target}, this chain's one`,
+      "model escalation. Another cap after it is a decomposition verdict, not a third tier.",
+    ].join("\n"),
+  });
+  log(`#${run.issue} requeued on ${target} after ${cls}: ${evidence}`);
+  return { requeued: true };
+}
+
 /**
  * Enqueue a state-label → queue-label swap for projection (#201).
  *

package/src/setup-answers.ts

@@ -112,6 +112,103 @@ export function saveAnswersFile(path: string, answers: SetupAnswerValues): void
   writeFileSync(path, `${JSON.stringify(answers, null, 2)}\n`, "utf8");
 }
 
+/**
+ * The resumable setup transaction (#864).
+ *
+ * On 2026-08-21 `setup policy` failed at its post-interview gates several times
+ * and each retry discarded the confirmed answers: the same `runs-settled` value
+ * was typed five times while the real fault was elsewhere. The interview is the
+ * expensive half and it had already succeeded — only the apply phase failed —
+ * so the answers are saved on that failure and named in the retry command.
+ *
+ * The binding is what stops this being a stale cache. A resume file carries the
+ * hash of the config it was answered against, and the version that wrote it:
+ * replaying an old plan over newer policy is precisely the silent fake here, and
+ * refusing loudly is cheaper than a config quietly rolled back.
+ */
+export interface SetupResumeFile {
+  /** Config bytes hash at interview time; a change refuses the replay. */
+  configHash: string;
+  /** The package version that recorded it — a schema fence across upgrades. */
+  version: string;
+  /** The area/project the answers belong to, for the printed retry command. */
+  area?: string;
+  project?: string;
+  answers: SetupAnswerValues;
+}
+
+const resumeFileSchema = z.object({
+  configHash: z.string().min(1),
+  version: z.string().min(1),
+  area: z.string().optional(),
+  project: z.string().optional(),
+  answers: answerFileSchema,
+});
+
+/** The config generation a resume file is bound to: its exact bytes, or `absent`. */
+export function setupConfigHash(read: () => string | undefined): string {
+  const raw = read();
+  if (raw === undefined) return "absent";
+  return new Bun.CryptoHasher("sha256").update(raw).digest("hex").slice(0, 32);
+}
+
+/**
+ * Persist the confirmed answers of a failed apply phase, 0600 because it holds
+ * whatever the interview held — no more, and in the same account's state
+ * directory the config itself lives in.
+ */
+export function saveSetupResume(path: string, file: SetupResumeFile): void {
+  writeFileSync(path, `${JSON.stringify(file, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
+}
+
+export type SetupResumeRead =
+  | { ok: true; file: SetupResumeFile }
+  | { ok: false; problem: string };
+
+/**
+ * Read a resume file and check its binding against the live config.
+ *
+ * A hash mismatch is refused rather than merged: the answers describe a
+ * decision made about a different config, and applying them would overwrite
+ * whatever changed it. So is a version mismatch — the answer keys are a schema,
+ * and an upgraded package may have renamed one.
+ */
+export function readSetupResume(path: string, liveConfigHash: string, version: string): SetupResumeRead {
+  let raw: string;
+  try {
+    raw = readFileSync(path, "utf8");
+  } catch (err) {
+    return { ok: false, problem: `could not read resume file "${path}": ${err instanceof Error ? err.message : String(err)}` };
+  }
+  let decoded: unknown;
+  try {
+    decoded = JSON.parse(raw);
+  } catch (err) {
+    return { ok: false, problem: `resume file "${path}" is not valid JSON: ${err instanceof Error ? err.message : String(err)}` };
+  }
+  const parsed = resumeFileSchema.safeParse(decoded);
+  if (!parsed.success) {
+    return { ok: false, problem: `resume file "${path}" is not a setup resume record` };
+  }
+  if (parsed.data.version !== version) {
+    return {
+      ok: false,
+      problem:
+        `resume file "${path}" was written by omp-conductor ${parsed.data.version}, this is ${version} — ` +
+        "the answer keys are a schema, so re-run the interview rather than replaying answers across an upgrade",
+    };
+  }
+  if (parsed.data.configHash !== liveConfigHash) {
+    return {
+      ok: false,
+      problem:
+        `resume file "${path}" was answered against a different config (recorded ${parsed.data.configHash}, ` +
+        `live ${liveConfigHash}) — replaying it would overwrite whatever changed it. Re-run the interview`,
+    };
+  }
+  return { ok: true, file: parsed.data };
+}
+
 /** Turns exhausted scripted stdin into an actionable non-interactive failure. */
 export function guardNonInteractiveUi(inner: TerminalUi): TerminalUi {
   const exhausted = (title: string, options: PromptOptions): never => {