omp-conductor 0.4.2 → 0.4.3

package/src/daemon.ts

@@ -37,6 +37,8 @@ import { createReportOutbox, formatOpenReports } from "./reports.ts";
 import { recordReleaseBlock } from "./release-policy.ts";
 import { branchName, route } from "./routing.ts";
 import type { Routed, UnroutableReason } from "./routing.ts";
+import { evaluateDecisionConditions, probeNpmVersion } from "./decisions.ts";
+import { classifyRun, type ClassifyFacts } from "./failure-class.ts";
 import { dbPath, openStore } from "./store.ts";
 import { makeTracker } from "./tracker/github.ts";
 import { RELEASE_SHAPES } from "./types.ts";
@@ -53,6 +55,8 @@ import type {
   RepoTarget,
   ReportRecord,
   ResolvedGrants,
+  FailureClass,
+  RecoveryAction,
   RunRecord,
   RunState,
   SettlementFlag,
@@ -2191,6 +2195,46 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
   // through the pool without blocking this five-minute tick.
   await settlePushedGreen(d);
 
+  // Immediately after settlement and before any routing, so a class is on the
+  // row before the next dispatch decision reads its budgets (#132). Above the
+  // pause gate deliberately: classification and label reconciliation are
+  // maintenance, and the four phantom `agent:failed` issues this exists to clear
+  // are exactly as misleading on a parked fleet as on a busy one.
+  //
+  // Each is guarded on its own: a tracker that fails mid-classification must not
+  // stop the label reconcile, and neither may stop the tick.
+  try {
+    await classifyAndRecover(d);
+  } catch (err) {
+    log(`classification sweep failed: ${errText(err)}`);
+  }
+  try {
+    await reconcileStaleLabels(d);
+  } catch (err) {
+    log(`label reconcile failed: ${errText(err)}`);
+  }
+
+  // Ledger maintenance, above the pause gate for the same reason the stall watch
+  // is: a paused fleet still owes its operator the questions it asked, and a
+  // condition that came true while dispatch was parked is exactly the thing the
+  // orchestrator has to see on its next tick.
+  //
+  // Expiry is synchronous (one UPDATE); condition evaluation is fire-and-forget
+  // because it shells out to `gh` and `npm`, and a registry that hangs must cost
+  // one unevaluated condition rather than the tick.
+  for (const expired of d.store.expireDueDecisions(d.project.name, Date.now())) {
+    log(`decision ${expired.id} expired unanswered after seven days: ${expired.question}`);
+  }
+  void evaluateDecisionConditions(d.store, d.project.name, d.tracker, probeNpmVersion, Date.now)
+    .then((met) => {
+      for (const decision of met) {
+        log(`decision ${decision.id} condition met (${decision.condition ?? "?"}) — surfacing on the next tick`);
+      }
+    })
+    .catch((err: unknown) => {
+      log(`decision condition pass failed: ${errText(err)}`);
+    });
+
   // A paused fleet claims nothing. Checked first so pausing takes effect on the
   // next tick without signalling the process.
   if (isPaused()) return;
@@ -2746,6 +2790,299 @@ export function prepareConductor(): void {
   setPaused(true);
 }
 
+/** Bounded per tick: each row costs tracker calls to gather facts for. */
+const CLASSIFY_BATCH = 20;
+
+/** Tool calls quoted as evidence for a run that spun to its turn cap. */
+const SPIN_EVIDENCE_CALLS = 10;
+
+/**
+ * The last few tool names a transcript recorded, newest last.
+ *
+ * `turn-cap-spinning` escalates rather than requeueing, and the acceptance
+ * criterion is that the escalation carries evidence of what the worker was doing
+ * when it hit the cap — otherwise the orchestrator opens the transcript and
+ * re-derives it, which is the manual triage this whole sweep removes.
+ */
+export function lastToolCalls(sessionFile: string | undefined, limit = SPIN_EVIDENCE_CALLS): string[] {
+  if (sessionFile === undefined) return [];
+  let text: string;
+  try {
+    text = readFileSync(sessionFile, "utf8");
+  } catch {
+    return [];
+  }
+  const names: string[] = [];
+  for (const line of text.split("\n")) {
+    if (line.length === 0) continue;
+    let row: unknown;
+    try {
+      row = JSON.parse(line) as unknown;
+    } catch {
+      continue;
+    }
+    if (row === null || typeof row !== "object") continue;
+    const rec = row as { readonly [key: string]: unknown };
+    // Both shapes the harness has written: a top-level tool event, and a tool
+    // block inside an assistant message.
+    const direct = rec["toolName"];
+    if (typeof direct === "string") {
+      names.push(direct);
+      continue;
+    }
+    const message = rec["message"];
+    if (message === null || typeof message !== "object") continue;
+    const content = (message as { readonly [key: string]: unknown })["content"];
+    if (!Array.isArray(content)) continue;
+    for (const part of content) {
+      if (part === null || typeof part !== "object") continue;
+      const p = part as { readonly [key: string]: unknown };
+      if (p["type"] !== "tool_use") continue;
+      const name = p["name"];
+      if (typeof name === "string") names.push(name);
+    }
+  }
+  return names.slice(-limit);
+}
+
+/**
+ * Classify every unclassified terminal run, persist the verdict, and perform the
+ * one recovery its class names (#132).
+ *
+ * Half this fleet's spend produced no merged PR, and every one of those runs
+ * ended at a human who re-derived the same triage by hand and then threw the
+ * conclusion away. The mechanical classes — a cancelled runner, a kill from a
+ * daemon restart, a green PR whose base moved, a row whose PR had already merged
+ * — need no judgement at all; the genuinely human ones are worth a person's
+ * attention only if they arrive with their evidence already gathered.
+ *
+ * Facts are fetched per row and only the ones that row needs: a `killed` row
+ * costs nothing, a `failed` row with a PR costs a state read and a check read.
+ * A `pushed-green` row that classifies to nothing is left completely untouched —
+ * it is healthy, and writing a class onto it would take it out of this sweep for
+ * good.
+ */
+export async function classifyAndRecover(d: Deps): Promise<void> {
+  const { project, tracker, store } = d;
+  for (const run of store.runsNeedingClassification(project.name, CLASSIFY_BATCH)) {
+    const facts: ClassifyFacts = {};
+    try {
+      if (run.prUrl !== undefined) {
+        const pr = await tracker.prState(run.prUrl);
+        if (pr !== undefined) facts.pr = pr;
+        if (run.state === "pushed-green" && facts.pr === "open") {
+          facts.mergeable = await tracker.mergeable(run.prUrl);
+        }
+        if (run.state === "failed" && facts.pr === "open") {
+          facts.checks = await tracker.checkConclusions(run.prUrl);
+        }
+      }
+    } catch (err) {
+      // Per row, like every other sweep here: one unreachable PR must not stop
+      // the rest from being classified. The next tick asks again for free.
+      log(`#${run.issue} not classified: fact gathering failed (${errText(err)}) — retrying next tick`);
+      continue;
+    }
+
+    const { cls, recovery, evidence } = classifyRun(run, facts);
+
+    // A healthy green PR is not a failure of any class. Leaving the row
+    // unclassified is what keeps it eligible for the sweep on the tick where its
+    // base does move under it.
+    if (run.state === "pushed-green" && cls === "unknown") continue;
+
+    store.updateRun(run.id, { failureClass: cls, recoveryAction: recovery });
+    log(`#${run.issue} classified ${cls} → ${recovery}: ${evidence}`);
+    await recoverRun(d, run, cls, recovery, evidence);
+  }
+}
+
+/** Performs the one action a class names. Never chooses one of its own. */
+async function recoverRun(
+  d: Deps,
+  run: RunRecord,
+  cls: FailureClass,
+  recovery: RecoveryAction,
+  evidence: string,
+): Promise<void> {
+  const { project, tracker, store } = d;
+  const inProgress = project.stateLabels.inProgress;
+
+  if (recovery === "settle") {
+    // Label before row, copied from `settlePushedGreen` where the order is the
+    // whole safety argument: writing the terminal state first would put this row
+    // beyond every later tick, and a tracker that then failed on the label would
+    // strand `agent:in-progress` with nothing left to retry it (#18).
+    if (!(await releaseInProgress(d, run.issue, `PR merged: ${evidence}`))) return;
+    store.updateRun(run.id, { state: "merged", endedAt: Date.now(), recoveredAt: Date.now() });
+    log(`#${run.issue} settled from ${cls}: ${evidence}`);
+    return;
+  }
+
+  if (recovery === "continue") {
+    // The branch is retained and its PR is open, so #50's continuation guard
+    // admits it: the next tick reattaches the branch and briefs a rebase.
+    store.updateRun(run.id, {
+      state: "killed",
+      lastError:
+        "merge-conflict: base moved under a green PR — rebase, regenerate recorded artifacts, push without force",
+      recoveredAt: Date.now(),
+    });
+    if (!(await swapToQueue(d, run.issue, inProgress))) return;
+    log(`#${run.issue} requeued for a rebase continuation: ${evidence}`);
+    return;
+  }
+
+  if (recovery === "requeue") {
+    // Only when the tracker still shows this issue as ours to hand back. An
+    // issue that is closed, or has no state label, was resolved by another route
+    // and requeueing it would dispatch work nobody asked for.
+    const state = await tracker.issueState(run.issue).catch(() => undefined);
+    if (state !== "open") {
+      log(`#${run.issue} not requeued from ${cls}: issue is ${state ?? "unreadable"}`);
+      return;
+    }
+    const label = cls === "orphan-clean" ? inProgress : project.stateLabels.failed;
+    if (!(await swapToQueue(d, run.issue, label))) return;
+    store.updateRun(run.id, { recoveredAt: Date.now() });
+    log(`#${run.issue} requeued from ${cls}: ${evidence}`);
+    return;
+  }
+
+  if (recovery === "rerun-checks") {
+    if (run.prUrl === undefined) return;
+    try {
+      await tracker.rerunFailedChecks(run.prUrl);
+    } catch (err) {
+      log(`#${run.issue} check re-run failed (${errText(err)}) — retrying next tick`);
+      return;
+    }
+    // Back to pending rather than green: the existing settle sweep re-verifies
+    // it against the recorded head on a later tick, so nothing here has to guess
+    // whether the re-run passed.
+    store.updateRun(run.id, { state: "pushed-pending", lastError: undefined, recoveredAt: Date.now() });
+    log(`#${run.issue} re-ran infrastructure checks: ${evidence}`);
+    return;
+  }
+
+  if (recovery === "escalate") {
+    const detail = [evidence];
+    if (cls === "turn-cap-spinning") {
+      const calls = lastToolCalls(run.sessionFile);
+      detail.push(
+        calls.length === 0
+          ? "transcript unreadable — no tool calls could be recovered"
+          : `Last ${calls.length} tool calls: ${calls.join(" → ")}`,
+      );
+    }
+    if (run.lastError !== undefined && cls !== "question") detail.push(run.lastError);
+    detail.push(`Session: ${run.sessionFile ?? "(no transcript)"}`);
+    // The class and the run are in the summary, which is what the notifications
+    // ledger dedupes on — so one class escalates once per run rather than every
+    // five minutes.
+    await safeEscalate(d, {
+      tier: 1,
+      project: project.name,
+      issue: run.issue,
+      runId: run.id,
+      summary: `[${cls}] #${run.issue} attempt ${run.attempt}: ${evidence}`,
+      detail: detail.join("\n"),
+    });
+    // The hand-off IS the recovery for these classes: there is nothing else this
+    // package can do, and leaving the row unrecovered would re-escalate forever.
+    store.updateRun(run.id, { recoveredAt: Date.now() });
+    return;
+  }
+
+  // `hold` (orphan-dirty) and `none`: recorded, nothing performed. The existing
+  // unsalvaged-WIP admission hold already fails dispatch closed until an
+  // operator acknowledges the tree, which is the only safe move when the
+  // worktree holds the only copy of real work.
+}
+
+/**
+ * Swap a state label for the queue label, in that order.
+ *
+ * Both writes go through the same Tracker port the dispatcher claimed with, so
+ * orphan detection stays trustworthy. Returns false when either half failed, so
+ * the caller leaves the row for the next tick rather than recording a recovery
+ * that did not happen.
+ */
+async function swapToQueue(
+  d: Pick<Deps, "project" | "tracker">,
+  issue: number,
+  label: string,
+): Promise<boolean> {
+  try {
+    await d.tracker.removeLabel(issue, label);
+    await d.tracker.addLabel(issue, d.project.queueLabel);
+    return true;
+  } catch (err) {
+    log(`#${issue} could not be requeued (${errText(err)}) — retrying next tick`);
+    return false;
+  }
+}
+
+/** Bounded listing per state label, so one reconcile cannot walk a whole repo. */
+const RECONCILE_LIMIT = 50;
+
+/**
+ * Clear state labels from issues that no longer need them (#132's `superseded`).
+ *
+ * Two structural signals, both cheap and both observed on this fleet: an issue
+ * that is closed but still carries `agent:*`, and an open issue carrying
+ * `failed` whose sub-issues have all closed. On 2026-08-09 four issues (#307,
+ * #297, #140, #82) carried `agent:failed` while every one of them was already
+ * complete — the label was residue of a turns-cap kill from two days earlier,
+ * and nothing in the loop ever revisited it. The board counted four phantom
+ * failures while the genuinely stuck issues were invisible.
+ *
+ * Positive evidence only. A tracker that cannot list answers empty, and an empty
+ * answer removes nothing: a reconcile that guessed would strip the interlock
+ * that keeps two workers off one issue.
+ */
+export async function reconcileStaleLabels(d: Deps): Promise<void> {
+  const { project, tracker, store } = d;
+  const labels = [project.stateLabels.failed, project.stateLabels.blocked, project.stateLabels.inProgress];
+
+  for (const label of labels) {
+    const carrying = await tracker.listLabeled(label, RECONCILE_LIMIT).catch(() => []);
+    for (const issue of carrying) {
+      if (issue.state === "closed") {
+        // Never retain an `agent:*` label on a closed issue: the work is done by
+        // some route, and the label only makes the board lie about it.
+        try {
+          await tracker.removeLabel(issue.number, label);
+          log(`#${issue.number} reconciled: closed issue no longer carries ${label}`);
+        } catch (err) {
+          log(`#${issue.number} could not drop ${label} (${errText(err)}) — retrying next tick`);
+        }
+        continue;
+      }
+
+      if (label !== project.stateLabels.failed) continue;
+      const children = await tracker.childrenOf(issue.number).catch(() => []);
+      if (children.length === 0 || children.some((c) => c.state !== "closed")) continue;
+
+      const key = `${project.name}:superseded:${issue.number}`;
+      if (store.wasNotified(key)) continue;
+      const list = children.map((c) => `#${c.number}`).join(", ");
+      try {
+        await tracker.removeLabel(issue.number, label);
+        await tracker.comment(
+          issue.number,
+          `superseded: all sub-issues closed (${list}) — this label was stale; propose closing if the ` +
+            `acceptance criteria are met on the default branch.`,
+        );
+        store.markNotified(key);
+        log(`#${issue.number} reconciled: superseded by ${list}`);
+      } catch (err) {
+        log(`#${issue.number} could not be reconciled (${errText(err)}) — retrying next tick`);
+      }
+    }
+  }
+}
+
 /**
  * Settles `claimed`/`running` rows left by a dead daemon process and, before
  * marking each one `orphaned`, salvages any dirty worktree.
@@ -2957,9 +3294,13 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
       // Its own principal, distinct from every worker slot. The orchestrator
       // reads the state directory and its briefs and must have no read or
       // write access to any run checkout — so `workspaceRoot` and the mirror
-      // are denied outright here, on top of the #127 tool-layer jail. Two
-      // layers because they fail differently: the jail refuses a structured
-      // tool call, the principal refuses the syscall `bash` would make.
+      // are denied outright here. This is the ONLY mechanical file gate an
+      // orchestrator gets: the tool-layer jail was deleted in 0.4.3 (#143)
+      // because it could only be installed in sessions this daemon spawns, and
+      // the principal refuses the syscall `bash` would make anyway, which the
+      // jail never did. It is also conditional, and honestly so: on a host with
+      // no isolating mechanism there is no principal and no gate, which is what
+      // `status` reports as `unprotected`.
       // Same rule as a worker's, for the same reason: under `uid-pool` the
       // orchestrator is its own uid, so every tree it must reach lives outside
       // the 0700 private state directory. Its principal is in `conductor-runs`

package/src/decisions.ts

@@ -0,0 +1,164 @@
+/**
+ * The pending-decision ledger's grammar, evaluation and digest (#136).
+ *
+ * A question the orchestrator sends its operator used to live in exactly one
+ * place: the model's context. A compaction, a restart, or a tick that ran long
+ * lost the question *and* the fact that one was owed — after which the session
+ * either asked again (the operator answers twice) or dropped it silently (the
+ * decision never lands, and nothing says a decision is outstanding).
+ *
+ * The store holds the rows. This module holds the two things around them that
+ * are not storage: what a machine-checkable precondition looks like, and how an
+ * open row is rendered into a tick prompt.
+ */
+
+import type { DecisionRecord, Store, Tracker } from "./types.ts";
+
+/**
+ * A precondition whose truth this package can check on its own.
+ *
+ * Exactly three kinds, deliberately. Each one is a question the tracker or npm
+ * already answers, so the row moves from "parked" to "act on this" without a
+ * human re-reading it. Anything richer — a label appearing, a workflow going
+ * green — is a follow-on issue rather than a grammar nobody validated.
+ */
+export type DecisionCondition =
+  | { kind: "pr-merged"; url: string }
+  | { kind: "issue-closed"; issue: number }
+  | { kind: "npm-version"; spec: string };
+
+/** `pkg@version`, including a scoped package (`@scope/pkg@1.2.3`). */
+const NPM_SPEC = /^(@[^/@\s]+\/)?[^@\s]+@[^\s]+$/;
+
+/**
+ * Parse a raw condition, or `undefined` when it is not one of the three forms.
+ *
+ * Parsing on read rather than on write is what keeps a row this build does not
+ * understand *listable*: the raw string is stored verbatim, so a future grammar
+ * does not make an old row unloadable, and an unparseable one simply never
+ * reports itself met.
+ */
+export function parseCondition(raw: string): DecisionCondition | undefined {
+  const text = raw.trim();
+  const at = text.indexOf(":");
+  if (at <= 0) return undefined;
+  const kind = text.slice(0, at);
+  const rest = text.slice(at + 1).trim();
+  if (rest.length === 0) return undefined;
+
+  if (kind === "pr-merged") {
+    return rest.startsWith("https://") ? { kind: "pr-merged", url: rest } : undefined;
+  }
+  if (kind === "issue-closed") {
+    if (!/^\d+$/.test(rest)) return undefined;
+    const issue = Number.parseInt(rest, 10);
+    return issue > 0 ? { kind: "issue-closed", issue } : undefined;
+  }
+  if (kind === "npm-version") {
+    return NPM_SPEC.test(rest) ? { kind: "npm-version", spec: rest } : undefined;
+  }
+  return undefined;
+}
+
+/** The three accepted forms, for a refusal that can be acted on in one turn. */
+export const CONDITION_FORMS = [
+  "pr-merged:https://github.com/owner/repo/pull/123",
+  "issue-closed:123",
+  "npm-version:omp-conductor@0.4.3",
+] as const;
+
+/** Probe for `npm-version`, injectable so tests never reach the network. */
+export type NpmProbe = (spec: string) => Promise<boolean>;
+
+/**
+ * `npm view <spec> version` — exit 0 with non-empty output means the version is
+ * published. Bounded at 10 s because this runs inside a tick: a registry that
+ * hangs must cost one unevaluated condition, not the tick.
+ */
+export const probeNpmVersion: NpmProbe = async (spec) => {
+  const proc = Bun.spawn(["npm", "view", spec, "version"], { stdout: "pipe", stderr: "ignore" });
+  const timer = setTimeout(() => {
+    proc.kill();
+  }, 10_000);
+  try {
+    const [text, code] = await Promise.all([new Response(proc.stdout).text(), proc.exited]);
+    return code === 0 && text.trim().length > 0;
+  } catch {
+    return false;
+  } finally {
+    clearTimeout(timer);
+  }
+};
+
+/**
+ * Check every open decision that carries a condition and has not met it yet.
+ *
+ * Errors are swallowed per row on purpose: this runs fire-and-forget beside a
+ * tick, and one deleted PR or one flaky `gh` call must not stop the rest of the
+ * pass. A condition that could not be checked is simply not met this time, and
+ * the next tick asks again for free.
+ *
+ * Returns the rows that just became met, so the caller can log what changed
+ * rather than a count.
+ */
+export async function evaluateDecisionConditions(
+  store: Store,
+  project: string,
+  tracker: Tracker,
+  probeNpm: NpmProbe,
+  now: () => number,
+): Promise<DecisionRecord[]> {
+  const met: DecisionRecord[] = [];
+  for (const decision of store.openDecisions(project)) {
+    if (decision.condition === undefined || decision.conditionMetAt !== undefined) continue;
+    const condition = parseCondition(decision.condition);
+    if (condition === undefined) continue;
+    let satisfied = false;
+    try {
+      if (condition.kind === "pr-merged") {
+        satisfied = (await tracker.prState(condition.url)) === "merged";
+      } else if (condition.kind === "issue-closed") {
+        satisfied = (await tracker.issueState(condition.issue)) === "closed";
+      } else {
+        satisfied = await probeNpm(condition.spec);
+      }
+    } catch {
+      continue;
+    }
+    if (!satisfied) continue;
+    if (store.markDecisionConditionMet(decision.id, now())) {
+      met.push({ ...decision, conditionMetAt: now() });
+    }
+  }
+  return met;
+}
+
+/** Whole hours, then days — the granularity an operator reads a digest at. */
+function age(since: number, now: number): string {
+  const minutes = Math.max(0, Math.round((now - since) / 60_000));
+  if (minutes < 60) return `${minutes}m`;
+  const hours = Math.round(minutes / 60);
+  return hours < 48 ? `${hours}h` : `${Math.round(hours / 24)}d`;
+}
+
+/**
+ * The open-decision block appended to a tick prompt, or `""` when nothing is
+ * owed.
+ *
+ * This is the half that makes the ledger worth having: the session is told what
+ * it still owes on every tick, from the store, so "what did I ask them?" stops
+ * being a memory question. A row whose condition has come true is flagged, not
+ * merely listed — that is the difference between a parked question and one to
+ * act on now.
+ */
+export function formatDecisionDigest(open: readonly DecisionRecord[], now = Date.now()): string {
+  if (open.length === 0) return "";
+  const lines = [
+    `Open operator decisions (${open.length}) — resolve or withdraw each with omp-conductor decision resolve|withdraw <id>:`,
+  ];
+  for (const d of open) {
+    const flag = d.conditionMetAt === undefined ? "" : " [CONDITION MET — act on this now]";
+    lines.push(`- ${d.id} (${age(d.askedAt, now)}, blocks ${d.blocks ?? "nothing"})${flag}: ${d.question}`);
+  }
+  return lines.join("\n");
+}