omp-conductor 0.19.2 → 0.19.4

package/src/daemon.ts

@@ -38,7 +38,12 @@ import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-
 import { graphHint } from "./graph.ts";
 import { hostConstraintsNotice } from "./host.ts";
 import { acquireOnceLease, healthCheck, livingDaemon, probeUnit, SYSTEMD_UNIT } from "./lifecycle.ts";
-import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
+import {
+  readTickRequestReason,
+  requestImmediateTick,
+  resolveTickConfigCwd,
+  STALL_MARKER_FILE,
+} from "./orchestrator-tick.ts";
 import { SPEND_SAMPLE_ROWS, SPEND_SAMPLE_RUNS, runDoctor } from "./doctor.ts";
 import { judgeSpendTelemetry, type SpendTelemetryVerdict } from "./spend-telemetry.ts";
 import {
@@ -62,6 +67,7 @@ import {
   herdrPaneOmpStarts,
   openWorkerPane,
   reconcileWorkerPanes,
+  PANE_REATTEMPT_MAX,
   releaseOrphanedWorkerPane,
   releaseWorkerPane,
   reportWorkerPaneState,
@@ -110,11 +116,13 @@ import {
 import {
   adoptSalvagedPrs,
   classifyAndRecover,
+  harnessLogDir,
   collectSettlementFlags,
   formatQuarantinedRuns,
   formatSalvagedRuns,
   reactToProviderCredit,
   readSessionError,
+  readTerminalEvidence,
   reconcileOrphanedRuns,
   reconcileGroomingClosures,
   reconcileStaleLabels,
@@ -127,6 +135,7 @@ import { materializeOmpSettings } from "./omp-settings.ts";
 import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
 import {
   infraLogSignature,
+  classifyRun,
   infraSignatureVersion,
   providerCreditRefusal,
   providerTransientFault,
@@ -327,6 +336,8 @@ interface Deps {
   workerControls: WorkerControlRegistry;
   /** Session seam for lifecycle integration tests; production uses the real harness. */
   workerDeps?: RunWorkerDeps;
+  /** Harness per-process logs used to classify terminal worker routes. */
+  harnessLogDir?: string;
   integrity: IntegrityGate;
   stall: StallGate;
   /**
@@ -1424,11 +1435,20 @@ export interface WorkerControlRegistry {
     runId: string,
     onPhase?: (phase: WorkerPausePhase) => void,
   ): WorkerControlSlot;
-  pause(project: string, issue: number): Promise<WorkerControlResult>;
+  /** `source` names who asked (board/cli/dashboard) — recorded as pause
+   *  provenance so "who paused this and when" is answerable later (#997). */
+  pause(project: string, issue: number, source: string): Promise<WorkerControlResult>;
   resume(project: string, issue: number): WorkerControlResult;
   stop(project: string, issue: number, reason: string): Promise<WorkerControlResult>;
-  /** Live runs whose phase is not `running` — what /healthz and the board show. */
-  snapshot(project: string): { issue: number; runId: string; phase: WorkerPausePhase }[];
+  /** Live runs whose phase is not `running` — what /healthz and the board
+   *  show — each carrying its pause provenance when one was recorded (#997). */
+  snapshot(project: string): {
+    issue: number;
+    runId: string;
+    phase: WorkerPausePhase;
+    source?: string;
+    pausedAtMs?: number;
+  }[];
 }
 
 /** Authoritative controls for sessions owned by this daemon process. */
@@ -1442,6 +1462,8 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
     stopError?: string;
     finished: PromiseWithResolvers<void>;
     onPhase?: (phase: WorkerPausePhase) => void;
+    /** Who asked for the live pause, and when — cleared on resume (#997). */
+    pausedBy?: { source: string; at: number };
   }
 
   const active = new Map<string, Entry>();
@@ -1476,12 +1498,13 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
         },
       };
     },
-    async pause(project, issue) {
+    async pause(project, issue, source) {
       const entry = active.get(key(project, issue));
       if (entry?.control === undefined) return { kind: "not-active" };
       try {
         await entry.control.pause();
         const phase = entry.control.phase();
+        entry.pausedBy = { source, at: Date.now() };
         entry.onPhase?.(phase);
         return { kind: "ok", runId: entry.runId, phase };
       } catch (err) {
@@ -1498,6 +1521,7 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
       try {
         entry.control.resume();
         const phase = entry.control.phase();
+        delete entry.pausedBy;
         entry.onPhase?.(phase);
         return { kind: "ok", runId: entry.runId, phase };
       } catch (err) {
@@ -1533,11 +1557,25 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
       return { kind: "stopped", runId: entry.runId, reason: entry.stopReason! };
     },
     snapshot(project) {
-      const workers: { issue: number; runId: string; phase: WorkerPausePhase }[] = [];
+      const workers: {
+        issue: number;
+        runId: string;
+        phase: WorkerPausePhase;
+        source?: string;
+        pausedAtMs?: number;
+      }[] = [];
       for (const entry of active.values()) {
         if (entry.project !== project || entry.control === undefined) continue;
         const phase = entry.control.phase();
-        if (phase !== "running") workers.push({ issue: entry.issue, runId: entry.runId, phase });
+        if (phase === "running") continue;
+        workers.push({
+          issue: entry.issue,
+          runId: entry.runId,
+          phase,
+          ...(entry.pausedBy === undefined
+            ? {}
+            : { source: entry.pausedBy.source, pausedAtMs: entry.pausedBy.at }),
+        });
       }
       return workers;
     },
@@ -2804,6 +2842,7 @@ export async function handleIssue(
       log(`#${issue} stopped by operator on attempt ${attempt}: ${result.stoppedReason}`);
     } else if (state === "blocked") {
       swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
+      wakeOrchestratorForBlockedRun(project.name, issue);
       await safeEscalate(d, {
         tier: 1,
         project: project.name,
@@ -3035,8 +3074,27 @@ export async function handleIssue(
  * releasing one whose run is not live; the authoritative child is never
  * signalled, and no pane is ever closed.
  */
+/**
+ * Pane re-establishment attempts spent per run, for this daemon process only
+ * (#998).
+ *
+ * Deliberately in memory rather than on the run row: a restart is exactly the
+ * event that makes another attempt worth making (Herdr came back), so a fresh
+ * process legitimately gets a fresh budget, and giving up never marks a run
+ * permanently unrepresentable. Pruned against the live set on every pass, so it
+ * cannot outgrow the fleet.
+ */
+const paneAttemptsSpent = new Map<string, number>();
+
 function reconcilePanes(d: Deps, project: string, log: (message: string) => void): void {
   const live = d.store.liveRuns(project);
+  const liveKeys = new Set(live.map((run) => `${project}\u0000${run.id}`));
+  for (const key of [...paneAttemptsSpent.keys()]) {
+    if (key.startsWith(`${project}\u0000`) && !liveKeys.has(key)) paneAttemptsSpent.delete(key);
+  }
+  const attempts = new Map(
+    live.map((run) => [run.id, paneAttemptsSpent.get(`${project}\u0000${run.id}`) ?? 0]),
+  );
   const result = reconcileWorkerPanes(
     live.map((run) => ({
       runId: run.id,
@@ -3048,6 +3106,8 @@ function reconcilePanes(d: Deps, project: string, log: (message: string) => void
       ...(run.paneLabel === undefined ? {} : { paneLabel: run.paneLabel }),
       ...(run.sessionFile === undefined ? {} : { sessionFile: run.sessionFile }),
     })),
+    {},
+    attempts,
   );
   if (!result.ok) {
     // An unreadable workspace is not evidence that anything is stale, so nothing
@@ -3068,9 +3128,33 @@ function reconcilePanes(d: Deps, project: string, log: (message: string) => void
         });
         log(`#${runIssue(live, outcome.runId)} herdr pane re-associated as ${outcome.paneId} after a restart`);
         break;
-      case "untracked":
+      case "untracked": {
         d.store.updateRun(outcome.runId, { paneId: null, paneLabel: null, paneUnavailable: outcome.reason });
-        log(`#${runIssue(live, outcome.runId)} has no herdr pane: ${outcome.reason}`);
+        if (outcome.attempted !== true) {
+          // Cost no attempt (no recorded pid): say so every pass, as before.
+          log(`#${runIssue(live, outcome.runId)} has no herdr pane: ${outcome.reason}`);
+          break;
+        }
+        // #998: a terminal refusing to split will refuse the next one too, and
+        // each retry made the pile worse — spend the whole budget at once.
+        const key = `${project}\u0000${outcome.runId}`;
+        const spent = outcome.reason.includes("pane_split_failed")
+          ? PANE_REATTEMPT_MAX
+          : (paneAttemptsSpent.get(key) ?? 0) + 1;
+        paneAttemptsSpent.set(key, spent);
+        // Once per run, at the moment the budget runs out — not once per pass.
+        // Four identical lines a minute for #986 is what made the real cause
+        // (a leaked pane per attempt) hard to see on 2026-08-23.
+        log(
+          spent >= PANE_REATTEMPT_MAX
+            ? `#${runIssue(live, outcome.runId)} has no herdr pane after ${spent} attempt(s), not retrying: ${outcome.reason}`
+            : `#${runIssue(live, outcome.runId)} has no herdr pane (attempt ${spent}/${PANE_REATTEMPT_MAX}): ${outcome.reason}`,
+        );
+        break;
+      }
+      case "attempts-exhausted":
+        // Already reported when the budget ran out; the run keeps working with
+        // no pane, and a later restart gets a fresh budget.
         break;
       case "stale-released":
         log(`released stale herdr pane ${outcome.paneId} (run ${outcome.runId || "unidentified"} is not live)`);
@@ -3880,7 +3964,10 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
     if (await settleStopBeforeSession()) return;
     if (await settleDrainBeforeSession()) return;
 
-    store.updateRun(runId, { worktree: worktreePath });
+    // A review revision is a new worker process on the same run row. Clear the
+    // prior process evidence before launch so a failed wake cannot inherit an
+    // earlier round's harness route.
+    store.updateRun(runId, { worktree: worktreePath, workerPid: null, terminalEvidence: null });
 
     // The row's own counters are cumulative across the revision: the revision
     // worker meters its own session from zero, so its deltas are added to the
@@ -3893,6 +3980,7 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
     log(`#${issue} review round ${revision.round} → resuming ${branch} (${priorSessionFile})`);
 
 
+    let workerStartedAt: number | undefined;
     let result: WorkerResult;
     try {
       const runAllowanceUsd = runSpendAllowanceUsd(caps);
@@ -3918,7 +4006,9 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
         socketPath: join(sessionDir, "ipc.sock"),
         verbSocketPath: verbListener.path,
         onSpawn: (pid) => {
+          workerStartedAt = Date.now();
           verbListener?.bindPid(pid);
+          store.updateRun(runId, { workerPid: pid });
         },
         onChildLog: (line) => {
           log(`#${issue} ${line}`);
@@ -4013,6 +4103,18 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
       ...(verified.reason === undefined ? [] : ["", verified.reason]),
       result.report,
     ].join("\n");
+    const latestRun = store.getRun(runId) ?? run;
+    const terminalEvidence =
+      state === "failed" || state === "killed"
+        ? readTerminalEvidence(
+            {
+              sessionFile: result.sessionFile,
+              startedAt: workerStartedAt ?? latestRun.startedAt,
+              ...(latestRun.workerPid === undefined ? {} : { workerPid: latestRun.workerPid }),
+            },
+            d.harnessLogDir ?? harnessLogDir(),
+          )
+        : undefined;
 
     const terminalPatch: Partial<RunRecord> = {
       endedAt: Date.now(),
@@ -4033,9 +4135,76 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
       ...(result.headSha === undefined ? {} : { headSha: result.headSha }),
       sessionFile: result.sessionFile,
       ...(result.graphTools === undefined ? {} : { graphTools: result.graphTools }),
+      ...(terminalEvidence === undefined ? {} : { terminalEvidence }),
       report: finalReport,
       ...settlement?.patch,
     };
+    const terminalClassification =
+      terminalEvidence === undefined
+        ? undefined
+        : classifyRun(
+            {
+              ...latestRun,
+              ...terminalPatch,
+              state,
+              terminalEvidence,
+            },
+            {},
+            caps,
+          );
+    if (terminalClassification?.cls === "model-empty-stop") {
+      const recoveredAt = Date.now();
+      store.updateRun(runId, {
+        ...terminalPatch,
+        state: "failed",
+        failureClass: terminalClassification.cls,
+        recoveryAction: terminalClassification.recovery,
+        recoveredAt,
+      });
+      const continuations = store.continuationsFor(project.name, issue);
+      if (hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
+        // Keep the same review row, round, findings, session, branch and PR.
+        // Restore the exact pushed-green origin state while moving this stop
+        // into the explicit continuation counter; the next claim must not
+        // depend on re-verifying a failed-state origin at an unchanged head.
+        const chargedRun = store.getRun(runId) ?? latestRun;
+        store.updateRun(runId, {
+          ...terminalPatch,
+          state: "pushed-green",
+          failureClass: null,
+          recoveryAction: null,
+          recoveredAt: null,
+          continuationCharges: (chargedRun.continuationCharges ?? 0) + 1,
+          workerPid: null,
+          lastError: terminalClassification.evidence,
+        });
+        store.requeueReviewRevision(revision.id);
+        log(
+          `#${issue} review round ${revision.round} re-queued after a provider empty-stop ` +
+            `(${continuations}/${caps.maxContinuationsPerIssue} continuations): ${terminalClassification.evidence}`,
+        );
+        return;
+      }
+
+      store.settleReviewRevision(revision.id, "failed", recoveredAt);
+      swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
+      const model = result.model ?? latestRun.resolvedModel ?? latestRun.model ?? "configured model";
+      const provider = result.provider ?? latestRun.resolvedProvider ?? "provider";
+      await safeEscalate(d, {
+        tier: 1,
+        project: project.name,
+        issue,
+        runId,
+        summary: `#${issue} review round ${revision.round}: the model kept returning empty responses`,
+        detail:
+          `${provider}/${model} ended ${continuations} review session(s) with empty responses until ` +
+          `the harness retry cap. The same review round was preserved while continuation budget remained; ` +
+          `that budget is now exhausted at ${caps.maxContinuationsPerIssue}. No implementation failure or ` +
+          `additional review round was charged.\n\n${terminalClassification.evidence}`,
+      });
+      log(`#${issue} review round ${revision.round} provider empty-stops exhausted the continuation budget`);
+      return;
+    }
     // #903: an infrastructure kill spends no review round.
     //
     // A dispatch that dies before the resumed session takes a turn — a host
@@ -4128,6 +4297,7 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
       log(`#${issue} review round ${revision.round} stopped by operator: ${result.stoppedReason}`);
     } else if (state === "blocked") {
       swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
+      wakeOrchestratorForBlockedRun(project.name, issue);
       await safeEscalate(d, {
         tier: 1,
         project: project.name,
@@ -5060,6 +5230,53 @@ export function wakeOrchestratorForMetConditions(
 }
 
 
+/**
+ * Wake the orchestrator because a worker stopped to ask a question (#990).
+ *
+ * `question` is the third-largest class in this ledger — 58 of 839 runs — and
+ * every one of them is a parked worker holding a slot until someone reads the
+ * question. Nothing woke the orchestrator for it: the only wake was a decision
+ * condition being met, so on a fleet with a 1800s heartbeat a tier-1 question
+ * whose answer is one comment could sit unread for half an hour.
+ *
+ * Deliberately narrow. Only the blocked path wakes: a failed run, a cap kill
+ * and a clean settle are the next scheduled tick's business, and waking on
+ * every terminal state turns the heartbeat into a busy loop — which would be a
+ * worse fleet than the one that waits.
+ *
+ * Debounced by the marker itself rather than by a counter: an unconsumed
+ * request already asks for exactly the wake this call wants, so three workers
+ * blocking in one pass write one request and the *first* question's number
+ * survives as the reason. Best effort throughout — a marker that cannot be
+ * written is logged, and the settlement it belongs to is never affected.
+ */
+export function wakeOrchestratorForBlockedRun(
+  projectName: string,
+  issue: number,
+  writeLog: (line: string) => void = log,
+): void {
+  const tickCwd = resolveTickConfigCwd(projectName);
+  if (tickCwd === undefined) {
+    writeLog(
+      `#${issue} is blocked but no tick config cwd — the question waits for the next heartbeat`,
+    );
+    return;
+  }
+  const pending = readTickRequestReason(tickCwd);
+  if (pending !== undefined) {
+    writeLog(`#${issue} is blocked; a tick request is already pending (${pending})`);
+    return;
+  }
+  const reason = `tier1-question #${issue}`;
+  if (requestImmediateTick(tickCwd, reason)) {
+    writeLog(`requested immediate tick at ${tickCwd}: ${reason}`);
+  } else {
+    writeLog(
+      `could not write tick request under ${tickCwd}; #${issue}'s question waits for the next heartbeat`,
+    );
+  }
+}
+
 /**
  * The first-tick verification for a fleet-initiated upgrade (#486).
  *
@@ -5803,8 +6020,9 @@ export interface DaemonHealthSnapshot {
   turnOverrides: TurnOverride[];
   dispatch?: DispatchSummary;
   codeGraph?: CodeGraphHealth;
-  /** Live workers in a non-running pause phase; absent/empty = nothing paused. */
-  workers?: { issue: number; runId: string; phase: WorkerPausePhase }[];
+  /** Live workers in a non-running pause phase; absent/empty = nothing paused.
+   *  `source`/`pausedAtMs` carry the pause provenance when one was recorded (#997). */
+  workers?: { issue: number; runId: string; phase: WorkerPausePhase; source?: string; pausedAtMs?: number }[];
   /**
    * The live orchestrator surface, attested by the running daemon (#832):
    * which mode this project's fleet is in, and — when the daemon hosts the
@@ -5961,8 +6179,12 @@ export async function turnLimitResponse(
 export async function workerControlResponse(
   req: Request,
   project: string,
-  store: Pick<Store, "latestRun" | "updateRun">,
+  store: Pick<Store, "latestRun" | "updateRun" | "recordMaterialEvent">,
   registry: WorkerControlRegistry,
+  /** Journal line for every accepted pause/resume — the live half of the
+   *  audit trail #997 asks for; the durable half is the material event. */
+  log: (line: string) => void = () => {},
+  now: () => number = Date.now,
 ): Promise<Response | undefined> {
   const url = new URL(req.url);
   const match = /^\/runs\/(\d+)\/(pause|resume|stop)$/.exec(url.pathname);
@@ -6002,16 +6224,44 @@ export async function workerControlResponse(
       return Response.json({ error: "reason must be at most 500 characters" }, { status: 400 });
     }
   }
+  // Who asked. The board, the CLI and the dashboard each name themselves so a
+  // pause is attributable afterwards (#997); an older caller that sends no
+  // source is recorded as such rather than guessed at.
+  const rawSource = Reflect.get(body, "source");
+  if (rawSource !== undefined && (typeof rawSource !== "string" || rawSource.trim() === "" || rawSource.trim().length > 40)) {
+    return Response.json({ error: "source must be a non-empty string of at most 40 characters when present" }, { status: 400 });
+  }
+  const source = typeof rawSource === "string" ? rawSource.trim() : "unattributed";
 
 
   const issue = Number(match[1]);
   const outcome =
     action === "pause"
-      ? await registry.pause(project, issue)
+      ? await registry.pause(project, issue, source)
       : action === "resume"
         ? registry.resume(project, issue)
         : await registry.stop(project, issue, reason!);
   if (outcome.kind === "ok") {
+    // The transition really happened — write both halves of the audit trail
+    // before answering (#997): one journal line for the live log, one durable
+    // material event the digest ledger keeps after the journal rotates. #986
+    // was parked 14 minutes by an unlogged keypress and nobody could say why.
+    const verb = action === "pause" ? "paused" : "resumed";
+    log(`#${issue} worker ${verb} via ${source} (run ${outcome.runId})`);
+    try {
+      store.recordMaterialEvent({
+        project,
+        category: "worker-control",
+        summary: `#${issue} worker ${verb} via ${source}`,
+        evidence: `run ${outcome.runId}, phase ${outcome.phase}`,
+        occurredAt: now(),
+        recordedAt: now(),
+      });
+    } catch (err) {
+      // The audit must never turn a successful transition into an error
+      // answer, but a swallowed write would be a silent audit gap — log it.
+      log(`#${issue} worker-control audit write failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
     return Response.json({ runId: outcome.runId, phase: outcome.phase });
   }
   if (outcome.kind === "refused") {
@@ -6089,8 +6339,10 @@ export async function workerControlResponse(
 
 export interface DaemonHttpProjectDeps {
   project: string;
-  store: Pick<Store, "latestRun" | "setTurnOverride" | "updateRun">;
+  store: Pick<Store, "latestRun" | "setTurnOverride" | "updateRun" | "recordMaterialEvent">;
   caps: () => Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">;
+  /** Project-scoped journal line — the live half of the pause audit (#997). */
+  log?: (line: string) => void;
 }
 
 export interface DaemonHttpDeps {
@@ -6163,6 +6415,7 @@ export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promi
     selected.project,
     selected.store,
     d.workerControls,
+    selected.log,
   );
   return workerControl ?? new Response("not found\n", { status: 404 });
 }
@@ -6524,6 +6777,22 @@ export function statusSnapshotFromStore(
     spendTelemetry: judgeSpendTelemetry(
       store.recentSpendSamples?.(p.name, SPEND_SAMPLE_ROWS) ?? [],
       SPEND_SAMPLE_RUNS,
+      {
+        // Config and an already-read status only — this snapshot probes nothing
+        // at render time, which is the property the figure above depends on
+        // (#970). The declaration alone is enough to know a dollar cap cannot
+        // fire; the window name rides along when a configured plan cap has
+        // already resolved one, and `doctor` (which may probe) reads it live.
+        declaredSubscription: (p.requireOauthProviders ?? []).length > 0,
+        ...(planUsage?.window === undefined
+          ? {}
+          : {
+              allowanceWindow:
+                planUsage.window.label === undefined
+                  ? planUsage.window.id
+                  : `${planUsage.window.id} (${planUsage.window.label})`,
+            }),
+      },
     ),
     // Read, never probed: the recorded row is the whole point (#919).
     ...(() => {
@@ -7461,6 +7730,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
           project: runtime.d.project.name,
           store,
           caps: () => runtime.d.caps,
+          // The same project prefix the runtime's own journal lines carry, so
+          // a pause audit reads like every other daemon line (#997).
+          log: (line: string) =>
+            log(runtimes.length === 1 ? line : `[${runtime.d.project.name}] ${line}`),
         })),
         turnLimits,
         workerControls,

package/src/dashboard/controls.ts

@@ -189,7 +189,7 @@ export async function dashboardWorkerControl(
   body: unknown,
   d: ControlDeps,
 ): Promise<ControlOutcome> {
-  const payload: Record<string, unknown> = { project: d.project.name };
+  const payload: Record<string, unknown> = { project: d.project.name, source: "dashboard" };
   if (action === "stop") {
     const reason = readReason(body);
     if (reason === undefined) {