omp-conductor 0.19.3 → 0.19.4

package/README.md

@@ -386,9 +386,10 @@ message on stderr) on every uncertainty:
 - the tick config does not parse — the agent name would be a guess
 - `herdr agent list` is unreachable, prints nothing, or prints output with no
   explicit `agents` array; only a real `agents: []` means "no agents"
-- an agent row is unreadable — a missing `name`/`pane_id`, or an `agent` field
-  present with a non-string value. An *absent* or `null` `agent` is the sticky
-  claim herdr reports after the agent exits, and stays a normal answer
+- an agent row is unreadable — a missing `pane_id`, a non-string non-null
+  `name`, or an `agent` field present with a non-string value. An absent or
+  `null` `name` is an unnamed pane; an absent or `null` `agent` is the sticky
+  claim herdr reports after the agent exits, and both stay normal answers
 - the configured agent name is not unique, or the claimed pane runs some other agent
 - `pane process-info` fails, or the claim is live `omp` but names no recognizable
   omp foreground PID — "cannot see it" is never reported as "it is stopped"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omp-conductor",
-  "version": "0.19.3",
+  "version": "0.19.4",
   "type": "module",
   "license": "MIT",
   "description": "A 24/7 dispatcher that takes ready GitHub issues to green, mergeable PRs using omp coding sessions, with tiered escalation first to an orchestrator session and then to a human.",

package/src/daemon.ts

@@ -116,11 +116,13 @@ import {
 import {
   adoptSalvagedPrs,
   classifyAndRecover,
+  harnessLogDir,
   collectSettlementFlags,
   formatQuarantinedRuns,
   formatSalvagedRuns,
   reactToProviderCredit,
   readSessionError,
+  readTerminalEvidence,
   reconcileOrphanedRuns,
   reconcileGroomingClosures,
   reconcileStaleLabels,
@@ -133,6 +135,7 @@ import { materializeOmpSettings } from "./omp-settings.ts";
 import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
 import {
   infraLogSignature,
+  classifyRun,
   infraSignatureVersion,
   providerCreditRefusal,
   providerTransientFault,
@@ -333,6 +336,8 @@ interface Deps {
   workerControls: WorkerControlRegistry;
   /** Session seam for lifecycle integration tests; production uses the real harness. */
   workerDeps?: RunWorkerDeps;
+  /** Harness per-process logs used to classify terminal worker routes. */
+  harnessLogDir?: string;
   integrity: IntegrityGate;
   stall: StallGate;
   /**
@@ -3959,7 +3964,10 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
     if (await settleStopBeforeSession()) return;
     if (await settleDrainBeforeSession()) return;
 
-    store.updateRun(runId, { worktree: worktreePath });
+    // A review revision is a new worker process on the same run row. Clear the
+    // prior process evidence before launch so a failed wake cannot inherit an
+    // earlier round's harness route.
+    store.updateRun(runId, { worktree: worktreePath, workerPid: null, terminalEvidence: null });
 
     // The row's own counters are cumulative across the revision: the revision
     // worker meters its own session from zero, so its deltas are added to the
@@ -3972,6 +3980,7 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
     log(`#${issue} review round ${revision.round} → resuming ${branch} (${priorSessionFile})`);
 
 
+    let workerStartedAt: number | undefined;
     let result: WorkerResult;
     try {
       const runAllowanceUsd = runSpendAllowanceUsd(caps);
@@ -3997,7 +4006,9 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
         socketPath: join(sessionDir, "ipc.sock"),
         verbSocketPath: verbListener.path,
         onSpawn: (pid) => {
+          workerStartedAt = Date.now();
           verbListener?.bindPid(pid);
+          store.updateRun(runId, { workerPid: pid });
         },
         onChildLog: (line) => {
           log(`#${issue} ${line}`);
@@ -4092,6 +4103,18 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
       ...(verified.reason === undefined ? [] : ["", verified.reason]),
       result.report,
     ].join("\n");
+    const latestRun = store.getRun(runId) ?? run;
+    const terminalEvidence =
+      state === "failed" || state === "killed"
+        ? readTerminalEvidence(
+            {
+              sessionFile: result.sessionFile,
+              startedAt: workerStartedAt ?? latestRun.startedAt,
+              ...(latestRun.workerPid === undefined ? {} : { workerPid: latestRun.workerPid }),
+            },
+            d.harnessLogDir ?? harnessLogDir(),
+          )
+        : undefined;
 
     const terminalPatch: Partial<RunRecord> = {
       endedAt: Date.now(),
@@ -4112,9 +4135,76 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
       ...(result.headSha === undefined ? {} : { headSha: result.headSha }),
       sessionFile: result.sessionFile,
       ...(result.graphTools === undefined ? {} : { graphTools: result.graphTools }),
+      ...(terminalEvidence === undefined ? {} : { terminalEvidence }),
       report: finalReport,
       ...settlement?.patch,
     };
+    const terminalClassification =
+      terminalEvidence === undefined
+        ? undefined
+        : classifyRun(
+            {
+              ...latestRun,
+              ...terminalPatch,
+              state,
+              terminalEvidence,
+            },
+            {},
+            caps,
+          );
+    if (terminalClassification?.cls === "model-empty-stop") {
+      const recoveredAt = Date.now();
+      store.updateRun(runId, {
+        ...terminalPatch,
+        state: "failed",
+        failureClass: terminalClassification.cls,
+        recoveryAction: terminalClassification.recovery,
+        recoveredAt,
+      });
+      const continuations = store.continuationsFor(project.name, issue);
+      if (hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
+        // Keep the same review row, round, findings, session, branch and PR.
+        // Restore the exact pushed-green origin state while moving this stop
+        // into the explicit continuation counter; the next claim must not
+        // depend on re-verifying a failed-state origin at an unchanged head.
+        const chargedRun = store.getRun(runId) ?? latestRun;
+        store.updateRun(runId, {
+          ...terminalPatch,
+          state: "pushed-green",
+          failureClass: null,
+          recoveryAction: null,
+          recoveredAt: null,
+          continuationCharges: (chargedRun.continuationCharges ?? 0) + 1,
+          workerPid: null,
+          lastError: terminalClassification.evidence,
+        });
+        store.requeueReviewRevision(revision.id);
+        log(
+          `#${issue} review round ${revision.round} re-queued after a provider empty-stop ` +
+            `(${continuations}/${caps.maxContinuationsPerIssue} continuations): ${terminalClassification.evidence}`,
+        );
+        return;
+      }
+
+      store.settleReviewRevision(revision.id, "failed", recoveredAt);
+      swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
+      const model = result.model ?? latestRun.resolvedModel ?? latestRun.model ?? "configured model";
+      const provider = result.provider ?? latestRun.resolvedProvider ?? "provider";
+      await safeEscalate(d, {
+        tier: 1,
+        project: project.name,
+        issue,
+        runId,
+        summary: `#${issue} review round ${revision.round}: the model kept returning empty responses`,
+        detail:
+          `${provider}/${model} ended ${continuations} review session(s) with empty responses until ` +
+          `the harness retry cap. The same review round was preserved while continuation budget remained; ` +
+          `that budget is now exhausted at ${caps.maxContinuationsPerIssue}. No implementation failure or ` +
+          `additional review round was charged.\n\n${terminalClassification.evidence}`,
+      });
+      log(`#${issue} review round ${revision.round} provider empty-stops exhausted the continuation budget`);
+      return;
+    }
     // #903: an infrastructure kill spends no review round.
     //
     // A dispatch that dies before the resumed session takes a turn — a host

package/src/fleet.ts

@@ -815,7 +815,7 @@ export function startHerdrFleet(projectName?: string, deps: HerdrStartDeps = {})
 }
 
 export interface HerdrAgent {
-  name: string;
+  name?: string;
   paneId: string;
   agent?: string;
   sessionPath?: string;
@@ -1568,16 +1568,22 @@ export function parseHerdrAgentList(rawOutput: string): HerdrAgent[] {
     const agent = row as { readonly [key: string]: unknown };
     const name = agent["name"];
     const paneId = agent["pane_id"];
-    if (typeof name !== "string" || typeof paneId !== "string") {
+    if (typeof paneId !== "string") {
+      throw new Error(
+        "herdr agent list row is missing a string `pane_id` — cannot identify the pane",
+      );
+    }
+    if (name !== undefined && name !== null && typeof name !== "string") {
       throw new Error(
-        "herdr agent list row is missing a string `name`/`pane_id` — " +
+        `herdr agent list row for pane ${paneId} has a non-string \`name\` (${typeof name}) — ` +
           "cannot tell whether it is the conductor pane",
       );
     }
+    const parsedName = typeof name === "string" ? name : undefined;
     const rawAgent = agent["agent"];
     if (rawAgent !== undefined && rawAgent !== null && typeof rawAgent !== "string") {
       throw new Error(
-        `herdr agent list row for ${name} has a non-string \`agent\` (${typeof rawAgent}) — ` +
+        `herdr agent list row for ${parsedName ?? `pane ${paneId}`} has a non-string \`agent\` (${typeof rawAgent}) — ` +
           `cannot tell whether an agent is live`,
       );
     }
@@ -1591,7 +1597,7 @@ export function parseHerdrAgentList(rawOutput: string): HerdrAgent[] {
       }
     }
     out.push({
-      name,
+      ...(parsedName === undefined ? {} : { name: parsedName }),
       paneId,
       ...(liveAgent === undefined ? {} : { agent: liveAgent }),
       ...(sessionPath === undefined ? {} : { sessionPath }),

package/src/settlement.ts

@@ -1287,10 +1287,10 @@ export function harnessLogDir(): string {
  *  per process. These name why a session ended when the row cannot: the run
  *  itself records no error, because from its side nothing failed (#986). */
 const HARNESS_ROUTE_MARKERS = [
-  "empty-stop-retry-cap",
-  "empty-stop-handled",
-  "bottom-checkCompaction",
-  "context-overflow",
+  { marker: "empty-stop-retry-cap", terminal: true },
+  { marker: "empty-stop-handled", terminal: false },
+  { marker: "bottom-checkCompaction", terminal: false },
+  { marker: "context-overflow", terminal: true },
 ] as const;
 
 /**
@@ -1373,13 +1373,27 @@ function readHarnessRoute(
   } catch {
     return undefined;
   }
-  // Last match wins: the terminal route is the one that ended the session, and
-  // `empty-stop-handled` appearing earlier is a retry that worked.
+  // A terminal failure route remains the cause even when the harness performs
+  // post-agent maintenance afterwards. The live #1001 logs record
+  // `empty-stop-retry-cap` and then `bottom-checkCompaction` milliseconds later;
+  // choosing the last marker hid the failure behind routine bookkeeping.
   let found: string | undefined;
-  for (const marker of HARNESS_ROUTE_MARKERS) {
-    const at = text.lastIndexOf(marker);
-    if (at === -1) continue;
-    if (found === undefined || at > text.lastIndexOf(found)) found = marker;
+  let foundAt = -1;
+  for (const route of HARNESS_ROUTE_MARKERS) {
+    if (!route.terminal) continue;
+    const at = text.lastIndexOf(route.marker);
+    if (at > foundAt) {
+      found = route.marker;
+      foundAt = at;
+    }
+  }
+  if (found !== undefined) return `harness route ${found}`;
+  for (const route of HARNESS_ROUTE_MARKERS) {
+    const at = text.lastIndexOf(route.marker);
+    if (at > foundAt) {
+      found = route.marker;
+      foundAt = at;
+    }
   }
   return found === undefined ? undefined : `harness route ${found}`;
 }

package/src/store.ts

@@ -3758,7 +3758,7 @@ export function openStore(dbPath: string): Store {
          ELSE 0
        END,
        continuationCharges = COALESCE(continuationCharges, 0) + CASE
-         WHEN state = 'failed' AND failureClass = 'returned-for-revision' THEN 1
+         WHEN state = 'failed' AND failureClass IN ('returned-for-revision', 'model-empty-stop') THEN 1
          WHEN state = 'killed' AND (failureClass IS NULL OR failureClass NOT IN (${CHARGEABLE_CONTINUATION_EXCLUSIONS})) THEN 1
          ELSE 0
        END