omp-conductor 0.19.2 → 0.19.4

package/src/failure-class.ts

@@ -170,6 +170,16 @@ const START_FAILURE_SIGNATURES = [
   // symptom this replaced (the child dying on the peer import) already read as
   // a start failure, so charging one here would be a regression dressed as a fix.
   "worker identity unavailable",
+  // The 2026-08-2x host faults (#986), each recorded verbatim in `lastError`
+  // by the launcher or the worker identity gate — all before the session took
+  // a turn:
+  //  - the child died before it could connect to the dispatcher;
+  //  - node could not resolve a package out of the worker harness tree;
+  //  - the worker's settings file was unreadable (the EACCES outage).
+  "exited 1 before connecting",
+  "cannot find package",
+  "enoent while resolving package",
+  "failed to read settings config",
 ] as const;
 
 /**
@@ -395,6 +405,128 @@ export function adminRestartAttribution(lastError: string | undefined): string |
   return firstLine.trim();
 }
 
+/** The launcher's own statement that the session child died mid-run, before
+ *  the session ended. Written by `omp.ts` when the child exits while the
+ *  dispatcher still owns it; the exit code is the child's own, so a signal
+ *  death reads >=128 (143 = SIGTERM, 137 = SIGKILL) — an external kill,
+ *  never anything the worker's work decided (#986). */
+const CHILD_EXIT_MARKER = /session child exited (\d+) before the session ended/;
+
+/**
+ * Evidence that this run's session child was terminated by a signal, or
+ * `undefined` when it was not.
+ *
+ * Read from the surfaces a run persists — `lastError` first, then `report`,
+ * because the prompt-throw path records the launcher's message as the report
+ * and leaves `lastError` empty. Deliberately narrowed to signal-range codes:
+ * a clean nonzero exit is some process's verdict nobody has named yet, and
+ * naming it `admin-kill` would waive an attempt that may have been spent.
+ */
+export function childSignalDeath(
+  run: Pick<RunRecord, "lastError" | "report">,
+): string | undefined {
+  const text = run.lastError ?? run.report;
+  if (text === undefined) return undefined;
+  const match = CHILD_EXIT_MARKER.exec(text);
+  if (match === null) return undefined;
+  const code = Number(match[1]);
+  if (!Number.isFinite(code) || code < 128) return undefined;
+  return match[0];
+}
+
+/** The reason the settle sweep writes beside a reviewer-closed PR
+ *  (`settlementFor`), matched as recorded evidence so rows written before
+ *  that sweep stamped a class read the same way (#986). */
+const REVIEWER_CLOSED_MARKER = "closed without merging";
+
+/**
+ * Evidence that a reviewer closed this run's pushed work without merging it,
+ * or `undefined` when nothing says so. Two independent signals, either
+ * sufficient: the tracker fact (`facts.pr === "closed"`), or the settle
+ * sweep's own recorded reason in `lastError`. A review decision is not a
+ * worker failure, which is exactly what `returned-for-revision` means; the
+ * only new reading here is applying it to rows whose PR closed while the row
+ * itself had already gone terminal-failed.
+ */
+export function reviewerClosed(
+  run: Pick<RunRecord, "lastError" | "prUrl">,
+  facts: ClassifyFacts,
+): string | undefined {
+  if (run.lastError !== undefined && run.lastError.includes(REVIEWER_CLOSED_MARKER)) {
+    return run.lastError.split("\n")[0]?.trim();
+  }
+  if (facts.pr === "closed" && run.prUrl !== undefined) {
+    return `${run.prUrl} closed without merging`;
+  }
+  return undefined;
+}
+
+/** The harness's own name for giving up on a provider that kept answering with
+ *  empty assistant turns: it retries, then ends the session on the retry cap
+ *  and says so in its maintenance routing (`route: "empty-stop-retry-cap"`,
+ *  beside `Assistant returned empty stop after retry cap; try switching
+ *  models`). A closed marker written by the harness, not vendor prose. */
+const EMPTY_STOP_MARKER = "empty-stop-retry-cap";
+
+/**
+ * Evidence that the provider answered with empty turns until the harness's own
+ * retry cap ended the session, or `undefined` when nothing recorded says so
+ * (#986).
+ *
+ * Read from the terminal evidence the settlement sweep recorded, because this
+ * marker is written by the harness's session log rather than by the transcript
+ * or the row — the run itself has no error, no verdict and no idea it died,
+ * which is precisely why two of these read as `unknown` on 2026-08-23 at
+ * 100/180 and 54/180 turns while a third run on the same model finished green
+ * in the same window.
+ *
+ * Provider flake, not worker logic: the recovery continues from whatever the
+ * attempt pushed rather than charging an implementation attempt for it.
+ */
+export function modelEmptyStop(
+  run: Pick<RunRecord, "terminalEvidence">,
+): string | undefined {
+  if (run.terminalEvidence === undefined) return undefined;
+  if (!run.terminalEvidence.includes(EMPTY_STOP_MARKER)) return undefined;
+  return run.terminalEvidence;
+}
+
+/**
+ * Evidence that this run ended without ever delivering a settlement verdict,
+ * or `undefined` when something on the row says otherwise (#986).
+ *
+ * The shape: state `failed`, turns taken, no error anywhere, and a final
+ * report whose words are mid-delivery narration — "Now I'll execute …", "Type
+ * check passes. Now the test suite:" — with no `state:`/`status:` line the
+ * result parser could read. The session reached its worker and died between
+ * turns without handing over; measured over two weeks of the reference fleet,
+ * this one shape was half of all `unknown`.
+ *
+ * Deliberately narrow at every edge: turn zero belongs to
+ * {@link neverStarted}, any error text keeps the row `unknown` (an
+ * unrecognised error says something happened), and an absent report leaves
+ * genuinely nothing to name. The class charges its attempt exactly as
+ * `unknown` did — only the name becomes specific.
+ */
+export function noVerdictExit(run: RunRecord): string | undefined {
+  if (run.state !== "failed") return undefined;
+  if (run.turns <= 0) return undefined;
+  // Blank counts as absent: a row whose `lastError` is an empty string carries
+  // no error information, and reading it as "something happened" is what kept
+  // four real rows unnamed. Any non-blank text, recognised or not, still wins —
+  // an unrecognised error says something happened.
+  if (run.lastError !== undefined && run.lastError.trim() !== "") return undefined;
+  if (run.report === undefined || run.report.trim() === "") return undefined;
+  // Mirror the two verdict spellings the result path reads (`worker.ts`
+  // STATE_LINE_PATTERN for prose settlements, `renderSettlement`'s leading
+  // `status:` line for structured yields). Either one present means a verdict
+  // was delivered and this function must stay silent.
+  if (/^state:\s*\S+\s*$/im.test(run.report)) return undefined;
+  if (/^status:\s*\S+\s*$/im.test(run.report)) return undefined;
+  const lastWords = run.report.split("\n")[0]?.trim() ?? "";
+  return `the session ended without delivering a settlement verdict; last words: "${lastWords.slice(0, 80)}"`;
+}
+
 export function classifyRun(
   run: RunRecord,
   facts: ClassifyFacts,
@@ -424,6 +556,23 @@ export function classifyRun(
     };
   }
 
+  // The launcher's own statement that the session child died on a signal
+  // mid-run: an external kill — a drain, a restart, the host — not anything
+  // the worker's work decided (#986). Ahead of the provider branches on
+  // purpose: a transcript can carry an older, long-recovered stream fault,
+  // and the launcher's terminal statement outranks whatever the session said
+  // earlier. Requeued free like every other administrative kill.
+  if (run.state === "failed") {
+    const marker = childSignalDeath(run);
+    if (marker !== undefined) {
+      return {
+        cls: "admin-kill",
+        recovery: "requeue",
+        evidence: `killed externally (${marker}) — the session child was terminated, not finished`,
+      };
+    }
+  }
+
   // A billing state, not an implementation failure. Its own class because it is
   // the one outage an operator fixes with a card rather than a diagnosis, and
   // because burying a self-describing provider error in `unknown` erodes what
@@ -507,6 +656,19 @@ export function classifyRun(
     }
   }
 
+  // A reviewer closed this run's pushed work without merging it: a review
+  // decision, not a worker failure (#986). Two rows reach here — the settle
+  // sweep's own recorded reason in `lastError` on a row that predates the
+  // class, and a live `closed` tracker fact on a row whose worker died after
+  // pushing but before claiming. Recovery `none` mirrors the settle sweep
+  // exactly: the queue label is the remedy, and no action is performed here.
+  if (run.state === "failed") {
+    const closed = reviewerClosed(run, facts);
+    if (closed !== undefined) {
+      return { cls: "returned-for-revision", recovery: "none", evidence: closed };
+    }
+  }
+
   if (run.state === "blocked") {
     return {
       cls: "question",
@@ -639,6 +801,36 @@ export function classifyRun(
     }
   }
 
+  // The provider answered with empty turns until the harness gave up (#986).
+  // Below every branch that reads a real error on purpose: a row carrying its
+  // own provider fault is that fault, and this marker only decides rows that
+  // would otherwise be `unknown`. Continues rather than requeues, because the
+  // attempt's pushed work is the honest place to resume from, and a provider
+  // flake must not charge an implementation attempt (#1001). `killed` rows
+  // never reach here — a cap kill is the operative fact about a run that hit
+  // its ceiling, whatever the provider did earlier in the session.
+  if (run.state === "failed") {
+    const emptyStop = modelEmptyStop(run);
+    if (emptyStop !== undefined) {
+      return {
+        cls: "model-empty-stop",
+        recovery: "continue",
+        evidence: `the provider returned empty turns until the harness's retry cap ended the session (${emptyStop}) — turns ${run.turns}/${run.maxTurns}`,
+      };
+    }
+  }
+
+  // The session reached its worker and died between turns without handing over
+  // a verdict (#986): the largest single shape inside `unknown`. Named rather
+  // than requeued blind — an operator reading `status` learns the run stopped
+  // mid-delivery, which is a different question from "nothing is known".
+  if (run.state === "failed") {
+    const narration = noVerdictExit(run);
+    if (narration !== undefined) {
+      return { cls: "no-verdict", recovery: "escalate", evidence: narration };
+    }
+  }
+
   // Deliberately escalate rather than retry. An unrecognised shape is a gap in
   // this table, and a silent requeue would spend a budget on a cause nobody has
   // named — the exact behaviour #132 exists to end.