faberun 0.3.0

package/src/engine/prompts.mjs

@@ -0,0 +1,289 @@
+import { validateWorkerResult } from "../contract/worker-result.mjs";
+import { scopeFindingsPromptSection } from "../contract/scope-findings.mjs";
+import { JUDGE_ENVELOPE_REASON, JUDGE_FINDING_ENVELOPE_REASON, JUDGE_LIMITS } from "../contract/judge-envelope.mjs";
+
+/** @typedef {{id: string, definitionOfDone: import("../contract/definition-of-done.mjs").DefinitionOfDoneItem[], taskPacket: {mode?: "execution"|"discovery"|"autonomous", objective: string, instructions: string[], writeFiles?: string[], writeRoots?: string[], verification: {argv: string[]}[]}}} JudgeNode */
+/** @typedef {{verdict: "pass"|"fail"|"invalid_judge_output", maxSeverity: "none"|"minor"|"major"|"critical", summary: string, findings: {severity: "minor"|"major"|"critical", description: string, evidence: string}[]}} JudgeVerdict */
+
+/**
+ * A node whose work is over and whose outcome needs nothing more from the
+ * controller: success or an explicit cancellation.
+ */
+export const TERMINAL = new Set([
+  "done",
+  "no-op",
+  "canceled",
+]);
+
+/**
+ * A node that stopped without succeeding and cannot proceed by itself. Parking
+ * is not finishing: the run needs attention, not a "done" report. Kept apart
+ * from `TERMINAL` so the watchdog can tell the two apart instead of returning
+ * the moment every node has stopped.
+ */
+export const PARKED = new Set([
+  "blocked",
+  "failed",
+  "exhausted",
+  "stalled",
+]);
+
+/** Every status a node never leaves on its own: terminal plus parked. */
+export const SETTLED = new Set([...TERMINAL, ...PARKED]);
+
+/** The statuses that count as a node's success. */
+export const SUCCESS = new Set(["done", "no-op"]);
+
+export const JUDGE_SCHEMA = {
+  type: "object",
+  additionalProperties: false,
+  properties: {
+    verdict: { type: "string", enum: ["pass", "fail"] },
+    maxSeverity: { type: "string", enum: ["none", "minor", "major", "critical"] },
+    summary: { type: "string" },
+    findings: {
+      type: "array",
+      items: {
+        type: "object",
+        additionalProperties: false,
+        properties: {
+          severity: { type: "string", enum: ["minor", "major", "critical"] },
+          description: { type: "string" },
+          evidence: { type: "string" },
+        },
+        required: ["severity", "description", "evidence"],
+      },
+    },
+  },
+  // Every property, and it has to be every property: OpenAI's structured
+  // output rejects the schema itself -- `invalid_json_schema`, 400, before the
+  // model runs -- unless `required` lists every key in `properties`. Dropping
+  // `findings` from here to spare Claude an occasional omission took the codex
+  // judge down completely, which is the worse trade: Claude's failure was
+  // intermittent and recoverable, this one was every invocation.
+  //
+  // The tolerance lives in `parseJudge`, which reads an absent `findings` as
+  // `[]`, and in the prompt below, which says out loud that a clean pass still
+  // sends the empty array.
+  required: ["verdict", "maxSeverity", "summary", "findings"],
+};
+
+/**
+ * @param {string} result
+ * @returns {JudgeVerdict}
+ */
+export function parseJudge(result) {
+  let parsed;
+  try {
+    parsed = /** @type {unknown} */ (JSON.parse(result));
+  } catch (error) {
+    throw new Error(`judge returned invalid JSON: ${error instanceof Error ? error.message : String(error)}`);
+  }
+  const verdict = /** @type {Record<string, unknown>} */ (parsed);
+  if (!verdict || typeof verdict !== "object" || Array.isArray(verdict)) throw new Error("judge result must be an object");
+  // Judge output is an external LLM boundary: models add fields beyond the
+  // schema (toolAction, confidence, …). Unknown keys are dropped here; every
+  // canonical field keeps strict value validation below.
+  const judgeVerdict = verdict.verdict;
+  const maxSeverity = verdict.maxSeverity;
+  if (typeof judgeVerdict !== "string" || !JUDGE_SCHEMA.properties.verdict.enum.includes(judgeVerdict)) throw new Error("judge verdict must be pass or fail");
+  if (typeof maxSeverity !== "string" || !JUDGE_SCHEMA.properties.maxSeverity.enum.includes(maxSeverity)) throw new Error("judge maxSeverity is invalid");
+  if (typeof verdict.summary !== "string") throw new Error("judge result is missing summary");
+  if (verdict.findings !== undefined && !Array.isArray(verdict.findings)) throw new Error("judge findings must be an array");
+  const rawFindings = /** @type {unknown[]} */ (verdict.findings ?? []);
+  if (Buffer.byteLength(verdict.summary, "utf8") > JUDGE_LIMITS.summaryBytes || rawFindings.length > JUDGE_LIMITS.findings) throw new Error(JUDGE_ENVELOPE_REASON);
+  const severityRank = { none: 0, minor: 1, major: 2, critical: 3 };
+  const findings = rawFindings.map((finding) => {
+    if (!finding || typeof finding !== "object" || Array.isArray(finding)) throw new Error("judge finding is invalid");
+    const record = /** @type {Record<string, unknown>} */ (finding);
+    const severity = record.severity;
+    if (typeof severity !== "string" || !["minor", "major", "critical"].includes(severity) || typeof record.description !== "string" || typeof record.evidence !== "string") {
+      throw new Error("judge finding is invalid");
+    }
+    if (Buffer.byteLength(record.description, "utf8") > JUDGE_LIMITS.descriptionBytes || Buffer.byteLength(record.evidence, "utf8") > JUDGE_LIMITS.evidenceBytes) throw new Error(JUDGE_FINDING_ENVELOPE_REASON);
+    return {
+      severity: /** @type {"minor"|"major"|"critical"} */ (severity),
+      description: record.description,
+      evidence: record.evidence,
+    };
+  });
+  const findingSeverities = findings.map((finding) => finding.severity);
+  const actualMax = findingSeverities.reduce(
+    (highest, severity) => severityRank[severity] > severityRank[highest] ? severity : highest,
+    /** @type {"none"|"minor"|"major"|"critical"} */ ("none"),
+  );
+  if (actualMax !== maxSeverity) throw new Error("judge maxSeverity does not match findings");
+  if ((judgeVerdict === "pass") !== (maxSeverity === "none")) throw new Error("judge verdict and maxSeverity are inconsistent");
+  return {
+    verdict: /** @type {"pass"|"fail"} */ (judgeVerdict),
+    maxSeverity: /** @type {"none"|"minor"|"major"|"critical"} */ (maxSeverity),
+    summary: verdict.summary,
+    findings,
+  };
+}
+
+/**
+ * How much of a red command's captured output the judge prompt carries, per
+ * stream, and the total budget across all red commands. Green commands send no
+ * output at all: the controller already ran them and their result is the
+ * `passed` bit, so echoing 2 KiB stdout plus 2 KiB stderr per green command per
+ * attempt is pure spend. This campaign already lost a node to that: a packet
+ * with seven verification commands serialized the whole `state.verification`
+ * and crossed the 64 KiB judge-prompt guard.
+ */
+const JUDGE_RED_STREAM_BYTES = 2 * 1024;
+const JUDGE_RED_TOTAL_BYTES = 16 * 1024;
+
+/**
+ * The judge's view of controller verification: `{argv, passed}` for every green
+ * command and output tails only for red ones. The state carries up to 2 KiB per
+ * stream per attempt for every command, green or red; the judge only needs the
+ * red evidence, and bounding the red evidence in aggregate keeps the whole
+ * prompt inside the dispatch guard.
+ *
+ * @param {unknown} verification
+ * @returns {{passed: boolean, error?: string, commands: Array<Record<string, unknown>>}}
+ */
+function judgeVerificationEvidence(verification) {
+  const record = verification && typeof verification === "object" && !Array.isArray(verification)
+    ? /** @type {{passed?: unknown, error?: unknown, commands?: unknown}} */ (verification)
+    : {};
+  const commands = Array.isArray(record.commands) ? record.commands : [];
+  let budget = JUDGE_RED_TOTAL_BYTES;
+  const compact = commands.map((command) => {
+    const entry = command && typeof command === "object" && !Array.isArray(command)
+      ? /** @type {Record<string, unknown>} */ (command)
+      : {};
+    if (entry.passed === true) return { argv: entry.argv, passed: true };
+    /** @type {Record<string, unknown>} */
+    const red = { argv: entry.argv, passed: false };
+    const attempts = Array.isArray(entry.attempts) ? entry.attempts : [];
+    const last = attempts.length ? /** @type {Record<string, unknown>} */ (attempts[attempts.length - 1]) : null;
+    if (typeof last?.exitCode === "number") red.exitCode = last.exitCode;
+    if (last?.timedOut === true) red.timedOut = true;
+    for (const stream of /** @type {const} */ (["stdout", "stderr"])) {
+      const text = typeof last?.[stream] === "string" ? last[stream] : "";
+      if (!text.trim() || budget <= 0) continue;
+      const bounded = boundedPromptText(text, Math.min(JUDGE_RED_STREAM_BYTES, budget));
+      budget -= Buffer.byteLength(bounded, "utf8");
+      red[stream] = bounded;
+    }
+    return red;
+  });
+  return {
+    passed: record.passed === true,
+    ...(typeof record.error === "string" && record.error ? { error: record.error } : {}),
+    commands: compact,
+  };
+}
+
+/**
+ * @param {JudgeNode} node
+ * @param {unknown} workerResult
+ * @param {{diff?: unknown[], verification?: unknown, deterministic?: unknown, scopeFindings?: {unexpectedPaths: string[]}|null, previousAttempt?: string}} context
+ * @returns {string}
+ */
+export function judgePrompt(node, workerResult, context = {}) {
+  const criteria = node.definitionOfDone.length
+    ? judgeDoDChecklist(node.definitionOfDone, context.deterministic)
+    : "- The requested work is complete, correct, tested, and limited to scope.";
+  const taskInstructions = node.taskPacket.instructions.length
+    ? node.taskPacket.instructions.map((item, index) => `${index + 1}. ${item}`).join("\n")
+    : "(none)";
+  const writeBoundary = node.taskPacket.mode === "autonomous"
+    ? node.taskPacket.writeRoots ?? []
+    : node.taskPacket.writeFiles ?? [];
+  const writeBoundaryLabel = node.taskPacket.mode === "autonomous" ? "Write roots" : "Write files";
+  const writeFiles = writeBoundary.length
+    ? writeBoundary.map((path) => `- ${path}`).join("\n")
+    : "- (none)";
+  const verification = node.taskPacket.verification.length
+    ? node.taskPacket.verification.map((command) => `- ${command.argv.join(" ")}`).join("\n")
+    : "- (none)";
+  let structured = null;
+  try { structured = typeof workerResult === "string" ? JSON.parse(workerResult) : workerResult; } catch {
+    // Non-JSON worker result: leave it withheld from the judge prompt.
+  }
+  if (structured) {
+    try { structured = validateWorkerResult(structured); } catch { structured = null; }
+  }
+  const diff = Array.isArray(context.diff) ? /** @type {unknown[]} */ (context.diff).slice(0, 64) : [];
+  const verificationResult = judgeVerificationEvidence(context.verification);
+  const scopeSection = scopeFindingsPromptSection(context.scopeFindings);
+  return `Review node ${node.id} independently. The review context is closed: inspect only the ${node.taskPacket.mode === "autonomous" ? "write roots" : "write files"} below and do not perform repository-wide discovery. Do not re-run the verification commands — the controller already executed them and attached the results; re-running suites duplicates cost without adding evidence.\n\n` +
+    `${writeBoundaryLabel}:\n${writeFiles}\n\nVerification commands (already executed by the controller):\n${verification}\n\n` +
+    `Task brief:\n${node.taskPacket.objective}\n\nInstructions:\n${taskInstructions}\n\nDefinition of Done:\n${criteria}\n\n` +
+    `Worker result (structured):\n${structured ? JSON.stringify(structured) : "(invalid worker result withheld)"}\n\n` +
+    `Controller diff paths:\n${diff.length ? diff.map((path) => `- ${path}`).join("\n") : "- (none)"}\n\n` +
+    `Controller verification:\n${JSON.stringify(verificationResult)}\n\n` +
+    (scopeSection ? `${scopeSection}\n\n` : "") +
+    (context.previousAttempt ? `${context.previousAttempt}\n\n` : "") +
+    "Return only the JSON object required by the output schema, with every field present: a verdict with nothing to report still carries `findings: []`, never an omitted key. The verdict is a record, not a report: keep `summary` within " + JUDGE_LIMITS.summaryBytes + " bytes, use at most " + JUDGE_LIMITS.findings + " findings, and keep each finding's `description` within " + JUDGE_LIMITS.descriptionBytes + " bytes and its `evidence` within " + JUDGE_LIMITS.evidenceBytes + " bytes. A verdict that overshoots this envelope is rejected unread, however sound the arbitration. Evidence must be concrete. " +
+    "Use verdict pass only when findings is empty and maxSeverity is none. " +
+    "Use verdict fail whenever findings is non-empty, including advisory findings below failOn. " +
+    (node.definitionOfDone.length
+      ? "Arbitrate only the judgment items; deterministic items are already proven by the controller and must not be re-arbitrated. " +
+        "Use pass only when every judgment item is satisfied. Every finding must cite the id of the judgment item it addresses; a fail verdict whose findings cite no id is a protocol failure."
+      : "Use pass only when every Definition of Done item is satisfied. " +
+        "Assess every Definition of Done item by its id and cite the id you are addressing in each finding.");
+}
+
+/**
+ * The Definition of Done checklist for the judge prompt: judgment items the
+ * judge must arbitrate, plus each deterministic item with the controller-run
+ * proof result when one is attached.
+ *
+ * @param {import("../contract/definition-of-done.mjs").DefinitionOfDoneItem[]} items
+ * @param {unknown} deterministic
+ * @returns {string}
+ */
+function judgeDoDChecklist(items, deterministic) {
+  const results = Array.isArray(deterministic)
+    ? /** @type {Array<{id: string, pass: boolean, detail: string}>} */ (deterministic)
+    : null;
+  const parts = [];
+  const judgmentItems = items.filter((item) => item.judgment === true);
+  parts.push(judgmentItems.length
+    ? `Judgment items — arbitrate only these:\n${judgmentItems.map((item) => `- [${item.id}] ${item.text} (judgment)`).join("\n")}`
+    : "No judgment items require arbitration.");
+  const proofItems = items.filter((item) => item.proof !== undefined);
+  if (proofItems.length) {
+    parts.push(`Deterministic items — already proven by the controller; do not re-arbitrate them:\n${proofItems.map((item) => {
+      const proof = /** @type {import("../contract/definition-of-done.mjs").DefinitionOfDoneProof} */ (item.proof);
+      const result = results?.find((entry) => entry.id === item.id);
+      const outcome = result === undefined
+        ? "controller proof recorded"
+        : result.pass ? "PASS" : `FAIL — ${result.detail}`;
+      return `- [${item.id}] ${outcome} — ${item.text} (proof: ${proof.kind} ${proof.ref})`;
+    }).join("\n")}`);
+  }
+  return parts.join("\n\n");
+}
+
+/**
+ * @param {{prompt: string}} node
+ * @param {{summary: string, findings: {severity: "minor"|"major"|"critical", description: string, evidence: string}[]}} verdict
+ * @returns {string}
+ */
+export function retryPrompt(node, verdict) {
+  const findings = verdict.findings
+    .map((finding) => `- [${finding.severity}] ${boundedPromptText(finding.description, 2 * 1024)} Evidence: ${boundedPromptText(finding.evidence, 4 * 1024)}`)
+    .join("\n");
+  const prompt = `${node.prompt}\n\nA quality gate rejected the previous attempt. Fix the current working tree in place inside the closed context above.\n` +
+    `Gate summary: ${boundedPromptText(verdict.summary, 4 * 1024)}\n${findings}`;
+  return Buffer.byteLength(prompt, "utf8") > 64 * 1024
+    ? `${node.prompt}\n\nA quality gate rejected the previous attempt. Review the bounded structured findings in the state snapshot and fix the working tree inside the closed context.`
+    : prompt;
+}
+
+/**
+ * @param {unknown} value
+ * @param {number} maxBytes
+ * @returns {string}
+ */
+function boundedPromptText(value, maxBytes) {
+  const text = String(value ?? "");
+  const bytes = Buffer.from(text, "utf8");
+  return bytes.length <= maxBytes ? text : `${bytes.subarray(0, maxBytes - 1).toString("utf8")}…`;
+}
+

package/src/engine/recover.mjs

@@ -0,0 +1,300 @@
+/**
+ * Adopting what a previous controller left running.
+ *
+ * A resumed run finds nodes marked `running` whose process may be alive, dead,
+ * or finished-but-unrecorded. `recoverOrphan` decides which, from the pid, the
+ * process start token, the operation ledger and the transcript on disk -- never
+ * by re-dispatching hopefully. Adopt a finished turn, re-judge a finished
+ * worker, restart only what left no usable evidence.
+ */
+import { delay } from "../util.mjs";
+import { hasOperationIntent, operationNeedsRecovery, operationNextState, readOperationSettlement, settleInvocation } from "../run/operations.mjs";
+import { invocationOwned } from "./process-identity.mjs";
+import { invocationResult, terminateInvocation } from "./process.mjs";
+import { latestTimeoutSec } from "./backoff.mjs";
+import { markRecovering } from "./supervise.mjs";
+import { parseJudge } from "./prompts.mjs";
+import { persistedJudgeResult, persistedWorkerResult } from "./result-file.mjs";
+import { routeRuntimeForState, runtimeSnapshot } from "./failover.mjs";
+
+/** @typedef {import("./lifecycle.mjs").Invocation} Invocation */
+/** @typedef {import("../cli.mjs").LockHandle} LockHandle */
+/** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
+/** @typedef {import("../harnesses/index.mjs").ProviderEnvelope} ProviderEnvelope */
+/** @typedef {ProviderEnvelope & {costProvenance?: "priced"}} PricedEnvelope */
+/** @typedef {import("../run/usage.mjs").RecoveryOutcome} RecoveryOutcome */
+/** @typedef {import("../contract/index.mjs").Usage} Usage */
+/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
+/** @typedef {import("../contract/index.mjs").ValidatedNode} ValidatedNode */
+
+/**
+ * @param {string} runDir
+ * @param {ValidatedContract} contract
+ * @param {ValidatedNode} node
+ * @param {NodeSnapshot} state
+ * @param {LockHandle} lock
+ * @returns {Promise<RecoveryOutcome|null>}
+ */
+export async function recoverOrphan(runDir, contract, node, state, lock) {
+  const invocation = state.invocations?.at(-1);
+  if (state.status !== "running") return null;
+  if (!invocation) return { kind: "restart", reason: `node ${state.id} has no live invocation` };
+  if (hasOperationIntent(runDir, invocation.id) && operationNeedsRecovery(runDir, invocation.id)) {
+    // Persist the classification before inspecting any provider or workspace
+    // evidence. A controller can therefore die again without losing the fact
+    // that this invocation's external effect was ambiguous.
+    settleInvocation(runDir, invocation, {
+      status: "unknown_effect",
+      terminalOutcome: "unknown_effect",
+      unknownEffect: true,
+      classification: "unknown_effect",
+      reason: `invocation ${invocation.id} has an intent but no terminal settlement`,
+      nextState: operationNextState(state),
+    });
+  }
+  const settlement = readOperationSettlement(runDir, invocation.id);
+  const runtime = invocation?.runtimeId
+    ? runtimeSnapshot(contract, invocation.runtimeId)
+    : routeRuntimeForState(contract, node, state, invocation?.phase === "judge" ? "judge" : "worker");
+  const startedAt = Date.parse(invocation.startedAt);
+  const timeoutSec = latestTimeoutSec(state, node.timeoutSec ?? contract.timeoutSec);
+  const deadline = Number.isFinite(startedAt) && Number.isFinite(timeoutSec)
+    ? startedAt + timeoutSec * 1_000
+    : null;
+  if (deadline === null || !Number.isFinite(deadline)) {
+    if (invocationOwned(invocation)) await terminateInvocation(invocation, { runDir });
+    const terminal = invocationResult(invocation, runtime, { preferStructured: invocation.phase === "judge" });
+    if (terminal?.status === "done") {
+      if (invocation.phase === "judge") return adoptOrRejudgeJudge(state, contract, node, invocation, terminal);
+      return /** @type {RecoveryOutcome} */ ({ kind: "adopted", ...terminal, phase: invocation.phase, invocationId: invocation.id });
+    }
+    const persisted = invocation.phase === "worker"
+      ? persistedWorkerResult(runDir, state, invocation, settlement)
+      : persistedJudgeResult(state, invocation, settlement);
+    if (persisted) {
+      if (invocation.phase === "judge") {
+        return /** @type {RecoveryOutcome} */ ({ kind: "adopted", phase: "judge", result: persisted, invocationId: invocation.id });
+      }
+      return /** @type {RecoveryOutcome} */ ({ kind: "adopted", phase: "worker", result: persisted, invocationId: invocation.id });
+    }
+    return restartRecovery(invocation, terminal, `invocation ${invocation.id} has no reliable start time or timeout deadline`);
+  }
+  if (invocation.closedAt !== null && !Number.isFinite(Date.parse(invocation.closedAt))) {
+    if (invocationOwned(invocation)) await terminateInvocation(invocation, { runDir });
+    const terminal = invocationResult(invocation, runtime, { preferStructured: invocation.phase === "judge" });
+    if (terminal?.status === "done") {
+      if (invocation.phase === "judge") return adoptOrRejudgeJudge(state, contract, node, invocation, terminal);
+      return /** @type {RecoveryOutcome} */ ({ kind: "adopted", ...terminal, phase: invocation.phase, invocationId: invocation.id });
+    }
+    const persisted = invocation.phase === "worker"
+      ? persistedWorkerResult(runDir, state, invocation, settlement)
+      : persistedJudgeResult(state, invocation, settlement);
+    if (persisted) {
+      if (invocation.phase === "judge") {
+        return /** @type {RecoveryOutcome} */ ({ kind: "adopted", phase: "judge", result: persisted, invocationId: invocation.id });
+      }
+      return /** @type {RecoveryOutcome} */ ({ kind: "adopted", phase: "worker", result: persisted, invocationId: invocation.id });
+    }
+    return restartRecovery(invocation, terminal, `invocation ${invocation.id} has no reliable close time`);
+  }
+  if (invocation && invocationOwned(invocation)) {
+    if (Date.now() > deadline) {
+      await terminateInvocation(invocation, { runDir });
+      return restartRecovery(invocation, invocationResult(invocation, runtime), `${invocation.phase} invocation ${invocation.id} exceeded its wall-clock budget`);
+    }
+    // A resumed controller waiting on a live orphan is a bounded recovery, not
+    // progress. The heartbeat records that phase against the orphan's own
+    // deadline so the supervisor judges `until + grace` instead of a stale
+    // `lastProgressAt`; the poll must never refresh `lastProgressAt` itself, or
+    // an orphan that is alive but never advancing would look like progress.
+    const recoveryUntil = typeof invocation.deadlineAt === "string" ? invocation.deadlineAt : new Date(deadline).toISOString();
+    markRecovering(runDir, recoveryUntil);
+    while (invocationOwned(invocation) && Date.now() < deadline) {
+      const result = invocationResult(invocation, runtime, { preferStructured: invocation.phase === "judge" });
+      if (result?.status === "done") {
+        if (Date.now() > deadline || (invocation.closedAt !== null && Date.parse(invocation.closedAt) > deadline)) {
+          await terminateInvocation(invocation, { runDir });
+          return restartRecovery(invocation, result, `${invocation.phase} invocation ${invocation.id} completed after its wall-clock budget`);
+        }
+        await terminateInvocation(invocation, { runDir });
+        if (invocation.phase === "judge") return adoptOrRejudgeJudge(state, contract, node, invocation, result);
+        return /** @type {RecoveryOutcome} */ ({ kind: "adopted", ...result, phase: invocation.phase, invocationId: invocation.id });
+      }
+      await delay(Math.min(contract.pollIntervalMs, 250));
+      markRecovering(runDir, recoveryUntil);
+    }
+    const expired = Date.now() >= deadline;
+    if (invocationOwned(invocation)) await terminateInvocation(invocation, { runDir });
+    if (expired) return restartRecovery(invocation, null, `${invocation.phase} invocation ${invocation.id} exceeded its wall-clock budget`);
+    if (invocation.closedAt === null) {
+      const persisted = invocation.phase === "worker"
+        ? persistedWorkerResult(runDir, state, invocation, settlement)
+        : persistedJudgeResult(state, invocation, settlement);
+      if (persisted) return /** @type {RecoveryOutcome} */ ({ kind: "adopted", phase: invocation.phase === "judge" ? "judge" : "worker", result: persisted, invocationId: invocation.id });
+      return restartRecovery(invocation, null, `${invocation.phase} invocation ${invocation.id} has no reliable close time`);
+    }
+    if (invocation.phase === "judge") {
+      const result = invocationResult(invocation, runtime, { preferStructured: true, exitCode: invocation.exitCode, signal: invocation.signal });
+      if (result?.status === "done") return adoptOrRejudgeJudge(state, contract, node, invocation, result);
+      if (result?.status === "exhausted") return {
+        kind: "exhausted",
+        phase: "judge",
+        invocationId: invocation.id,
+        usage: result.usage,
+        costUsd: result.costUsd,
+        costProvenance: result.costProvenance,
+        exhaustedUntil: result.exhaustedUntil ?? null,
+        error: result.error,
+        reason: result.error?.message,
+      };
+      const persisted = persistedJudgeResult(state, invocation, settlement);
+      if (persisted) return /** @type {RecoveryOutcome} */ ({ kind: "adopted", phase: "judge", result: persisted, invocationId: invocation.id });
+      return rejudgeOrRestart(state, contract, node, invocation, result);
+    }
+  }
+  if (invocation) {
+    const closedAt = Date.parse(invocation.closedAt ?? "");
+    if (!Number.isFinite(closedAt) || closedAt > deadline) {
+      return restartRecovery(invocation, null, `${invocation.phase} invocation ${invocation.id} completed after its wall-clock budget`);
+    }
+    if (invocation.closedAt === null) {
+      return restartRecovery(invocation, null, `${invocation.phase} invocation ${invocation.id} has no reliable close time`);
+    }
+    const result = invocationResult(invocation, runtime, { preferStructured: invocation.phase === "judge", exitCode: invocation.exitCode, signal: invocation.signal });
+    if (invocation.phase === "judge") {
+      if (result?.status === "done") return adoptOrRejudgeJudge(state, contract, node, invocation, result);
+      if (result?.status === "exhausted") return {
+        kind: "exhausted",
+        phase: "judge",
+        invocationId: invocation.id,
+        usage: result.usage,
+        costUsd: result.costUsd,
+        costProvenance: result.costProvenance,
+        exhaustedUntil: result.exhaustedUntil ?? null,
+        error: result.error,
+        reason: result.error?.message,
+      };
+      return rejudgeOrRestart(state, contract, node, invocation, result);
+    }
+    if (result?.status === "done") return /** @type {RecoveryOutcome} */ ({ kind: "adopted", ...result, phase: invocation.phase, invocationId: invocation.id });
+    if (result?.status === "exhausted") return /** @type {RecoveryOutcome} */ ({ kind: "exhausted", ...result, phase: invocation.phase, invocationId: invocation.id, reason: result.error?.message });
+    const persisted = persistedWorkerResult(runDir, state, invocation, settlement);
+    if (persisted) return /** @type {RecoveryOutcome} */ ({ kind: "adopted", phase: "worker", result: persisted, invocationId: invocation.id });
+    return restartRecovery(invocation, result, `${invocation.phase} invocation ${invocation.id} died without a completed stream`);
+  }
+  return { kind: "restart", reason: `node ${state.id} died without a completed stream` };
+}
+/** @param {Invocation} invocation @param {PricedEnvelope|null} result @param {string} reason @returns {RecoveryOutcome} */
+function restartRecovery(invocation, result, reason) {
+  return /** @type {RecoveryOutcome} */ ({
+    kind: "restart",
+    phase: invocation.phase,
+    invocationId: invocation.id,
+    usage: result?.usage ?? invocation.usage,
+    costUsd: result?.costUsd ?? invocation.costUsd,
+    costProvenance: result?.costProvenance ?? invocation.costProvenance,
+    reason,
+  });
+}
+/**
+ * @param {Record<string, unknown>} override
+ * @param {string|undefined} invocationId
+ * @returns {RecoveryOutcome}
+ */
+export function recoveryFromOverride(override, invocationId) {
+  const decision = override.decision;
+  return /** @type {RecoveryOutcome} */ ({
+    kind: decision === "rejudge"
+      ? "rejudge"
+      : decision === "restart" || decision === "safe_replay"
+        ? "restart"
+        : decision === "reconciled"
+          ? "reconciled"
+          : "adopted",
+    phase: override.phase,
+    result: override.result,
+    usage: override.usage,
+    costUsd: override.costUsd,
+    costProvenance: override.costProvenance,
+    reason: override.reason,
+    invocationId,
+  });
+}
+/**
+ * @param {NodeSnapshot} state
+ * @param {ValidatedContract} contract
+ * @param {ValidatedNode} node
+ * @param {Invocation} judgeInvocation
+ * @param {PricedEnvelope|null} [judgeResult]
+ * @returns {RecoveryOutcome}
+ */
+function rejudgeOrRestart(state, contract, node, judgeInvocation, judgeResult = null) {
+  const workerInvocation = [...(state.invocations ?? [])].reverse().find((item) => item.phase === "worker");
+  const workerResult = workerInvocation && invocationResult(
+    workerInvocation,
+    workerInvocation.runtimeId ? runtimeSnapshot(contract, workerInvocation.runtimeId) : routeRuntimeForState(contract, node, state, "worker"),
+  );
+  if (workerResult?.status === "done") {
+    return /** @type {RecoveryOutcome} */ ({
+      kind: "rejudge",
+      phase: "worker",
+      result: workerResult.result,
+      usage: judgeResult?.usage,
+      costUsd: judgeResult?.costUsd,
+      costProvenance: judgeResult?.costProvenance,
+      invocationId: judgeInvocation.id,
+    });
+  }
+  return /** @type {RecoveryOutcome} */ ({
+    kind: "restart",
+    phase: "judge",
+    invocationId: judgeInvocation.id,
+    usage: judgeResult?.usage,
+    costUsd: judgeResult?.costUsd,
+    costProvenance: judgeResult?.costProvenance,
+    reason: `judge invocation ${judgeInvocation.id} completed but its worker stream is unavailable`,
+  });
+}
+/**
+ * @param {NodeSnapshot} state
+ * @param {ValidatedContract} contract
+ * @param {ValidatedNode} node
+ * @param {Invocation} judgeInvocation
+ * @param {PricedEnvelope} result
+ * @returns {RecoveryOutcome}
+ */
+function adoptOrRejudgeJudge(state, contract, node, judgeInvocation, result) {
+  try {
+    parseJudge(result.result ?? "");
+    return /** @type {RecoveryOutcome} */ ({
+      kind: "adopted",
+      phase: "judge",
+      result: result.result,
+      usage: result.usage,
+      costUsd: result.costUsd,
+      costProvenance: result.costProvenance,
+      invocationId: judgeInvocation.id,
+    });
+  } catch {
+    return rejudgeOrRestart(state, contract, node, judgeInvocation, result);
+  }
+}
+/**
+ * @param {Invocation[]|undefined} invocations
+ * @param {string|undefined} invocationId
+ * @param {Usage|undefined} [usage]
+ * @param {number|null|undefined} [costUsd]
+ * @param {"priced"|undefined} [costProvenance]
+ * @returns {Invocation[]}
+ */
+export function closePersistedInvocation(invocations, invocationId, usage = undefined, costUsd = undefined, costProvenance = undefined) {
+  return (invocations ?? []).map((invocation) => invocation.id === invocationId ? {
+    ...invocation,
+    status: "closed",
+    usage: usage ?? invocation.usage,
+    costUsd: costUsd ?? invocation.costUsd ?? null,
+    costProvenance: costProvenance ?? invocation.costProvenance,
+    closedAt: invocation.closedAt ?? new Date().toISOString(),
+    updatedAt: new Date().toISOString(),
+  } : invocation);
+}