faberun 0.9.0 → 0.11.0

package/src/engine/scheduler.mjs

@@ -31,15 +31,15 @@ import {
   terminalErrorCode,
 } from "./lifecycle.mjs";
 import { delay, errorCode } from "../util.mjs";
-import { alreadyNotified, notifyQueueFor, notifyQueuesByRun, renderCampaignHandoffSafely } from "./notify-queue.mjs";
-import { detectStalls, terminateProcess } from "./process.mjs";
+import { alreadyNotified, emitNodeAdvisories, notifyQueueFor, notifyQueuesByRun, renderCampaignHandoffSafely } from "./notify-queue.mjs";
+import { detectStalls, invocationAlive, terminateProcess } from "./process.mjs";
 import { transition, writeNode } from "./state.mjs";
 import { listNodeSnapshots, readNodeSnapshot } from "../run/node-store.mjs";
 import { render, renderFinalReport, writeFindingsArtifact } from "../report/final.mjs";
 import { operationNextState, providerReceipts, settleInvocation } from "../run/operations.mjs";
 import { appendUsageRecord, invocationCost, invocationUsage, recordInvocationUsage } from "../run/usage.mjs";
 import { captureNodeScopeBoundaries, checkWorkerScope, emptyScope } from "./scope.mjs";
-import { validateContract } from "../contract/index.mjs";
+import { validateContractForLaunch } from "../campaign/chain.mjs";
 import { validateNodeSnapshot } from "../contract/snapshot.mjs";
 import { finalVerificationCommands, sharedVerificationCommands } from "../contract/final-verification.mjs";
 import { startJudge, startWorker } from "./dispatch.mjs";
@@ -131,16 +131,22 @@ export function nodeBudgetBasisMs(contract, node) {
  * never touches phase 2's parked `blocked`/`failed`/`exhausted`/`stalled`
  * states, and never overwrites the `runtime_tier_exhausted` waiting shape.
  *
+ * A node whose closed job is being settled in the background (its id is a key
+ * of `pendingSettlements`) is not this dead end either: its invocation has
+ * already exited and left `running`, but the settlement promise still owns
+ * deciding what happens to it, so it is left alone until that promise resolves.
+ *
  * @param {string} runDir
  * @param {Map<string, NodeSnapshot>} states
  * @param {Map<string, Job>} running
  * @param {LockHandle|null} lock
+ * @param {Map<string, Promise<void>>} [pendingSettlements]
  * @returns {string[]} the node ids this pass parked
  */
-export function enforceRunningInvariant(runDir, states, running, lock) {
+export function enforceRunningInvariant(runDir, states, running, lock, pendingSettlements = new Map()) {
   const parked = [];
   for (const [nodeId, state] of states) {
-    if (state.status !== "running" || running.has(nodeId)) continue;
+    if (state.status !== "running" || running.has(nodeId) || pendingSettlements.has(nodeId)) continue;
     transition(runDir, state, "blocked", {
       phase: state.phase,
       error: {
@@ -175,14 +181,17 @@ function renderFingerprint(states) {
 
 /**
  * @param {string} contractPath
- * @param {{detachedBootstrap?: boolean}} [options] `detachedBootstrap` is set
- *   only by the CLI entry when this process is its own detached child, and
- *   makes the controller wait for the launcher's acknowledgement
+ * @param {{detachedBootstrap?: boolean, baseRef?: string}} [options]
+ *   `detachedBootstrap` is set only by the CLI entry when this process is its
+ *   own detached child, and makes the controller wait for the launcher's
+ *   acknowledgement; `baseRef` is the CLI's own `--base-ref`, re-validated
+ *   here so a launch and the run it starts agree about what the contract was
+ *   checked against
  * @returns {Promise<RunOutcome>}
  */
 export async function runContract(contractPath, options = {}) {
   const absoluteContractPath = resolve(contractPath);
-  const contract = validateContract(JSON.parse(readFileSync(absoluteContractPath, "utf8")), absoluteContractPath);
+  const contract = validateContractForLaunch(JSON.parse(readFileSync(absoluteContractPath, "utf8")), absoluteContractPath, { baseRef: options.baseRef });
   const runDir = join(contract.cwd, ".runs", contract.id);
   if (existsSync(runDir)) throw new Error(`run already exists: ${runDir}`);
   mkdirSync(join(contract.cwd, ".runs"), { recursive: true });
@@ -319,12 +328,13 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
     statusFingerprint = fingerprint;
     render(runDir, runsDir, contract, states, renderLock);
   };
-  // The tick that owns a running node's verification can be minutes long
-  // (`executeControllerVerification` is awaited on the critical path below),
-  // and that whole time status.json would otherwise report whatever the last
-  // tick left it at. A timer renders between ticks too; it is cheap even when
-  // idle because `renderFingerprint` still change-detects, so a quiet run
-  // writes nothing extra.
+  // A node's own settlement (its controller verification, its judge round) can
+  // be minutes long, and it now runs off the tick's critical path in
+  // `pendingSettlements` so dispatch never waits behind it -- but that whole
+  // time status.json would otherwise report whatever the last tick left it at.
+  // A timer renders between ticks too; it is cheap even when idle because
+  // `renderFingerprint` still change-detects, so a quiet run writes nothing
+  // extra.
   const statusTimer = setInterval(() => renderStatusIfChanged(), contract.pollIntervalMs);
   statusTimer.unref();
   let handoffFingerprint = statesFingerprint(states);
@@ -364,14 +374,31 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
 
   /** @type {Map<string, Job>} */
   const running = new Map();
+  // One promise per node currently settling a closed job -- its controller
+  // verification, its candidate verification, its judge round -- kept off the
+  // tick's critical path so an eligible sibling still dispatches into a free
+  // slot while this node's own invocation has already exited. A node's own
+  // steps stay ordered because only one settlement per node is ever in flight
+  // (see the dispatch loop below); a *different* node's settlement is queued
+  // behind whichever one is already running (`settlementQueue`), not run
+  // alongside it -- `finalizeClosedJobs` shares state a concurrent second call
+  // would corrupt: the phase-continuation selection that picks at most one
+  // live node to carry a session forward, and `repo/integrate.mjs`'s one
+  // candidate ref and worktree per run. Only *dispatching a sibling* skips
+  // ahead of a node's settlement; two nodes' settlements never interleave.
+  /** @type {Map<string, Promise<void>>} */
+  const pendingSettlements = new Map();
+  /** @type {Promise<void>} */
+  let settlementQueue = Promise.resolve();
   let canceled = false;
   const cancel = () => { canceled = true; };
   process.once("SIGINT", cancel);
   process.once("SIGTERM", cancel);
   process.once("SIGHUP", cancel);
   // The heartbeat's `at` is owned by an unref'd timer inside this writer, never
-  // by the loop body below: the loop awaits controller verification on its own
-  // critical path and must keep answering "the process is alive" while it does.
+  // by the loop body below: a node's settlement can still run for minutes off
+  // the tick's critical path (`pendingSettlements`), and the heartbeat must
+  // keep answering "the process is alive" while it does.
   const heartbeat = createHeartbeat({ runDir, intervalMs: HEARTBEAT_INTERVAL_MS });
   const heartbeatNodes = new Map(contract.nodes.map((node) => [node.id, node]));
   let heartbeatFingerprint = statesFingerprint(states);
@@ -380,11 +407,89 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
   const activeHeartbeatNodes = () => [...states.values()]
     .filter((state) => state.status === "running" && heartbeatNodes.has(state.id))
     .map((state) => ({ nodeId: state.id, budgetBasis: nodeBudgetBasisMs(contract, /** @type {ValidatedNode} */ (heartbeatNodes.get(state.id))) }));
+  // A programmer error surfacing inside a background settlement must still
+  // crash the whole run, exactly as an unguarded `await finalizeClosedJobs`
+  // used to: it is recorded here and thrown from the top of the loop on the
+  // very next tick, rather than immediately, so it cannot itself become the
+  // block a sibling's dispatch is waiting behind. A lost lock is not this --
+  // the loop's own `lock.assert()` calls surface that same condition on their
+  // own schedule, so it is left for them.
+  /** @type {unknown} */
+  let backgroundSettlementFailure = null;
+  // Mark every job that closed this tick as settling, and free its slot,
+  // without waiting for any of them: that alone is what lets an eligible
+  // sibling dispatch into the freed slot while this node's minutes-long
+  // controller verification or judge round is still running. The actual
+  // settlement work is chained onto `settlementQueue`, one node at a time in
+  // the order its job closed, so it still runs exactly as serialized against
+  // every *other* node's settlement as it did when this loop awaited
+  // `finalizeClosedJobs` directly -- `finalizeClosedJobs` runs against a
+  // one-entry map per node, so this is one call per node rather than the one
+  // batched call it used to be, but the chain still runs them one at a time.
+  // A settlement may itself dispatch the node's next phase (a judge, a
+  // revision) through the same `startJudge`/`startWorker` calls dispatch below
+  // uses; those land in `slot`, not the real `running`, so they are copied
+  // back into `running` in a `finally` -- unconditionally, win or lose, so a
+  // job a settlement started before failing (a lost lock, a programmer error)
+  // is still visible to the cleanup sweeps below rather than leaked. The
+  // node's own steps stay ordered by never starting a second settlement for a
+  // node whose first has not yet cleared `pendingSettlements`.
+  const settleClosedJobsInBackground = () => {
+    for (const [nodeId, job] of [...running]) {
+      if (pendingSettlements.has(nodeId) || !job.closed || invocationAlive(job.invocation)) continue;
+      running.delete(nodeId);
+      const slot = new Map([[nodeId, job]]);
+      const settlement = settlementQueue
+        .then(() => finalizeClosedJobs(contract, runDir, states, slot, lock, campaign.path))
+        .finally(() => {
+          for (const [settledId, settledJob] of slot) running.set(settledId, settledJob);
+          pendingSettlements.delete(nodeId);
+        });
+      // The queue itself must never reject -- a rejected settlement (a lost
+      // lock, a programmer error) would otherwise wedge every node queued
+      // behind it. The rejection still reaches whoever awaits the real
+      // `settlement` promise (`pendingSettlements`, below).
+      settlementQueue = settlement.catch(() => {});
+      // Handled here so an in-flight settlement never becomes an unhandled
+      // rejection when nobody happens to await `pendingSettlements` before the
+      // process exits; the original promise, still held below, carries the
+      // rejection to whichever checkpoint (cancel, shutdown) awaits it.
+      settlement.catch((error) => {
+        if (!(error instanceof LockLostError) && backgroundSettlementFailure === null) backgroundSettlementFailure = error;
+      });
+      pendingSettlements.set(nodeId, settlement);
+    }
+  };
+  // Captured once, before the loop, rather than re-derived every tick: it is
+  // "parked when this controller invocation started" (autoRetryParkedNodes's
+  // own contract), and a node a background settlement parks between two ticks
+  // -- rather than synchronously within one, as it always did before
+  // settlement moved off the tick's critical path -- must still read as newly
+  // parked whichever later tick first observes it, not as already-parked
+  // because a per-tick snapshot happened to be taken after it landed.
+  const parkedBefore = new Set([...states.values()].filter((state) => PARKED.has(state.status)).map((state) => state.id));
+  const anyUnsettled = () => [...states.values()].some((state) => !SETTLED.has(state.status));
   try {
-    while ([...states.values()].some((state) => !SETTLED.has(state.status))) {
+    // A `while` that re-checked this at the very top of every tick would exit
+    // the instant a background settlement flips the run's last unsettled node
+    // straight to a terminal status between two ticks -- before the tick body
+    // that would have run `autoRetryParkedNodes` against that new status ever
+    // gets to. The entry guard skips the loop entirely when there is nothing
+    // to do at all (a resume of an already-settled run still does zero
+    // iterations); once inside, the exit check moves to the bottom, after the
+    // body, so that body always sees a freshly-parked node at least once
+    // before the loop is allowed to end.
+    if (anyUnsettled()) for (;;) {
       lock.assert();
+      if (backgroundSettlementFailure !== null) throw backgroundSettlementFailure;
       if (existsSync(join(runDir, "cancel.request.json"))) canceled = true;
       if (canceled) {
+        // A settlement already in flight owns the one decision a cancellation
+        // must not race: what this node's own invocation resolved to. Let it
+        // land on its real terminal status (and, when it re-dispatched a judge
+        // or a revision, on the new job that landed in `running`) before this
+        // branch decides which nodes are merely canceled.
+        await Promise.all([...pendingSettlements.values()]);
         const jobs = [...running.values()];
         await Promise.all(jobs.map((job) => terminateProcess(job)));
         const envelopes = new Map();
@@ -408,8 +513,14 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
         break;
       }
 
-      const parkedBefore = new Set([...states.values()].filter((state) => PARKED.has(state.status)).map((state) => state.id));
-      await finalizeClosedJobs(contract, runDir, states, running, lock, campaign.path);
+      // Advisory spend lines are checked every tick, closed job or not: a
+      // crossing must be visible while the spend is happening on a node still
+      // running, not only once it closes. `finalizeClosedJobs` used to open
+      // with this same check; calling it once here, rather than once per
+      // node settled this tick, is what keeps it at one pass per tick now
+      // that settlement runs per node in `settleClosedJobsInBackground`.
+      await emitNodeAdvisories(contract, runDir, states);
+      settleClosedJobsInBackground();
       await detectStalls(contract, running, async (job, status, error) => {
         const envelope = recordInvocationUsage(job);
         job.state.usage = invocationUsage(job.state);
@@ -450,8 +561,11 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
         writeNode(runDir, job.state, lock);
       });
       // A node left `running` with no job is a dead end; park it before the
-      // dispatch pass so it cannot hide behind a healthy sibling.
-      enforceRunningInvariant(runDir, states, running, lock);
+      // dispatch pass so it cannot hide behind a healthy sibling. A node
+      // whose closed job is settling in the background is not that dead end
+      // -- `pendingSettlements` is what tells this apart from one truly
+      // abandoned.
+      enforceRunningInvariant(runDir, states, running, lock, pendingSettlements);
       // Before dependants are blocked, a node that parked on this tick gets
       // its one automatic retry: it becomes pending, so `blockDependents` sees
       // nothing to block and the dependants stay `pending`/`phase: "waiting"`
@@ -463,14 +577,37 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
       if (slots > 0) {
         const ready = contract.nodes.filter((node) => {
           const state = states.get(node.id);
-          return state?.status === "pending" && node.dependsOn.every((id) => states.get(id)?.status === "done");
+          return state?.status === "pending" && !pendingSettlements.has(node.id)
+            && node.dependsOn.every((id) => states.get(id)?.status === "done");
         });
         for (const node of ready.slice(0, slots)) {
           const state = states.get(node.id);
           if (!state || routingBackoffActive(state, state.phase)) continue;
+          // A node recovered pending a re-ask judge (its own worker attempt
+          // already accepted, `state.result` durable) reaches `settleDone` /
+          // `integrateAttempt` exactly like a closed job's own settlement
+          // does, on the same per-run candidate ref and worktree
+          // `settlementQueue` exists to serialize -- so it is dispatched the
+          // same way: chained onto the queue rather than awaited here, using
+          // its own one-entry `slot` merged back into `running` in a
+          // `finally`. The node's own order is untouched (still one entry at
+          // a time, gated by `pendingSettlements`); only the tick stops
+          // waiting behind it.
           if (state.phase === "judge" && state.result) {
-            await applyJudgeRound(await startJudge(contract, node, state, runDir, running, state.result, lock, states, campaign.path),
-              contract, node, state, runDir, running, lock, states, campaign.path, state.result);
+            const workerResult = state.result;
+            const slot = new Map();
+            const settlement = settlementQueue
+              .then(() => startJudge(contract, node, state, runDir, slot, workerResult, lock, states, campaign.path))
+              .then((round) => applyJudgeRound(round, contract, node, state, runDir, slot, lock, states, campaign.path, workerResult))
+              .finally(() => {
+                for (const [settledId, settledJob] of slot) running.set(settledId, settledJob);
+                pendingSettlements.delete(node.id);
+              });
+            settlementQueue = settlement.catch(() => {});
+            settlement.catch((error) => {
+              if (!(error instanceof LockLostError) && backgroundSettlementFailure === null) backgroundSettlementFailure = error;
+            });
+            pendingSettlements.set(node.id, settlement);
             continue;
           }
           state.attempt += 1;
@@ -499,10 +636,46 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
         }
       }
       heartbeat.setActive(activeHeartbeatNodes());
-      if ([...states.values()].some((state) => !SETTLED.has(state.status))) await delay(contract.pollIntervalMs);
+      // A background settlement can flip a node straight to a parked
+      // terminal status during this same tick's own later awaits
+      // (`detectStalls`, `notifyStateChanges`), after the auto-retry pass
+      // above already ran and saw it as not-yet-parked. Running it once more
+      // here, immediately before the exit check, is what stops the loop from
+      // mistaking that freshly-parked node for settled and exiting before it
+      // ever got its automatic retry; a node it reopens dispatches on the
+      // next tick rather than this one, which `anyUnsettled()` below still
+      // correctly keeps the loop alive for.
+      autoRetryParkedNodes(contract, runDir, states, lock, parkedBefore);
+      if (!anyUnsettled()) break;
+      await delay(contract.pollIntervalMs);
     }
+    // The loop above exits (by the bottom check above or the cancel branch's
+    // `break`) the instant every node's state looks settled, but a
+    // settlement's own trailing work -- sealing a candidate's acceptance,
+    // removing its worktree, enqueueing its terminal notification -- can
+    // still be running after the transition that made the state look
+    // terminal. Nothing past this point (the final render, the findings
+    // artifact, the run-terminal notification, releasing the lock) may run
+    // ahead of that trailing work.
+    await Promise.all([...pendingSettlements.values()]);
+    // A settlement's own `finally` deletes it from `pendingSettlements` the
+    // instant it settles, win or lose -- so a rejection recorded here can
+    // already be gone from the map above by the time this line runs, with
+    // nothing left to await it. The loop's own top-of-tick check
+    // (`if (backgroundSettlementFailure !== null) throw ...`) cannot save
+    // that case either: the loop has already exited. Checked again here,
+    // once, so a background settlement failure can never read as a clean
+    // finish just because it settled on the same tick the run's last node
+    // did.
+    if (backgroundSettlementFailure !== null) throw backgroundSettlementFailure;
   } catch (error) {
     if (!(error instanceof LockLostError)) throw error;
+    // A settlement still merges whatever it started into `running` in its own
+    // `finally`, win or lose, regardless of whether anything awaits it; wait
+    // for that to land (never rejecting itself, so a second lock-loss here
+    // cannot mask the one already being handled) before the termination sweep
+    // reads `running`.
+    await Promise.allSettled([...pendingSettlements.values()]);
     await Promise.all([...running.values()].map((job) => terminateProcess(job)));
     notifyQueuesByRun.delete(runDir);
     return { runDir, states, ok: false, error };

package/src/engine/settle-judge.mjs

@@ -0,0 +1,141 @@
+/**
+ * The judge branch of `finalizeClosedJobs`, and the workspace comparison its
+ * fail-closed write check depends on. Split out of `lifecycle.mjs` for the
+ * same reason `engine/settle.mjs` was: `lifecycle.mjs` and
+ * `test/engine/judge.test.mjs` both sit on the 800-line ceiling
+ * `test/repo/source-shape.test.mjs` enforces.
+ *
+ * `settleJudgeRound` runs only after `lifecycle.mjs` has already called
+ * `judgeWorkspaceWriteViolation` and found nothing: the check has to run
+ * before any branch here (a failed provider, a bounded re-dispatch, or a
+ * verdict) can adopt or launder a judge's own write, so it cannot live inside
+ * this function without reintroducing the escape it closes.
+ *
+ * `clearTierExhaustion` and `handleProviderExhaustion` are `lifecycle.mjs`'s
+ * own -- importing them back from there would recreate the exact
+ * `lifecycle.mjs` <-> `review.mjs` cycle `test/repo/source-shape.test.mjs`
+ * once caught and `AGENTS.md` records as fixed, so the caller passes them in
+ * instead.
+ */
+import { judgeReaskOutstanding } from "./judge-gate.mjs";
+import { judgeVerdictEvidence } from "../contract/review-modes.mjs";
+import {
+  JUDGE_MAX_FAILURES,
+  applyJudgeProtocolFailure,
+  applyJudgeResult,
+  applyJudgeRound,
+  settleUnavailableJudge,
+} from "./review.mjs";
+import { networkTransition } from "./backoff.mjs";
+import { startJudge } from "./dispatch.mjs";
+import { writeNode } from "./state.mjs";
+import { compareWorkspaceSnapshot } from "../repo/workspace.mjs";
+import { errorCode, errorMessage, excerpt } from "../util.mjs";
+
+/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
+/** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
+/** @typedef {import("./process.mjs").Job} Job */
+/** @typedef {import("../run/lock.mjs").LockRecord} LockRecord */
+/** @typedef {ReturnType<typeof import("../run/lock.mjs").acquire>} LockHandle */
+/** @typedef {import("../harnesses/index.mjs").ProviderEnvelope} ProviderEnvelope */
+/** @typedef {ProviderEnvelope & {costProvenance?: "priced"}} PricedEnvelope */
+
+/**
+ * Whether a closed judge job wrote into its own workspace, or a workspace
+ * comparison error that means the same thing: either is a violation, never
+ * silence. `compareWorkspaceSnapshot` throws `snapshot_ignore_changed` when
+ * the judge edits an ignore source, `snapshot_symlink_escape` when it creates
+ * a symlink out of the tree, and `snapshot_too_large` when it adds enough
+ * entries -- all ordinary judge writes that must fail closed exactly like the
+ * worker path (`engine/scope.mjs`'s `checkWorkerScope`), never vanish into a
+ * "no writes" result. `job.scopeBaseline` is null only when the capture at
+ * dispatch time itself failed -- `startJudge` degrades to unchecked rather
+ * than refuse to run the judge at all -- and that is the one case with
+ * nothing to compare, not a violation.
+ *
+ * @param {Job} job
+ * @returns {{message: string}|null}
+ */
+export function judgeWorkspaceWriteViolation(job) {
+  const baseline = job.scopeBaseline;
+  if (!baseline) return null;
+  try {
+    const { unexpectedPaths } = compareWorkspaceSnapshot(/** @type {import("../repo/workspace.mjs").WorkspaceSnapshot} */ (baseline), job.cwd);
+    if (!unexpectedPaths.length) return null;
+    return { message: excerpt(`judge wrote into its own workspace (${unexpectedPaths.length}): ${unexpectedPaths.slice(0, 8).join(", ")}`) ?? "judge wrote into its own workspace" };
+  } catch (error) {
+    return { message: excerpt(`judge workspace comparison failed (${errorCode(error) ?? "scope_snapshot_invalid"}): ${errorMessage(error)}`) ?? "judge workspace comparison failed" };
+  }
+}
+
+/**
+ * Settle a closed judge invocation whose write check already passed: a
+ * provider that failed outright gets one bounded re-dispatch, then review-mode
+ * settlement; anything else that is not exactly one usable verdict (none at
+ * all, several of them, an unparseable one, a stream cut off before its
+ * terminal envelope, or a phase killed on its wall clock) takes the same
+ * bounded re-ask; a clean verdict is applied.
+ *
+ * @param {ValidatedContract} contract
+ * @param {Job} job
+ * @param {NodeSnapshot} state
+ * @param {string} runDir
+ * @param {Map<string, Job>} running
+ * @param {LockHandle} lock
+ * @param {Map<string, NodeSnapshot>} states
+ * @param {string} campaignPath
+ * @param {PricedEnvelope} envelope
+ * @param {{clearTierExhaustion: (state: NodeSnapshot) => void, handleProviderExhaustion: typeof import("./lifecycle.mjs").handleProviderExhaustion}} hooks
+ *   `lifecycle.mjs`'s own tier-exhaustion clear and provider-exhaustion router, passed in rather than imported back.
+ * @returns {Promise<void>}
+ */
+export async function settleJudgeRound(contract, job, state, runDir, running, lock, states, campaignPath, envelope, { clearTierExhaustion, handleProviderExhaustion }) {
+  const node = job.node;
+  // A judge provider that failed outright (its turn died, its tool host was
+  // gone) is a provider failure, never a verdict: the gate cannot adopt a
+  // result the judge could not ground in inspection. Re-dispatch the judge
+  // once on the same routing, then settle by review mode so a judge failure
+  // is surfaced, never silently settled. A stream that never reached its
+  // terminal envelope is a protocol defect instead and takes the bounded
+  // re-ask below.
+  if (envelope.status === "failed" && envelope.error?.code !== "incomplete_stream") {
+    // A judge that lost its socket is not an unavailable judge. It buys the
+    // same bounded network waits a worker does, on the runtime it already
+    // warmed, and spends none of the one re-dispatch counted below.
+    const network = networkTransition(contract, node, state, "judge", envelope, job.exitCode);
+    if (network && handleProviderExhaustion(contract, runDir, node, state, "judge", envelope, job.runtime.id, lock, states, campaignPath, network)) return;
+    clearTierExhaustion(state);
+    // The provider died on the bounded re-ask itself, so the one permitted
+    // re-ask is spent: settle by review mode here rather than dispatch a
+    // third judge invocation behind a fresh failure count.
+    if (judgeReaskOutstanding(state)) {
+      await applyJudgeProtocolFailure(contract, node, state, runDir, running, lock, states, campaignPath, envelope.error?.message ?? "judge provider failed");
+      return;
+    }
+    state.judgeFailures = (state.judgeFailures ?? 0) + 1;
+    if (state.judgeFailures < JUDGE_MAX_FAILURES) {
+      writeNode(runDir, state, lock);
+      await applyJudgeRound(await startJudge(contract, node, state, runDir, running, state.result, lock, states, campaignPath),
+        contract, node, state, runDir, running, lock, states, campaignPath, state.result);
+      return;
+    }
+    await settleUnavailableJudge(contract, node, state, runDir, lock, states, campaignPath, envelope.error?.message ?? "judge provider failed");
+    return;
+  }
+  // Whatever else this invocation produced, it is not exactly one usable
+  // verdict: no verdict at all, several of them in separate agent messages,
+  // an unparseable one, a stream cut off before its terminal envelope, or a
+  // phase killed on its wall clock. One bounded re-ask, then the review mode
+  // decides — advisory completes, blocking enters attention with the work
+  // preserved so a retry in place can re-judge it.
+  const evidence = judgeVerdictEvidence(envelope);
+  if (!evidence.ok) {
+    const network = networkTransition(contract, node, state, "judge", envelope, job.exitCode);
+    if (network && handleProviderExhaustion(contract, runDir, node, state, "judge", envelope, job.runtime.id, lock, states, campaignPath, network)) return;
+    clearTierExhaustion(state);
+    await applyJudgeProtocolFailure(contract, node, state, runDir, running, lock, states, campaignPath, evidence.reason);
+    return;
+  }
+  clearTierExhaustion(state);
+  await applyJudgeResult(contract, node, state, evidence.result, runDir, lock, running, states, campaignPath);
+}

package/src/engine/settle.mjs

@@ -165,7 +165,7 @@ export async function settleDone(contract, node, state, runDir, lock, states, ca
     attemptSha: sealed.sha,
     branch: state.worktree.branch,
     verificationEvidence: state.verification,
-    verifyCandidate: (candidateWorkspace) => verifyCandidateWorkspace(contract, node, state, runDir, candidateWorkspace),
+    verifyCandidate: (candidateWorkspace) => verifyCandidateWorkspace(contract, node, state, runDir, candidateWorkspace, lock),
     onAccepted: async (transaction) => {
       const acceptedPath = state.worktree?.path ?? attemptWorktreePath(runDir, contract.id, node.id, transaction.attempt);
       if (state.attempt === transaction.attempt && state.status !== "done") {

package/src/engine/verify.mjs

@@ -11,8 +11,10 @@
 import { attemptWorkspace } from "../repo/worktree.mjs";
 import { boundedUtf8, errorMessage } from "../util.mjs";
 import { compactVerification } from "../contract/verification.mjs";
-import { finalVerificationCommands, sharedVerificationCommands } from "../contract/final-verification.mjs";
+import { finalVerificationCommands, phaseTerminalNode, sharedVerificationCommands } from "../contract/final-verification.mjs";
 import { join } from "node:path";
+import { listNodeSnapshots, readNodeSnapshot } from "../run/node-store.mjs";
+import { SETTLED } from "./prompts.mjs";
 import { terminateInvocation } from "./process.mjs";
 import { writeNode } from "./state.mjs";
 import { runVerification } from "./run-command.mjs";
@@ -28,6 +30,34 @@ import { runVerification } from "./run-command.mjs";
 /** @typedef {import("../contract/index.mjs").VerificationState} VerificationState */
 /** @typedef {{index: number, total: number, argv: string}} VerificationProgress */
 
+/**
+ * The sibling phase-terminal node ids already settled, read straight off
+ * disk: every node snapshot is persisted from the run's first tick (see
+ * `runContract`), so a node still `pending` reads back honestly, not as
+ * missing. `finalVerificationCommands` uses this to decide, at the moment a
+ * phase-terminal node's own verification runs, whether it is the one that
+ * closes the phase -- recomputed fresh on every call, so a retried node sees
+ * its siblings' current status each time, not a decision frozen from an
+ * earlier attempt.
+ *
+ * @param {string} runDir
+ * @param {ValidatedContract} contract
+ * @param {ValidatedNode} node
+ * @returns {Set<string>}
+ */
+function settledSiblingIds(runDir, contract, node) {
+  const ids = new Set();
+  for (const name of listNodeSnapshots(runDir)) {
+    const id = name.slice(0, -".json".length);
+    if (id === node.id) continue;
+    const sibling = contract.nodes.find((candidate) => candidate.id === id);
+    if (!sibling || !phaseTerminalNode(contract, sibling)) continue;
+    const snapshot = /** @type {{status?: string}} */ (readNodeSnapshot(runDir, id));
+    if (SETTLED.has(/** @type {string} */ (snapshot.status))) ids.add(id);
+  }
+  return ids;
+}
+
 /**
  * The bounded `k/n · argv` shape a running node's status surfaces while a
  * verification command is in flight: the command's 1-based position among
@@ -108,7 +138,7 @@ export async function executeControllerVerification(contract, runDir, node, stat
   };
   writeNode(runDir, state, lock);
   const workspace = attemptWorkspace(state) ?? contract.cwd;
-  const commands = [...node.taskPacket.verification, ...sharedVerificationCommands(contract), ...finalVerificationCommands(contract, node)];
+  const commands = [...node.taskPacket.verification, ...sharedVerificationCommands(contract), ...finalVerificationCommands(contract, node, settledSiblingIds(runDir, contract, node))];
   /** @param {VerificationAttempt} attempt @returns {VerificationProgress} */
   const progressFor = (attempt) => verificationProgress(attempt.commandIndex + 1, commands.length, /** @type {VerificationCommand|undefined} */ (commands[attempt.commandIndex])?.argv);
   try {
@@ -221,16 +251,37 @@ function argvEqual(a, b) {
   if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false;
   return a.every((item, index) => item === b[index]);
 }
+/**
+ * Marks whether the integration candidate's own verification pass is running
+ * right now, nested inside the attempt's own already-completed verification
+ * record rather than as a new node-snapshot field: that record's validator
+ * (`validateVerificationSnapshot`) accepts extra keys, where the node
+ * snapshot's own strict field list would reject one. Status surfaces read it
+ * to tell "still verifying the sealed candidate" apart from "waiting on the
+ * judge", which otherwise both read as whatever phase the attempt last
+ * dispatched under.
+ *
+ * @param {string} runDir
+ * @param {NodeSnapshot} state
+ * @param {LockHandle|null} lock
+ * @param {boolean} active
+ */
+function markCandidateVerification(runDir, state, lock, active) {
+  state.verification = /** @type {VerificationState} */ ({ ...(state.verification ?? { passed: false, commands: [], completed: false }), candidate: active });
+  writeNode(runDir, state, lock);
+}
 /**
  * @param {ValidatedContract} contract
  * @param {ValidatedNode} node
  * @param {NodeSnapshot} state
  * @param {string} runDir
  * @param {string} workspace
+ * @param {LockHandle|null} [lock]
  * @returns {Promise<import("../repo/integrate.mjs").CandidateEvidence>}
  */
-export async function verifyCandidateWorkspace(contract, node, state, runDir, workspace) {
-  const commands = [...node.taskPacket.verification, ...sharedVerificationCommands(contract), ...finalVerificationCommands(contract, node)];
+export async function verifyCandidateWorkspace(contract, node, state, runDir, workspace, lock = null) {
+  const commands = [...node.taskPacket.verification, ...sharedVerificationCommands(contract), ...finalVerificationCommands(contract, node, settledSiblingIds(runDir, contract, node))];
+  markCandidateVerification(runDir, state, lock, true);
   try {
     const result = await runVerification(commands, workspace, {
       logDir: join(runDir, "logs", `${node.id}.${state.attempt}.candidate-verification`),
@@ -252,5 +303,7 @@ export async function verifyCandidateWorkspace(contract, node, state, runDir, wo
     return settled.retried ? { ...compacted, retried: settled.retried } : compacted;
   } catch (error) {
     return { passed: false, error: boundedUtf8(errorMessage(error), 4 * 1024) };
+  } finally {
+    markCandidateVerification(runDir, state, lock, false);
   }
 }