faberun 0.8.0 → 0.10.0

package/src/contract/final-verification.mjs

@@ -3,11 +3,13 @@
  * rule 12).
  *
  * `contract.finalVerification` is the contract-wide proof that the phase as a
- * whole closes: the controller runs it before the judge on the phase-terminal
- * node (the node no other node depends on), so no final checkpoint is ever
- * approved on partial verification. `contract.sharedVerification` carries the
- * same command schema but is appended to every node's attempt and integration
- * candidate, for the fast repository ratchets a node's write set can break.
+ * whole closes: the controller runs it before the judge, once, on whichever
+ * phase-terminal node (a node no other node depends on) is the last of them
+ * to settle, so no final checkpoint is ever approved on partial verification
+ * and a flake on one phase-terminal node cannot also cost its siblings a
+ * revision. `contract.sharedVerification` carries the same command schema but
+ * is appended to every node's attempt and integration candidate, for the fast
+ * repository ratchets a node's write set can break.
  * The persisted node-snapshot shape for verification evidence lives here too,
  * next to the schema it records.
  */
@@ -58,20 +60,49 @@ export function sharedVerificationCommands(contract) {
   return contract.sharedVerification ?? [];
 }
 
+/**
+ * Whether no other node in the contract depends on this one. `finalVerification`
+ * is a candidate to run on a phase-terminal node only; a node with a dependant
+ * never carries it, no matter which of its siblings settles last.
+ *
+ * @param {{nodes: {id: string, dependsOn: string[]}[]}} contract
+ * @param {{id: string}} node
+ * @returns {boolean}
+ */
+export function phaseTerminalNode(contract, node) {
+  return !contract.nodes.some((candidate) => candidate.dependsOn.includes(node.id));
+}
+
 /**
  * The contract's `finalVerification` commands when this node is the one that
- * closes the phase, otherwise none. A node is phase-terminal when no other
- * node in the contract depends on it.
+ * closes the phase, otherwise none. A phase can have several phase-terminal
+ * nodes (several nodes no other node depends on); the suite runs once for the
+ * phase, on whichever of them is the last to settle, not on every one of them.
+ *
+ * `settledIds` is the set of sibling node ids (any status a node does not
+ * leave on its own -- see `SETTLED` in `engine/prompts.mjs`) already reached
+ * at the moment this call is made. A phase-terminal node carries the suite
+ * exactly when every *other* phase-terminal node is already in that set --
+ * whichever node that is true for, in whatever order nodes actually settle,
+ * including a node retried after every sibling has already closed. Omitting
+ * `settledIds` (the scheduler's budget estimate, which has no run to inspect)
+ * falls back to the old per-node-shape answer: any phase-terminal node may
+ * still turn out to be the one that carries it, so the estimate stays an
+ * upper bound.
  *
  * @param {{finalVerification?: VerificationCommand[], nodes: {id: string, dependsOn: string[]}[]}} contract
  * @param {{id: string}} node
+ * @param {Set<string>} [settledIds]
  * @returns {VerificationCommand[]}
  */
-export function finalVerificationCommands(contract, node) {
+export function finalVerificationCommands(contract, node, settledIds) {
   const commands = contract.finalVerification ?? [];
   if (commands.length === 0) return [];
-  const hasDependant = contract.nodes.some((candidate) => candidate.dependsOn.includes(node.id));
-  return hasDependant ? [] : commands;
+  if (!phaseTerminalNode(contract, node)) return [];
+  if (!settledIds) return commands;
+  const siblings = contract.nodes.filter((candidate) => candidate.id !== node.id && phaseTerminalNode(contract, candidate));
+  const isLastToClose = siblings.every((sibling) => settledIds.has(sibling.id));
+  return isLastToClose ? commands : [];
 }
 
 /**

package/src/contract/task-packet.mjs

@@ -432,7 +432,7 @@ function renderDiscoveryPrompt(packet, nodeId) {
     ...bulletOrNone(packet.nonGoals),
     "",
     "## Required output",
-    'Return exactly one worker-result JSON object, with no markdown or prose. Set status to "done", missingContext to [], and artifacts to an array containing exactly one JSON-stringified execution task packet with every required taskPacket field. The packet readFiles and writeFiles must be non-empty and scoped to this repository.',
+    'Return exactly one worker-result JSON object, with no markdown or prose. Set status to "done", missingContext to [], and artifacts to an array containing exactly one JSON-stringified execution task packet with every required taskPacket field. The packet readFiles and writeFiles must be non-empty and scoped to this repository. Put structured findings meant to inform that packet in `output` (a JSON object, at most 65536 bytes); prose belongs in `summary`.',
     "",
     "## Verification",
     VERIFICATION_PARAGRAPH,

package/src/contract/worker-result.mjs

@@ -1,6 +1,6 @@
 import { Buffer } from "node:buffer";
 import { validateTaskPacket } from "./task-packet.mjs";
-import { requireText } from "./assert.mjs";
+import { assertObject, requireText } from "./assert.mjs";
 
 const RESULT_LIMITS = Object.freeze({
   bytes: 32 * 1024,
@@ -9,6 +9,10 @@ const RESULT_LIMITS = Object.freeze({
   itemBytes: 2 * 1024,
   artifactBytes: 16 * 1024,
   missingContextItems: 16,
+  // A discovery worker's structured findings are the deliverable, not
+  // incidental prose, so `output` gets its own ceiling outside the 32 KiB
+  // envelope instead of competing with summary/verification for it.
+  outputBytes: 64 * 1024,
 });
 
 /**
@@ -26,9 +30,11 @@ export const DERIVED_WORKER_RESULT_FIELDS = Object.freeze(["changedFiles"]);
 /**
  * The worker-result protocol: exactly one JSON object returned as the only
  * content of the final worker message. `blocked_context` requires at least one
- * missingContext entry; `done` requires none.
+ * missingContext entry; `done` requires none. `output` is optional and carries
+ * a discovery node's structured findings; an execution result must not
+ * declare it (enforced where the node's mode is known, not here).
  *
- * @typedef {{status: WorkerResultStatus, summary: string, verification: string[], artifacts: string[], missingContext: string[]}} WorkerResult
+ * @typedef {{status: WorkerResultStatus, summary: string, verification: string[], artifacts: string[], missingContext: string[], output?: Record<string, unknown>}} WorkerResult
  */
 
 /**
@@ -37,8 +43,9 @@ export const DERIVED_WORKER_RESULT_FIELDS = Object.freeze(["changedFiles"]);
  */
 export function parseWorkerResult(value) {
   if (typeof value !== "string") throw new TypeError("worker result must be JSON text");
-  if (Buffer.byteLength(value, "utf8") > RESULT_LIMITS.bytes) {
-    throw new TypeError(`worker result exceeds ${RESULT_LIMITS.bytes} bytes`);
+  const maxRawBytes = RESULT_LIMITS.bytes + RESULT_LIMITS.outputBytes;
+  if (Buffer.byteLength(value, "utf8") > maxRawBytes) {
+    throw new TypeError(`worker result exceeds ${maxRawBytes} bytes`);
   }
   let parsed;
   try {
@@ -86,17 +93,42 @@ export function validateWorkerResult(value) {
   if (record.status === "done" && missingContext.length > 0) {
     throw new TypeError("worker result.missingContext must be empty for done");
   }
-  const normalized = {
+  // `output` is accepted on any result here: the schema does not know which
+  // node mode produced it. Refusing it for execution results happens exactly
+  // once, at the ingestion point that does know the mode (resolveWorkerResult).
+  let output;
+  if (Object.hasOwn(record, "output") && record.output !== undefined) {
+    assertObject(record.output, "worker result.output");
+    if (Buffer.byteLength(JSON.stringify(record.output), "utf8") > RESULT_LIMITS.outputBytes) {
+      throw new TypeError(`worker result.output exceeds ${RESULT_LIMITS.outputBytes} bytes`);
+    }
+    output = /** @type {Record<string, unknown>} */ (record.output);
+  }
+  const envelope = {
     status: /** @type {WorkerResultStatus} */ (record.status),
     summary: /** @type {string} */ (record.summary),
     verification: [.../** @type {string[]} */ (record.verification)],
     artifacts: [.../** @type {string[]} */ (record.artifacts)],
     missingContext: [...missingContext],
   };
-  if (Buffer.byteLength(JSON.stringify(normalized), "utf8") > RESULT_LIMITS.bytes) {
+  // `output` is bounded on its own above and kept outside this envelope cap:
+  // it is a discovery node's deliverable, not incidental prose competing with
+  // summary/verification for the same 32 KiB budget.
+  if (Buffer.byteLength(JSON.stringify(envelope), "utf8") > RESULT_LIMITS.bytes) {
     throw new TypeError(`worker result exceeds ${RESULT_LIMITS.bytes} bytes`);
   }
-  return normalized;
+  return output === undefined ? envelope : { ...envelope, output };
+}
+
+/**
+ * A discovery node's structured findings, or null when the result carries
+ * none. The one accessor for `output` so callers never read the raw field.
+ *
+ * @param {WorkerResult} result
+ * @returns {Record<string, unknown>|null}
+ */
+export function discoveryOutput(result) {
+  return result.output ?? null;
 }
 
 /**

package/src/engine/lifecycle.mjs

@@ -54,6 +54,7 @@ import { appendUsageRecord, invocationCost, invocationUsage, recordInvocationUsa
 import { attemptWorkspace } from "../repo/worktree.mjs";
 import { executeControllerVerification } from "./verify.mjs";
 import {
+  ExecutionOutputNotAllowedError,
   materializeAttemptResult,
   readWorkerResultFile,
   resolveWorkerResult,
@@ -528,6 +529,16 @@ export async function finalizeClosedJobs(contract, runDir, states, running, lock
       try {
         workerResult = resolveWorkerResult(runDir, job.node, envelope.result);
       } catch (error) {
+        // A definitive protocol violation, not a malformed result: an
+        // execution node has no legitimate way to earn the repair path here.
+        if (error instanceof ExecutionOutputNotAllowedError) {
+          clearTierExhaustion(state);
+          transition(runDir, state, "failed", {
+            phase: "worker",
+            error: { code: "execution_output_not_allowed", message: errorMessage(error) },
+          }, lock);
+          continue;
+        }
         if (job.resultMaterialization) {
           clearTierExhaustion(state);
           transition(runDir, state, "failed", {

package/src/engine/result-file.mjs

@@ -19,10 +19,18 @@ import { extractJson } from "../harnesses/protocol.mjs";
 import { invocationResult } from "./process.mjs";
 import { join } from "node:path";
 import { parseJudge } from "./prompts.mjs";
-import { parseWorkerResult } from "../contract/worker-result.mjs";
+import { discoveryOutput, parseWorkerResult } from "../contract/worker-result.mjs";
 import { readJson, writeJsonAtomic, writeTextAtomic } from "../run/store.mjs";
 import { routeRuntimeForState, runtimeSnapshot } from "./failover.mjs";
 
+/**
+ * Thrown by `resolveWorkerResult` when an execution result declares `output`.
+ * A distinct type, not a plain TypeError: this is a definitive protocol
+ * violation the engine can fail terminally on sight, not a malformed result
+ * worth the generic invalid-result repair attempt.
+ */
+export class ExecutionOutputNotAllowedError extends TypeError {}
+
 /** @typedef {import("./lifecycle.mjs").Invocation} Invocation */
 /** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
 /** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
@@ -115,6 +123,10 @@ export function isResultMaterializationInvocation(invocation) {
   }
 }
 /**
+ * The single ingestion point that knows the node's mode, so it is the one
+ * place `output` is refused for an execution result: the schema itself
+ * accepts the field on any result, mode-blind.
+ *
  * @param {string} runDir
  * @param {ValidatedNode} node
  * @param {unknown} providerResult
@@ -122,9 +134,11 @@ export function isResultMaterializationInvocation(invocation) {
  */
 export function resolveWorkerResult(runDir, node, providerResult) {
   const fromFile = readWorkerResultFile(runDir, node.id);
-  if (fromFile) return fromFile;
-  const result = parseWorkerResult(String(extractJson(providerResult) ?? providerResult ?? ""));
-  persistWorkerResultFile(runDir, node.id, result);
+  const result = fromFile ?? parseWorkerResult(String(extractJson(providerResult) ?? providerResult ?? ""));
+  if (node.taskPacket.mode === "execution" && discoveryOutput(result) !== null) {
+    throw new ExecutionOutputNotAllowedError("worker result.output is only allowed for a discovery node, not execution");
+  }
+  if (!fromFile) persistWorkerResultFile(runDir, node.id, result);
   return result;
 }
 /**

package/src/engine/scheduler.mjs

@@ -31,15 +31,15 @@ import {
   terminalErrorCode,
 } from "./lifecycle.mjs";
 import { delay, errorCode } from "../util.mjs";
-import { alreadyNotified, notifyQueueFor, notifyQueuesByRun, renderCampaignHandoffSafely } from "./notify-queue.mjs";
-import { detectStalls, terminateProcess } from "./process.mjs";
+import { alreadyNotified, emitNodeAdvisories, notifyQueueFor, notifyQueuesByRun, renderCampaignHandoffSafely } from "./notify-queue.mjs";
+import { detectStalls, invocationAlive, terminateProcess } from "./process.mjs";
 import { transition, writeNode } from "./state.mjs";
 import { listNodeSnapshots, readNodeSnapshot } from "../run/node-store.mjs";
 import { render, renderFinalReport, writeFindingsArtifact } from "../report/final.mjs";
 import { operationNextState, providerReceipts, settleInvocation } from "../run/operations.mjs";
 import { appendUsageRecord, invocationCost, invocationUsage, recordInvocationUsage } from "../run/usage.mjs";
 import { captureNodeScopeBoundaries, checkWorkerScope, emptyScope } from "./scope.mjs";
-import { validateContract } from "../contract/index.mjs";
+import { validateContractForLaunch } from "../campaign/chain.mjs";
 import { validateNodeSnapshot } from "../contract/snapshot.mjs";
 import { finalVerificationCommands, sharedVerificationCommands } from "../contract/final-verification.mjs";
 import { startJudge, startWorker } from "./dispatch.mjs";
@@ -131,16 +131,22 @@ export function nodeBudgetBasisMs(contract, node) {
  * never touches phase 2's parked `blocked`/`failed`/`exhausted`/`stalled`
  * states, and never overwrites the `runtime_tier_exhausted` waiting shape.
  *
+ * A node whose closed job is being settled in the background (its id is a key
+ * of `pendingSettlements`) is not this dead end either: its invocation has
+ * already exited and left `running`, but the settlement promise still owns
+ * deciding what happens to it, so it is left alone until that promise resolves.
+ *
  * @param {string} runDir
  * @param {Map<string, NodeSnapshot>} states
  * @param {Map<string, Job>} running
  * @param {LockHandle|null} lock
+ * @param {Map<string, Promise<void>>} [pendingSettlements]
  * @returns {string[]} the node ids this pass parked
  */
-export function enforceRunningInvariant(runDir, states, running, lock) {
+export function enforceRunningInvariant(runDir, states, running, lock, pendingSettlements = new Map()) {
   const parked = [];
   for (const [nodeId, state] of states) {
-    if (state.status !== "running" || running.has(nodeId)) continue;
+    if (state.status !== "running" || running.has(nodeId) || pendingSettlements.has(nodeId)) continue;
     transition(runDir, state, "blocked", {
       phase: state.phase,
       error: {
@@ -175,14 +181,17 @@ function renderFingerprint(states) {
 
 /**
  * @param {string} contractPath
- * @param {{detachedBootstrap?: boolean}} [options] `detachedBootstrap` is set
- *   only by the CLI entry when this process is its own detached child, and
- *   makes the controller wait for the launcher's acknowledgement
+ * @param {{detachedBootstrap?: boolean, baseRef?: string}} [options]
+ *   `detachedBootstrap` is set only by the CLI entry when this process is its
+ *   own detached child, and makes the controller wait for the launcher's
+ *   acknowledgement; `baseRef` is the CLI's own `--base-ref`, re-validated
+ *   here so a launch and the run it starts agree about what the contract was
+ *   checked against
  * @returns {Promise<RunOutcome>}
  */
 export async function runContract(contractPath, options = {}) {
   const absoluteContractPath = resolve(contractPath);
-  const contract = validateContract(JSON.parse(readFileSync(absoluteContractPath, "utf8")), absoluteContractPath);
+  const contract = validateContractForLaunch(JSON.parse(readFileSync(absoluteContractPath, "utf8")), absoluteContractPath, { baseRef: options.baseRef });
   const runDir = join(contract.cwd, ".runs", contract.id);
   if (existsSync(runDir)) throw new Error(`run already exists: ${runDir}`);
   mkdirSync(join(contract.cwd, ".runs"), { recursive: true });
@@ -319,12 +328,13 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
     statusFingerprint = fingerprint;
     render(runDir, runsDir, contract, states, renderLock);
   };
-  // The tick that owns a running node's verification can be minutes long
-  // (`executeControllerVerification` is awaited on the critical path below),
-  // and that whole time status.json would otherwise report whatever the last
-  // tick left it at. A timer renders between ticks too; it is cheap even when
-  // idle because `renderFingerprint` still change-detects, so a quiet run
-  // writes nothing extra.
+  // A node's own settlement (its controller verification, its judge round) can
+  // be minutes long, and it now runs off the tick's critical path in
+  // `pendingSettlements` so dispatch never waits behind it -- but that whole
+  // time status.json would otherwise report whatever the last tick left it at.
+  // A timer renders between ticks too; it is cheap even when idle because
+  // `renderFingerprint` still change-detects, so a quiet run writes nothing
+  // extra.
   const statusTimer = setInterval(() => renderStatusIfChanged(), contract.pollIntervalMs);
   statusTimer.unref();
   let handoffFingerprint = statesFingerprint(states);
@@ -364,14 +374,31 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
 
   /** @type {Map<string, Job>} */
   const running = new Map();
+  // One promise per node currently settling a closed job -- its controller
+  // verification, its candidate verification, its judge round -- kept off the
+  // tick's critical path so an eligible sibling still dispatches into a free
+  // slot while this node's own invocation has already exited. A node's own
+  // steps stay ordered because only one settlement per node is ever in flight
+  // (see the dispatch loop below); a *different* node's settlement is queued
+  // behind whichever one is already running (`settlementQueue`), not run
+  // alongside it -- `finalizeClosedJobs` shares state a concurrent second call
+  // would corrupt: the phase-continuation selection that picks at most one
+  // live node to carry a session forward, and `repo/integrate.mjs`'s one
+  // candidate ref and worktree per run. Only *dispatching a sibling* skips
+  // ahead of a node's settlement; two nodes' settlements never interleave.
+  /** @type {Map<string, Promise<void>>} */
+  const pendingSettlements = new Map();
+  /** @type {Promise<void>} */
+  let settlementQueue = Promise.resolve();
   let canceled = false;
   const cancel = () => { canceled = true; };
   process.once("SIGINT", cancel);
   process.once("SIGTERM", cancel);
   process.once("SIGHUP", cancel);
   // The heartbeat's `at` is owned by an unref'd timer inside this writer, never
-  // by the loop body below: the loop awaits controller verification on its own
-  // critical path and must keep answering "the process is alive" while it does.
+  // by the loop body below: a node's settlement can still run for minutes off
+  // the tick's critical path (`pendingSettlements`), and the heartbeat must
+  // keep answering "the process is alive" while it does.
   const heartbeat = createHeartbeat({ runDir, intervalMs: HEARTBEAT_INTERVAL_MS });
   const heartbeatNodes = new Map(contract.nodes.map((node) => [node.id, node]));
   let heartbeatFingerprint = statesFingerprint(states);
@@ -380,11 +407,89 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
   const activeHeartbeatNodes = () => [...states.values()]
     .filter((state) => state.status === "running" && heartbeatNodes.has(state.id))
     .map((state) => ({ nodeId: state.id, budgetBasis: nodeBudgetBasisMs(contract, /** @type {ValidatedNode} */ (heartbeatNodes.get(state.id))) }));
+  // A programmer error surfacing inside a background settlement must still
+  // crash the whole run, exactly as an unguarded `await finalizeClosedJobs`
+  // used to: it is recorded here and thrown from the top of the loop on the
+  // very next tick, rather than immediately, so it cannot itself become the
+  // block a sibling's dispatch is waiting behind. A lost lock is not this --
+  // the loop's own `lock.assert()` calls surface that same condition on their
+  // own schedule, so it is left for them.
+  /** @type {unknown} */
+  let backgroundSettlementFailure = null;
+  // Mark every job that closed this tick as settling, and free its slot,
+  // without waiting for any of them: that alone is what lets an eligible
+  // sibling dispatch into the freed slot while this node's minutes-long
+  // controller verification or judge round is still running. The actual
+  // settlement work is chained onto `settlementQueue`, one node at a time in
+  // the order its job closed, so it still runs exactly as serialized against
+  // every *other* node's settlement as it did when this loop awaited
+  // `finalizeClosedJobs` directly -- `finalizeClosedJobs` runs against a
+  // one-entry map per node, so this is one call per node rather than the one
+  // batched call it used to be, but the chain still runs them one at a time.
+  // A settlement may itself dispatch the node's next phase (a judge, a
+  // revision) through the same `startJudge`/`startWorker` calls dispatch below
+  // uses; those land in `slot`, not the real `running`, so they are copied
+  // back into `running` in a `finally` -- unconditionally, win or lose, so a
+  // job a settlement started before failing (a lost lock, a programmer error)
+  // is still visible to the cleanup sweeps below rather than leaked. The
+  // node's own steps stay ordered by never starting a second settlement for a
+  // node whose first has not yet cleared `pendingSettlements`.
+  const settleClosedJobsInBackground = () => {
+    for (const [nodeId, job] of [...running]) {
+      if (pendingSettlements.has(nodeId) || !job.closed || invocationAlive(job.invocation)) continue;
+      running.delete(nodeId);
+      const slot = new Map([[nodeId, job]]);
+      const settlement = settlementQueue
+        .then(() => finalizeClosedJobs(contract, runDir, states, slot, lock, campaign.path))
+        .finally(() => {
+          for (const [settledId, settledJob] of slot) running.set(settledId, settledJob);
+          pendingSettlements.delete(nodeId);
+        });
+      // The queue itself must never reject -- a rejected settlement (a lost
+      // lock, a programmer error) would otherwise wedge every node queued
+      // behind it. The rejection still reaches whoever awaits the real
+      // `settlement` promise (`pendingSettlements`, below).
+      settlementQueue = settlement.catch(() => {});
+      // Handled here so an in-flight settlement never becomes an unhandled
+      // rejection when nobody happens to await `pendingSettlements` before the
+      // process exits; the original promise, still held below, carries the
+      // rejection to whichever checkpoint (cancel, shutdown) awaits it.
+      settlement.catch((error) => {
+        if (!(error instanceof LockLostError) && backgroundSettlementFailure === null) backgroundSettlementFailure = error;
+      });
+      pendingSettlements.set(nodeId, settlement);
+    }
+  };
+  // Captured once, before the loop, rather than re-derived every tick: it is
+  // "parked when this controller invocation started" (autoRetryParkedNodes's
+  // own contract), and a node a background settlement parks between two ticks
+  // -- rather than synchronously within one, as it always did before
+  // settlement moved off the tick's critical path -- must still read as newly
+  // parked whichever later tick first observes it, not as already-parked
+  // because a per-tick snapshot happened to be taken after it landed.
+  const parkedBefore = new Set([...states.values()].filter((state) => PARKED.has(state.status)).map((state) => state.id));
+  const anyUnsettled = () => [...states.values()].some((state) => !SETTLED.has(state.status));
   try {
-    while ([...states.values()].some((state) => !SETTLED.has(state.status))) {
+    // A `while` that re-checked this at the very top of every tick would exit
+    // the instant a background settlement flips the run's last unsettled node
+    // straight to a terminal status between two ticks -- before the tick body
+    // that would have run `autoRetryParkedNodes` against that new status ever
+    // gets to. The entry guard skips the loop entirely when there is nothing
+    // to do at all (a resume of an already-settled run still does zero
+    // iterations); once inside, the exit check moves to the bottom, after the
+    // body, so that body always sees a freshly-parked node at least once
+    // before the loop is allowed to end.
+    if (anyUnsettled()) for (;;) {
       lock.assert();
+      if (backgroundSettlementFailure !== null) throw backgroundSettlementFailure;
       if (existsSync(join(runDir, "cancel.request.json"))) canceled = true;
       if (canceled) {
+        // A settlement already in flight owns the one decision a cancellation
+        // must not race: what this node's own invocation resolved to. Let it
+        // land on its real terminal status (and, when it re-dispatched a judge
+        // or a revision, on the new job that landed in `running`) before this
+        // branch decides which nodes are merely canceled.
+        await Promise.all([...pendingSettlements.values()]);
         const jobs = [...running.values()];
         await Promise.all(jobs.map((job) => terminateProcess(job)));
         const envelopes = new Map();
@@ -408,8 +513,14 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
         break;
       }
 
-      const parkedBefore = new Set([...states.values()].filter((state) => PARKED.has(state.status)).map((state) => state.id));
-      await finalizeClosedJobs(contract, runDir, states, running, lock, campaign.path);
+      // Advisory spend lines are checked every tick, closed job or not: a
+      // crossing must be visible while the spend is happening on a node still
+      // running, not only once it closes. `finalizeClosedJobs` used to open
+      // with this same check; calling it once here, rather than once per
+      // node settled this tick, is what keeps it at one pass per tick now
+      // that settlement runs per node in `settleClosedJobsInBackground`.
+      await emitNodeAdvisories(contract, runDir, states);
+      settleClosedJobsInBackground();
       await detectStalls(contract, running, async (job, status, error) => {
         const envelope = recordInvocationUsage(job);
         job.state.usage = invocationUsage(job.state);
@@ -450,8 +561,11 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
         writeNode(runDir, job.state, lock);
       });
       // A node left `running` with no job is a dead end; park it before the
-      // dispatch pass so it cannot hide behind a healthy sibling.
-      enforceRunningInvariant(runDir, states, running, lock);
+      // dispatch pass so it cannot hide behind a healthy sibling. A node
+      // whose closed job is settling in the background is not that dead end
+      // -- `pendingSettlements` is what tells this apart from one truly
+      // abandoned.
+      enforceRunningInvariant(runDir, states, running, lock, pendingSettlements);
       // Before dependants are blocked, a node that parked on this tick gets
       // its one automatic retry: it becomes pending, so `blockDependents` sees
       // nothing to block and the dependants stay `pending`/`phase: "waiting"`
@@ -463,14 +577,37 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
       if (slots > 0) {
         const ready = contract.nodes.filter((node) => {
           const state = states.get(node.id);
-          return state?.status === "pending" && node.dependsOn.every((id) => states.get(id)?.status === "done");
+          return state?.status === "pending" && !pendingSettlements.has(node.id)
+            && node.dependsOn.every((id) => states.get(id)?.status === "done");
         });
         for (const node of ready.slice(0, slots)) {
           const state = states.get(node.id);
           if (!state || routingBackoffActive(state, state.phase)) continue;
+          // A node recovered pending a re-ask judge (its own worker attempt
+          // already accepted, `state.result` durable) reaches `settleDone` /
+          // `integrateAttempt` exactly like a closed job's own settlement
+          // does, on the same per-run candidate ref and worktree
+          // `settlementQueue` exists to serialize -- so it is dispatched the
+          // same way: chained onto the queue rather than awaited here, using
+          // its own one-entry `slot` merged back into `running` in a
+          // `finally`. The node's own order is untouched (still one entry at
+          // a time, gated by `pendingSettlements`); only the tick stops
+          // waiting behind it.
           if (state.phase === "judge" && state.result) {
-            await applyJudgeRound(await startJudge(contract, node, state, runDir, running, state.result, lock, states, campaign.path),
-              contract, node, state, runDir, running, lock, states, campaign.path, state.result);
+            const workerResult = state.result;
+            const slot = new Map();
+            const settlement = settlementQueue
+              .then(() => startJudge(contract, node, state, runDir, slot, workerResult, lock, states, campaign.path))
+              .then((round) => applyJudgeRound(round, contract, node, state, runDir, slot, lock, states, campaign.path, workerResult))
+              .finally(() => {
+                for (const [settledId, settledJob] of slot) running.set(settledId, settledJob);
+                pendingSettlements.delete(node.id);
+              });
+            settlementQueue = settlement.catch(() => {});
+            settlement.catch((error) => {
+              if (!(error instanceof LockLostError) && backgroundSettlementFailure === null) backgroundSettlementFailure = error;
+            });
+            pendingSettlements.set(node.id, settlement);
             continue;
           }
           state.attempt += 1;
@@ -499,10 +636,46 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
         }
       }
       heartbeat.setActive(activeHeartbeatNodes());
-      if ([...states.values()].some((state) => !SETTLED.has(state.status))) await delay(contract.pollIntervalMs);
+      // A background settlement can flip a node straight to a parked
+      // terminal status during this same tick's own later awaits
+      // (`detectStalls`, `notifyStateChanges`), after the auto-retry pass
+      // above already ran and saw it as not-yet-parked. Running it once more
+      // here, immediately before the exit check, is what stops the loop from
+      // mistaking that freshly-parked node for settled and exiting before it
+      // ever got its automatic retry; a node it reopens dispatches on the
+      // next tick rather than this one, which `anyUnsettled()` below still
+      // correctly keeps the loop alive for.
+      autoRetryParkedNodes(contract, runDir, states, lock, parkedBefore);
+      if (!anyUnsettled()) break;
+      await delay(contract.pollIntervalMs);
     }
+    // The loop above exits (by the bottom check above or the cancel branch's
+    // `break`) the instant every node's state looks settled, but a
+    // settlement's own trailing work -- sealing a candidate's acceptance,
+    // removing its worktree, enqueueing its terminal notification -- can
+    // still be running after the transition that made the state look
+    // terminal. Nothing past this point (the final render, the findings
+    // artifact, the run-terminal notification, releasing the lock) may run
+    // ahead of that trailing work.
+    await Promise.all([...pendingSettlements.values()]);
+    // A settlement's own `finally` deletes it from `pendingSettlements` the
+    // instant it settles, win or lose -- so a rejection recorded here can
+    // already be gone from the map above by the time this line runs, with
+    // nothing left to await it. The loop's own top-of-tick check
+    // (`if (backgroundSettlementFailure !== null) throw ...`) cannot save
+    // that case either: the loop has already exited. Checked again here,
+    // once, so a background settlement failure can never read as a clean
+    // finish just because it settled on the same tick the run's last node
+    // did.
+    if (backgroundSettlementFailure !== null) throw backgroundSettlementFailure;
   } catch (error) {
     if (!(error instanceof LockLostError)) throw error;
+    // A settlement still merges whatever it started into `running` in its own
+    // `finally`, win or lose, regardless of whether anything awaits it; wait
+    // for that to land (never rejecting itself, so a second lock-loss here
+    // cannot mask the one already being handled) before the termination sweep
+    // reads `running`.
+    await Promise.allSettled([...pendingSettlements.values()]);
     await Promise.all([...running.values()].map((job) => terminateProcess(job)));
     notifyQueuesByRun.delete(runDir);
     return { runDir, states, ok: false, error };

package/src/engine/settle.mjs

@@ -165,7 +165,7 @@ export async function settleDone(contract, node, state, runDir, lock, states, ca
     attemptSha: sealed.sha,
     branch: state.worktree.branch,
     verificationEvidence: state.verification,
-    verifyCandidate: (candidateWorkspace) => verifyCandidateWorkspace(contract, node, state, runDir, candidateWorkspace),
+    verifyCandidate: (candidateWorkspace) => verifyCandidateWorkspace(contract, node, state, runDir, candidateWorkspace, lock),
     onAccepted: async (transaction) => {
       const acceptedPath = state.worktree?.path ?? attemptWorktreePath(runDir, contract.id, node.id, transaction.attempt);
       if (state.attempt === transaction.attempt && state.status !== "done") {