faberun 0.12.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,217 @@
1
+ /**
2
+ * Which provider session a node's next invocation runs in: fresh, a
3
+ * continuation of a compatible earlier session in the same plan phase, or a
4
+ * rotation -- a fresh session that carries the prior nodes' structured
5
+ * summaries. Separate from dispatch.mjs, which spawns whatever this decides,
6
+ * because the decision reads only persisted node snapshots and the routing
7
+ * table, and because dispatch.mjs crossed the 800-line ceiling carrying it.
8
+ */
9
+ import { basename, join } from "node:path";
10
+ import { createHash } from "node:crypto";
11
+ import { boundedUtf8, stableJson } from "../util.mjs";
12
+ import { providerCommand } from "../harnesses/index.mjs";
13
+ import { readJson } from "../run/store.mjs";
14
+ import { routeRuntimeForState } from "./failover.mjs";
15
+ import { validateNodeSnapshot } from "../contract/snapshot.mjs";
16
+
17
+ /** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
18
+ /** @typedef {import("../contract/index.mjs").ValidatedNode} ValidatedNode */
19
+ /** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
20
+ /** @typedef {import("../contract/index.mjs").RuntimeSnapshot} RuntimeSnapshot */
21
+ /** @typedef {import("./process.mjs").Invocation} Invocation */
22
+
23
+ /** @typedef {{forceFresh?: boolean}} SessionPolicy */
24
+
25
+ /**
26
+ * Resolve the session policy one dispatch runs under, then consume the copy the
27
+ * node persisted. The explicit argument comes from a caller that is dispatching
28
+ * on the spot; `state.sessionPolicy` is the copy a rejection decision left when
29
+ * it handed the node back to the scheduler, whose own `startWorker` call passes
30
+ * nothing at all.
31
+ *
32
+ * Persisting is the whole point: `phaseInvocationPlan` rediscovers a compatible
33
+ * continuation from the persisted ledger, so nulling a local continuation id at
34
+ * the call site would let the scheduler's later dispatch find it again. Clearing
35
+ * the stored policy here makes it one-shot — it governs exactly the dispatch it
36
+ * was recorded for, and the next unrelated attempt reuses normally.
37
+ *
38
+ * @param {{sessionPolicy?: SessionPolicy|null}} state
39
+ * @param {SessionPolicy} [explicit]
40
+ * @returns {SessionPolicy}
41
+ */
42
+ export function forceFreshSession(state, explicit = {}) {
43
+ const persisted = /** @type {SessionPolicy|undefined} */ (state?.sessionPolicy ?? undefined);
44
+ const policy = { ...(persisted ?? {}), ...explicit };
45
+ if (state && state.sessionPolicy !== undefined && state.sessionPolicy !== null) state.sessionPolicy = null;
46
+ return policy;
47
+ }
48
+
49
+ /**
50
+ * Select the only continuation that is allowed for this plan phase and role.
51
+ * The search is intentionally limited to persisted node snapshots in this run.
52
+ *
53
+ * `policy.forceFresh` is the explicit session policy a rejection decision
54
+ * carries: it short-circuits the search before it can rediscover a compatible
55
+ * continuation, so a retry after a gate rejection starts a fresh provider
56
+ * session instead of re-reading the failed transcript. Nulling a local id at
57
+ * the call site is not enough, because this function rediscovers the prior
58
+ * continuation from the persisted ledger.
59
+ *
60
+ * @param {ValidatedContract} contract
61
+ * @param {ValidatedNode} node
62
+ * @param {NodeSnapshot} state
63
+ * @param {string} runDir
64
+ * @param {"worker"|"judge"} role
65
+ * @param {string} prompt
66
+ * @param {SessionPolicy} [policy]
67
+ * @returns {{prompt: string, continuationId: string|null, mode: "fresh"|"reuse"|"rotate"}}
68
+ */
69
+ export function phaseInvocationPlan(contract, node, state, runDir, role, prompt, policy = {}) {
70
+ if (policy.forceFresh === true) {
71
+ return { prompt, continuationId: null, mode: "fresh" };
72
+ }
73
+ const runId = basename(runDir);
74
+ const session = phaseSessionCandidates(contract, node, state, runDir, role).at(-1);
75
+ const runtime = routeRuntimeForState(contract, node, state, role);
76
+ const identityMatches = session && session.invocation.runId === runId
77
+ && session.invocation.campaignId === contract.campaignId
78
+ && session.invocation.planPhase === node.phase
79
+ && session.invocation.role === role
80
+ && session.invocation.harness === runtime.harness
81
+ && session.invocation.runtimeId === runtime.id
82
+ && session.invocation.runtimeFingerprint === fingerprintRuntime(runtime)
83
+ && session.invocation.model === runtime.model
84
+ && session.invocation.reasoning === (runtime.reasoning ?? null)
85
+ && session.invocation.sandbox === (runtime.sandbox ?? null);
86
+ const canContinue = runtime.capabilities.continuation === true;
87
+ // A node continuing its own earlier session keeps it. A phase sibling's
88
+ // session is reused only when the contract opts in (`phaseSessionReuse`):
89
+ // measured 2026-09-20 over 21 runs with both kinds of turn, a turn opened
90
+ // on a sibling's session cost 1.87x the fresh one at the same request
91
+ // count -- it began with 200k tokens of context instead of 45k and re-read
92
+ // them on every request -- while the rotation below hands the sibling's
93
+ // structured summary to a fresh session whose first request is already 90%
94
+ // served from the shared prefix cache.
95
+ const ownSession = session !== undefined && session.nodeId === node.id;
96
+ if (identityMatches && canContinue && (ownSession || contract.phaseSessionReuse === true)) {
97
+ return { prompt, continuationId: session.invocation.continuationId ?? null, mode: "reuse" };
98
+ }
99
+ // A harness that cannot continue at all, or a session picked up from a
100
+ // different phase-sibling node whose identity does not match this one, has
101
+ // no native continuity: the fresh attempt carries the prior nodes'
102
+ // structured summaries forward instead of starting blind.
103
+ if (session && (!canContinue || session.nodeId !== node.id)) {
104
+ return {
105
+ prompt: phaseHandoffPrompt(contract, node, state, runDir, role),
106
+ continuationId: null,
107
+ mode: "rotate",
108
+ };
109
+ }
110
+ // A capable harness continuing its own node whose identity merely drifted
111
+ // (the run directory moved, or a runtime edge) still gets the caller's own
112
+ // prompt — already carrying the node's bounded "Previous attempt" section —
113
+ // in a fresh session, never a synthesized handoff.
114
+ return { prompt, continuationId: null, mode: session ? "rotate" : "fresh" };
115
+ }
116
+
117
+ /**
118
+ * Continuation ids a live invocation is already driving, anywhere in the run.
119
+ *
120
+ * This is what makes concurrent nodes of one phase safe, and it is read from
121
+ * the persisted ledger rather than from an in-memory registry so a controller
122
+ * that took over a run inherits the claims instead of racing them.
123
+ *
124
+ * @param {ValidatedContract} contract
125
+ * @param {NodeSnapshot} currentState
126
+ * @param {string} runDir
127
+ * @returns {Set<string>}
128
+ */
129
+ function claimedContinuations(contract, currentState, runDir) {
130
+ /** @type {Set<string>} */
131
+ const claimed = new Set();
132
+ for (const candidate of contract.nodes) {
133
+ let state = candidate.id === currentState.id ? currentState : null;
134
+ if (!state) {
135
+ try { state = validateNodeSnapshot(readJson(join(runDir, "nodes", `${candidate.id}.json`)), candidate); } catch { continue; }
136
+ }
137
+ for (const invocation of state.invocations ?? []) {
138
+ if (invocation.status === "active" && invocation.continuationId) claimed.add(invocation.continuationId);
139
+ }
140
+ }
141
+ return claimed;
142
+ }
143
+
144
+ /**
145
+ * @param {ValidatedContract} contract
146
+ * @param {ValidatedNode} node
147
+ * @param {NodeSnapshot} currentState
148
+ * @param {string} runDir
149
+ * @param {"worker"|"judge"} role
150
+ * @returns {{nodeId: string, invocation: Invocation}[]}
151
+ */
152
+ function phaseSessionCandidates(contract, node, currentState, runDir, role) {
153
+ /** @type {{nodeId: string, invocation: Invocation}[]} */
154
+ const candidates = [];
155
+ const claimed = claimedContinuations(contract, currentState, runDir);
156
+ for (const candidate of contract.nodes) {
157
+ if (candidate.phase !== node.phase) continue;
158
+ let state = candidate.id === currentState.id ? currentState : null;
159
+ if (!state) {
160
+ try { state = validateNodeSnapshot(readJson(join(runDir, "nodes", `${candidate.id}.json`)), candidate); } catch { continue; }
161
+ }
162
+ for (const invocation of state.invocations ?? []) {
163
+ if (invocation.role !== role || invocation.planPhase !== node.phase || !invocation.continuationId) continue;
164
+ if (invocation.nodeId !== candidate.id || invocation.attempt !== state.attempt || invocation.workspace !== state.worktree?.path) continue;
165
+ // One provider session, one live turn. With `maxParallel` above one,
166
+ // two nodes of a phase can be dispatched in the same tick, and without
167
+ // this both would hand the same continuation id to their own provider
168
+ // process. The claim is read from the persisted ledger, which the
169
+ // in-tick dispatch already wrote for the node that went first.
170
+ if (claimed.has(invocation.continuationId)) continue;
171
+ candidates.push({ nodeId: candidate.id, invocation });
172
+ }
173
+ }
174
+ return candidates.sort((left, right) => {
175
+ const leftStarted = Date.parse(left.invocation.startedAt);
176
+ const rightStarted = Date.parse(right.invocation.startedAt);
177
+ if (leftStarted !== rightStarted) return leftStarted - rightStarted;
178
+ const leftUpdated = Date.parse(left.invocation.updatedAt);
179
+ const rightUpdated = Date.parse(right.invocation.updatedAt);
180
+ if (leftUpdated !== rightUpdated) return leftUpdated - rightUpdated;
181
+ return left.invocation.id.localeCompare(right.invocation.id);
182
+ });
183
+ }
184
+
185
+ /** @param {RuntimeSnapshot} runtime @returns {string} */
186
+ export function fingerprintRuntime(runtime) {
187
+ const executable = providerCommand(runtime, "").executable;
188
+ return createHash("sha256").update(stableJson({ runtime, executable })).digest("hex");
189
+ }
190
+
191
+ /** @param {ValidatedContract} contract @param {ValidatedNode} node @param {NodeSnapshot} state @param {string} runDir @param {"worker"|"judge"} role @returns {string} */
192
+ function phaseHandoffPrompt(contract, node, state, runDir, role) {
193
+ const summaries = phaseSessionCandidates(contract, node, state, runDir, role)
194
+ .map(({ nodeId }) => {
195
+ const candidate = contract.nodes.find((item) => item.id === nodeId);
196
+ let snapshot = null;
197
+ try { snapshot = readJson(join(runDir, "nodes", `${nodeId}.json`)); } catch {
198
+ // ENOENT or unreadable snapshot: this prior node contributes no summary.
199
+ }
200
+ const result = snapshot?.result;
201
+ const record = result && typeof result === "object" && !Array.isArray(result)
202
+ ? /** @type {Record<string, unknown>} */ (result)
203
+ : null;
204
+ const summary = typeof record?.summary === "string" ? record.summary : null;
205
+ return summary && candidate ? `${candidate.id}: ${boundedUtf8(summary, 1024)}` : null;
206
+ })
207
+ .filter(Boolean)
208
+ .slice(-8);
209
+ const handoff = [
210
+ `Continue phase ${node.phase} as the ${role} agent in a fresh provider session.`,
211
+ "Prior structured node summaries:",
212
+ summaries.length ? summaries.map((summary) => `- ${summary}`).join("\n") : "- (none)",
213
+ "Current closed task packet:",
214
+ boundedUtf8(node.prompt, 48 * 1024),
215
+ ].join("\n\n");
216
+ return boundedUtf8(handoff, 60 * 1024);
217
+ }
@@ -7,8 +7,8 @@
7
7
  * a judge decides, or when a run is done. That separation is the point: a stuck
8
8
  * provider is killed by the same code whatever it was asked to do.
9
9
  */
10
- import { SessionMetricsParser } from "../harnesses/exec-jsonl/index.mjs";
11
- import { closeSync, existsSync, fsyncSync, openSync, readFileSync, readSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
10
+ import { boundedRegion, monitorInvocation } from "./transcript.mjs";
11
+ import { closeSync, existsSync, fsyncSync, openSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
12
12
  import { dirname, join } from "node:path";
13
13
  import { errorCode, errorMessage } from "../util.mjs";
14
14
  import { fileURLToPath } from "node:url";
@@ -33,19 +33,14 @@ import { writeNodeSnapshot } from "../run/node-store.mjs";
33
33
  /** @typedef {ProviderEnvelope & {costProvenance?: "priced"}} PricedEnvelope */
34
34
  /** @typedef {import("node:child_process").ChildProcess} ChildProcess */
35
35
  /** @typedef {{prompt: string|null, stdout: string, stderr: string}} PathSet */
36
- /** @typedef {{id: string, pid: number, processGroupId: number|null, processStartToken: string|null, harness: string, runtimeId: string|null, runtimeFingerprint?: string, revision?: number, phase: string, promptPath: string|null, stdoutPath: string, stderrPath: string, startedAt: string, deadlineAt: string|null, updatedAt: string, closedAt: string|null, exitCode: number|null, signal: string|null, status: "active"|"closed"|"terminated", executable: string, snapshotPath?: string, usage?: Usage, usageEstimated?: boolean, costUsd?: number|null, costProvenance?: "priced", runId?: string, campaignId?: string, nodeId?: string, attempt?: number, workspace?: string, worktreeBranch?: string|null, worktreeBaseSha?: string|null, planPhase?: string, role?: "worker"|"judge", model?: string, reasoning?: string|null, sandbox?: string|null, continuationId?: string|null, continuationMode?: "fresh"|"reuse"|"rotate"}} Invocation */
36
+ /** @typedef {{id: string, pid: number, processGroupId: number|null, processStartToken: string|null, harness: string, runtimeId: string|null, runtimeFingerprint?: string, revision?: number, phase: string, promptPath: string|null, stdoutPath: string, stderrPath: string, startedAt: string, deadlineAt: string|null, updatedAt: string, closedAt: string|null, exitCode: number|null, signal: string|null, status: "active"|"closed"|"terminated", executable: string, snapshotPath?: string, usage?: Usage, usageEstimated?: boolean, costUsd?: number|null, costProvenance?: "priced", runId?: string, campaignId?: string, nodeId?: string, attempt?: number, workspace?: string, worktreeBranch?: string|null, worktreeBaseSha?: string|null, planPhase?: string, role?: "worker"|"judge", model?: string, reasoning?: string|null, sandbox?: string|null, continuationId?: string|null, continuationMode?: "fresh"|"reuse"|"rotate", session?: import("../harnesses/session-metrics.mjs").SessionLedger|null}} Invocation */
37
37
  /** @typedef {{pid: number|null, processGroupId?: number|null, processStartToken?: string|null}} InvocationProbe */
38
- /** @typedef {{child: ChildProcess, contract: ValidatedContract, node: ValidatedNode, state: NodeSnapshot, runtime: HarnessRuntime & {id: string|null}, cwd: string, paths: PathSet, phase: string, invocation: Invocation, startedAt: string, startedTicks: bigint, progressTicks: bigint, lastOutputAt: number, closed: boolean, exitCode: number|null, signal: string|null, spawnError: Error|null, terminating: Promise<void>|null, gateConfigPath: string, gateReleasePath: string, scopeBaseline?: unknown, scopeChecked?: boolean, scopeViolation?: boolean, resultMaterialization?: boolean, recoveryBaseline?: unknown, observeTimer?: ReturnType<typeof setInterval>, monitorOffset?: number, monitorParser?: import("../harnesses/exec-jsonl/index.mjs").SessionMetricsParser, lastEventCount?: number, observedOnce?: boolean, onClose?: (invocation: Invocation) => void, onInvocationUpdate?: (invocation: Invocation) => void, onProgress?: (state: NodeSnapshot) => void}} Job */
38
+ /** @typedef {{child: ChildProcess, contract: ValidatedContract, node: ValidatedNode, state: NodeSnapshot, runtime: HarnessRuntime & {id: string|null}, cwd: string, paths: PathSet, phase: string, invocation: Invocation, startedAt: string, startedTicks: bigint, progressTicks: bigint, lastOutputAt: number, closed: boolean, exitCode: number|null, signal: string|null, spawnError: Error|null, terminating: Promise<void>|null, gateConfigPath: string, gateReleasePath: string, scopeBaseline?: unknown, scopeChecked?: boolean, scopeViolation?: boolean, resultMaterialization?: boolean, recoveryBaseline?: unknown, observeTimer?: ReturnType<typeof setInterval>, monitorOffset?: number, monitorParser?: import("../harnesses/session-metrics.mjs").SessionMetricsParser, lastEventCount?: number, observedOnce?: boolean, onClose?: (invocation: Invocation) => void, onInvocationUpdate?: (invocation: Invocation) => void, onProgress?: (state: NodeSnapshot) => void}} Job */
39
39
  /** @typedef {{graceMs?: number, killGraceMs?: number, escalate?: boolean, runDir?: string, kill?: (pid: number, signal: string|number) => unknown, child?: ChildProcess|null}} TerminateOptions */
40
40
 
41
41
  const HERE = dirname(fileURLToPath(import.meta.url));
42
42
  const DEFAULT_GRACE_MS = 2_000;
43
43
  const GATE_PATH = join(HERE, "gate.mjs");
44
- const MAX_PROVIDER_LOG_BYTES = 512 * 1024;
45
- /** Fixed-size read for incremental transcript observation. */
46
- const MONITOR_CHUNK_BYTES = 64 * 1024;
47
- /** Per-observation read budget: one tick never blocks on a huge backlog. */
48
- const MONITOR_CALL_BUDGET_BYTES = 1024 * 1024;
49
44
  /**
50
45
  * @param {{contract: ValidatedContract, node: ValidatedNode, state: NodeSnapshot, runtime: HarnessRuntime & {id: string|null}, prompt: string, paths: PathSet, phase: string, workspace?: string, commandOptions?: import("../harnesses/index.mjs").CommandOptions, onInvocation: (invocation: Invocation, job: Job) => void, onInvocationUpdate?: (invocation: Invocation) => void, onProgress?: (state: NodeSnapshot) => void}} args
51
46
  * @returns {Job}
@@ -201,47 +196,6 @@ function observeInvocation(job) {
201
196
  // id must not be able to kill the job that is being observed.
202
197
  }
203
198
  }
204
- /**
205
- * Observe the transcript incrementally: read only the bytes appended since
206
- * the last observation, in fixed-size chunks folded into a parser whose
207
- * retained state never scales with the unread length — so the metrics
208
- * survive both a transcript that outgrows any fixed window and an
209
- * already-large transcript on the first call after a controller restart.
210
- * The gate caps the log only at close, so byte offsets stay valid while the
211
- * provider is live. Only newline-terminated records are evidence; a
212
- * trailing partial record stays unconsumed for the next observation. The
213
- * generic metrics are zero for a provider that does not expose them.
214
- *
215
- * @param {Job} job
216
- * @returns {{continuationId: string|null, turns: number, cacheReadInputTokens: number, toolCalls: number, completed: boolean}}
217
- */
218
- export function monitorInvocation(job) {
219
- try {
220
- const parser = job.monitorParser ?? (job.monitorParser = new SessionMetricsParser(job.runtime.harness));
221
- const size = statSync(job.paths.stdout).size;
222
- let offset = job.monitorOffset ?? 0;
223
- let budget = MONITOR_CALL_BUDGET_BYTES;
224
- if (size > offset) {
225
- const fd = openSync(job.paths.stdout, "r");
226
- try {
227
- const chunk = Buffer.alloc(MONITOR_CHUNK_BYTES);
228
- while (offset < size && budget > 0) {
229
- const read = readSync(fd, chunk, 0, Math.min(chunk.length, size - offset, budget), offset);
230
- if (read <= 0) break;
231
- parser.push(chunk.subarray(0, read));
232
- offset += read;
233
- budget -= read;
234
- }
235
- } finally {
236
- closeSync(fd);
237
- }
238
- job.monitorOffset = offset;
239
- }
240
- return { continuationId: parser.continuationId, ...parser.metrics() };
241
- } catch {
242
- return { continuationId: null, turns: 0, cacheReadInputTokens: 0, toolCalls: 0, completed: false };
243
- }
244
- }
245
199
  /**
246
200
  * @param {string} path
247
201
  */
@@ -321,7 +275,7 @@ export async function terminateInvocation(invocation, options = {}) {
321
275
  * sealing here, a timeout parks with an empty seal and the recorded work is
322
276
  * abandoned in a worktree the next attempt never reads.
323
277
  */
324
- const SEAL_BEFORE_KILL_CODES = new Set(["wall_clock_timeout", "stall_timeout"]);
278
+ const SEAL_BEFORE_KILL_CODES = new Set(["wall_clock_timeout", "stall_timeout", "turn_limit"]);
325
279
 
326
280
  /**
327
281
  * How long a `SIGSTOP`ped process group is given to actually stop before the
@@ -393,8 +347,8 @@ function resumeInvocation(invocation) {
393
347
  }
394
348
 
395
349
  /**
396
- * The pre-termination seam, filled: on `wall_clock_timeout` and `stall_timeout`
397
- * quiesce the provider, seal the attempt worktree, and only then let the caller
350
+ * The pre-termination seam, filled: on `wall_clock_timeout`, `stall_timeout`
351
+ * and `turn_limit` quiesce the provider, seal the attempt worktree, and only then let the caller
398
352
  * terminate it, so the next attempt is cut from the seal. The seal is bounded
399
353
  * by the same git timeout every synchronous git call uses
400
354
  * (`GIT_SYNC_TIMEOUT_MS`, overridable with `FABERUN_GIT_TIMEOUT_MS`);
@@ -487,6 +441,25 @@ export async function detectStalls(contract, running, onTimeout, onProgress, onB
487
441
  // as liveness, not only for an mtime that no longer does.
488
442
  job.lastOutputAt = Date.now();
489
443
  }
444
+ // The attempt's request ceiling: a turn still making requests past it
445
+ // is the runaway shape, not progress. measured 2026-09-20: two 600-request
446
+ // turns re-sent 190M and 167M tokens of context and produced no result;
447
+ // the 23 turns with no result held 25% of all context spend. Ends the
448
+ // attempt the way a timeout does: sealed, then one automatic retry.
449
+ // (`turns` counts provider requests for claude, dsh and agy; codex
450
+ // reports whole turns, so its cap is in effect a turn count.)
451
+ const turnCap = job.node?.maxTurns ?? contract.maxTurns;
452
+ if (typeof turnCap === "number" && monitored.turns >= turnCap) {
453
+ const limit = {
454
+ code: "turn_limit",
455
+ message: `${job.phase} made ${monitored.turns} provider requests, the attempt's maxTurns of ${turnCap}`,
456
+ };
457
+ await onBeforeTerminate(job, limit);
458
+ await terminateProcess(job);
459
+ running.delete(nodeId);
460
+ await onTimeout(job, "exhausted", limit);
461
+ continue;
462
+ }
490
463
  } else if (job.observedOnce !== true) {
491
464
  job.progressTicks = now;
492
465
  job.lastOutputAt = Date.now();
@@ -583,28 +556,6 @@ export function priceUsage(runtime, usage, reportedCostUsd) {
583
556
  }
584
557
  return { costUsd: total / 1_000_000, costProvenance: "priced" };
585
558
  }
586
- /**
587
- * @param {string} path
588
- * @param {number} maxBytes
589
- * @returns {string}
590
- */
591
- function boundedRegion(path, maxBytes = MAX_PROVIDER_LOG_BYTES) {
592
- try {
593
- return dropPartialLogLine(readFileSync(`${path}.tail`, "utf8"));
594
- } catch (error) {
595
- if (errorCode(error) !== "ENOENT") throw error;
596
- }
597
- const size = statSync(path).size;
598
- if (size <= maxBytes) return readFileSync(path, "utf8");
599
- const fd = openSync(path, "r");
600
- try {
601
- const bytes = Buffer.alloc(maxBytes);
602
- readSync(fd, bytes, 0, maxBytes, size - maxBytes);
603
- return dropPartialLogLine(bytes.toString("utf8"));
604
- } finally {
605
- closeSync(fd);
606
- }
607
- }
608
559
  /**
609
560
  * Signal the invocation's process group only when ownership is proven. Never
610
561
  * throws: ESRCH is gone, EPERM is a group this user cannot signal and therefore
@@ -746,36 +697,3 @@ export function logPaths(runDir, nodeId, phase, attempt) {
746
697
  stderr: join(runDir, "logs", `${stem}.err`),
747
698
  };
748
699
  }
749
- /**
750
- * @param {string} path
751
- * @param {number} [maxBytes]
752
- * @returns {string}
753
- */
754
- export function readBoundedTail(path, maxBytes = 512 * 1024) {
755
- try {
756
- try { return dropPartialLogLine(readFileSync(`${path}.tail`, "utf8")); } catch (tailError) {
757
- if (errorCode(tailError) !== "ENOENT") throw tailError;
758
- }
759
- const size = statSync(path).size;
760
- if (size <= maxBytes) return readFileSync(path, "utf8");
761
- const fd = openSync(path, "r");
762
- try {
763
- const bytes = Buffer.alloc(maxBytes);
764
- readSync(fd, bytes, 0, maxBytes, size - maxBytes);
765
- return dropPartialLogLine(bytes.toString("utf8"));
766
- } finally {
767
- closeSync(fd);
768
- }
769
- } catch (error) {
770
- if (errorCode(error) === "ENOENT") return "";
771
- throw error;
772
- }
773
- }
774
- /**
775
- * @param {unknown} value
776
- * @returns {string}
777
- */
778
- function dropPartialLogLine(value) {
779
- const newline = String(value).indexOf("\n");
780
- return newline < 0 ? "" : String(value).slice(newline + 1);
781
- }
@@ -7,7 +7,8 @@ import {
7
7
  applyJudgeRound,
8
8
  } from "./review.mjs";
9
9
  import { CONTRACT_VERSION, PROTOCOL_SCHEMA_VERSION } from "../harnesses/index.mjs";
10
- import { routingBackoffActive } from "./failover.mjs";
10
+ import { routeRuntimeForState, routingBackoffActive } from "./failover.mjs";
11
+ import { quotaHeldRuntimes, runningPerRuntime, runtimeHasCapacity } from "./capacity.mjs";
11
12
 
12
13
  import {
13
14
  bootstrapAttemptPath,
@@ -583,7 +584,7 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
583
584
  // A judge killed on its own wall clock produced no verdict. That is a
584
585
  // judge protocol defect, not a node outcome: it earns the one bounded
585
586
  // re-ask, and only then the review mode settles the node.
586
- if (job.phase === "judge" && error.code === "wall_clock_timeout") {
587
+ if (job.phase === "judge" && (error.code === "wall_clock_timeout" || error.code === "turn_limit")) {
587
588
  await applyJudgeProtocolFailure(contract, job.node, job.state, runDir, running, lock, states, campaign.path, error.message);
588
589
  return;
589
590
  }
@@ -614,9 +615,20 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
614
615
  return state?.status === "pending" && !pendingSettlements.has(node.id)
615
616
  && node.dependsOn.every((id) => states.get(id)?.status === "done");
616
617
  });
617
- for (const node of ready.slice(0, slots)) {
618
+ // Per-runtime capacity is judged per dispatch, not per tick: the
619
+ // counts include what this tick has already started, and a runtime a
620
+ // sibling is waiting out a quota reset on accepts nothing new.
621
+ const counts = runningPerRuntime(running.values());
622
+ const held = quotaHeldRuntimes(states.values(), Date.now());
623
+ let dispatched = 0;
624
+ for (const node of ready) {
625
+ if (dispatched >= slots) break;
618
626
  const state = states.get(node.id);
619
627
  if (!state || routingBackoffActive(state, state.phase)) continue;
628
+ const routed = routeRuntimeForState(contract, node, state, state.phase === "judge" ? "judge" : "worker");
629
+ if (!runtimeHasCapacity(routed.id, contract, counts, held)) continue;
630
+ counts.set(routed.id, (counts.get(routed.id) ?? 0) + 1);
631
+ dispatched += 1;
620
632
  // A node recovered pending a re-ask judge (its own worker attempt
621
633
  // already accepted, `state.result` durable) reaches `settleDone` /
622
634
  // `integrateAttempt` exactly like a closed job's own settlement
@@ -40,11 +40,20 @@ import { verifyCandidateWorkspace } from "./verify.mjs";
40
40
  /** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
41
41
  /** @typedef {import("../contract/index.mjs").ValidatedNode} ValidatedNode */
42
42
 
43
- /** Settle one worker-generation rejection: bounded revision when one remains, otherwise terminal exhausted/failed. @param {ValidatedContract} contract @param {ValidatedNode} node @param {NodeSnapshot} state @param {string} runDir @param {Map<string, Job>|null} running @param {LockHandle} lock @param {Map<string, NodeSnapshot>} states @param {string} campaignPath @param {JudgeVerdict} verdict @param {{code: string, label: string, phase?: "worker"|"judge", message?: string, forceFresh?: boolean}} options */
43
+ /**
44
+ * Settle one worker-generation rejection: bounded revision when one remains,
45
+ * otherwise terminal exhausted/failed. The revision budget is the node's
46
+ * (`gate.maxRevisions`, default 1) whether or not the gate reviews: a red
47
+ * deterministic verification earns the same fresh attempt with the failure in
48
+ * front of the worker that a judge rejection does. Measured 2026-09-20 in the
49
+ * orchestration-arms campaign: with the budget behind `gate.enabled`, a node
50
+ * under `gate: false` died on one timing test that flaked under load, and its
51
+ * dependant with it, while the judged twin of the same node got its retry.
52
+ * @param {ValidatedContract} contract @param {ValidatedNode} node @param {NodeSnapshot} state @param {string} runDir @param {Map<string, Job>|null} running @param {LockHandle} lock @param {Map<string, NodeSnapshot>} states @param {string} campaignPath @param {JudgeVerdict} verdict @param {{code: string, label: string, phase?: "worker"|"judge", message?: string, forceFresh?: boolean}} options */
44
53
  export function applyRejection(contract, node, state, runDir, running, lock, states, campaignPath, verdict, options) {
45
54
  const { code, label, phase = "worker", message = verdict.summary, forceFresh = true } = options;
46
55
  state.gate = verdict;
47
- if (node.gate.enabled && state.revisions < (node.gate.maxRevisions ?? 1)) {
56
+ if (state.revisions < (node.gate.maxRevisions ?? 1)) {
48
57
  resetPhaseRouting(state);
49
58
  state.revisions += 1;
50
59
  // The fresh-session decision travels with the node, not just this call:
@@ -0,0 +1,145 @@
1
+ /**
2
+ * The provider transcript as the controller reads it: bounded tail reads at
3
+ * settlement, incremental observation while the provider runs (the liveness
4
+ * signal and the per-request ledger), and the drain at close. Separate from
5
+ * process.mjs, which owns the process itself -- spawn, gate, kill, seal --
6
+ * because reading the log never touches the process, and because process.mjs
7
+ * crossed the 800-line ceiling carrying both jobs.
8
+ */
9
+ import { closeSync, openSync, readFileSync, readSync, statSync } from "node:fs";
10
+ import { Buffer } from "node:buffer";
11
+ import { SessionMetricsParser } from "../harnesses/session-metrics.mjs";
12
+ import { errorCode } from "../util.mjs";
13
+
14
+ /** @typedef {import("./process.mjs").Job} Job */
15
+
16
+ const MAX_PROVIDER_LOG_BYTES = 512 * 1024;
17
+ /** Fixed-size read for incremental transcript observation. */
18
+ const MONITOR_CHUNK_BYTES = 64 * 1024;
19
+ /** Per-observation read budget: one tick never blocks on a huge backlog. */
20
+ const MONITOR_CALL_BUDGET_BYTES = 1024 * 1024;
21
+ /** Settlement drain bound: 64 budgets is 64 MiB, above the largest stored transcript (25.8 MB, measured 2026-09-20). */
22
+ const MONITOR_DRAIN_CALLS = 64;
23
+
24
+ /**
25
+ * Observe the transcript incrementally: read only the bytes appended since
26
+ * the last observation, in fixed-size chunks folded into a parser whose
27
+ * retained state never scales with the unread length — so the metrics
28
+ * survive both a transcript that outgrows any fixed window and an
29
+ * already-large transcript on the first call after a controller restart.
30
+ * The gate caps the log only at close, so byte offsets stay valid while the
31
+ * provider is live. Only newline-terminated records are evidence; a
32
+ * trailing partial record stays unconsumed for the next observation. The
33
+ * generic metrics are zero for a provider that does not expose them.
34
+ *
35
+ * @param {Job} job
36
+ * @returns {{continuationId: string|null, turns: number, cacheReadInputTokens: number, toolCalls: number, completed: boolean}}
37
+ */
38
+ export function monitorInvocation(job) {
39
+ try {
40
+ const parser = job.monitorParser ?? (job.monitorParser = new SessionMetricsParser(job.runtime.harness));
41
+ const size = statSync(job.paths.stdout).size;
42
+ let offset = job.monitorOffset ?? 0;
43
+ let budget = MONITOR_CALL_BUDGET_BYTES;
44
+ if (size > offset) {
45
+ const fd = openSync(job.paths.stdout, "r");
46
+ try {
47
+ const chunk = Buffer.alloc(MONITOR_CHUNK_BYTES);
48
+ while (offset < size && budget > 0) {
49
+ const read = readSync(fd, chunk, 0, Math.min(chunk.length, size - offset, budget), offset);
50
+ if (read <= 0) break;
51
+ parser.push(chunk.subarray(0, read));
52
+ offset += read;
53
+ budget -= read;
54
+ }
55
+ } finally {
56
+ closeSync(fd);
57
+ }
58
+ job.monitorOffset = offset;
59
+ }
60
+ return { continuationId: parser.continuationId, ...parser.metrics() };
61
+ } catch {
62
+ return { continuationId: null, turns: 0, cacheReadInputTokens: 0, toolCalls: 0, completed: false };
63
+ }
64
+ }
65
+
66
+ /**
67
+ * Drain what the live monitor has not read yet and return the per-request
68
+ * ledger the transcript proves. Called at settlement, before any outcome is
69
+ * decided: the gate caps the log to its last 512 KiB at close, so the head of
70
+ * a long turn -- the requests that show how its context grew -- survives only
71
+ * in what the monitor folded while the provider was alive (measured
72
+ * 2026-09-20: 132 of 715 stored logs sit exactly at the cap). A harness the
73
+ * monitor never observed (no live stream) is read here from offset zero.
74
+ *
75
+ * @param {Job} job
76
+ * @returns {import("../harnesses/session-metrics.mjs").SessionLedger}
77
+ */
78
+ export function sessionLedger(job) {
79
+ for (let call = 0; call < MONITOR_DRAIN_CALLS; call += 1) {
80
+ const before = job.monitorOffset ?? 0;
81
+ monitorInvocation(job);
82
+ if ((job.monitorOffset ?? 0) === before) break;
83
+ }
84
+ const parser = job.monitorParser ?? (job.monitorParser = new SessionMetricsParser(job.runtime.harness));
85
+ parser.flush();
86
+ return parser.session();
87
+ }
88
+
89
+ /**
90
+ * @param {string} path
91
+ * @param {number} maxBytes
92
+ * @returns {string}
93
+ */
94
+ export function boundedRegion(path, maxBytes = MAX_PROVIDER_LOG_BYTES) {
95
+ try {
96
+ return dropPartialLogLine(readFileSync(`${path}.tail`, "utf8"));
97
+ } catch (error) {
98
+ if (errorCode(error) !== "ENOENT") throw error;
99
+ }
100
+ const size = statSync(path).size;
101
+ if (size <= maxBytes) return readFileSync(path, "utf8");
102
+ const fd = openSync(path, "r");
103
+ try {
104
+ const bytes = Buffer.alloc(maxBytes);
105
+ readSync(fd, bytes, 0, maxBytes, size - maxBytes);
106
+ return dropPartialLogLine(bytes.toString("utf8"));
107
+ } finally {
108
+ closeSync(fd);
109
+ }
110
+ }
111
+
112
+ /**
113
+ * @param {string} path
114
+ * @param {number} [maxBytes]
115
+ * @returns {string}
116
+ */
117
+ export function readBoundedTail(path, maxBytes = 512 * 1024) {
118
+ try {
119
+ try { return dropPartialLogLine(readFileSync(`${path}.tail`, "utf8")); } catch (tailError) {
120
+ if (errorCode(tailError) !== "ENOENT") throw tailError;
121
+ }
122
+ const size = statSync(path).size;
123
+ if (size <= maxBytes) return readFileSync(path, "utf8");
124
+ const fd = openSync(path, "r");
125
+ try {
126
+ const bytes = Buffer.alloc(maxBytes);
127
+ readSync(fd, bytes, 0, maxBytes, size - maxBytes);
128
+ return dropPartialLogLine(bytes.toString("utf8"));
129
+ } finally {
130
+ closeSync(fd);
131
+ }
132
+ } catch (error) {
133
+ if (errorCode(error) === "ENOENT") return "";
134
+ throw error;
135
+ }
136
+ }
137
+
138
+ /**
139
+ * @param {unknown} value
140
+ * @returns {string}
141
+ */
142
+ function dropPartialLogLine(value) {
143
+ const newline = String(value).indexOf("\n");
144
+ return newline < 0 ? "" : String(value).slice(newline + 1);
145
+ }
@@ -84,6 +84,11 @@ export const claudeHarness = {
84
84
  ];
85
85
  if (options.toolPolicy) args.push("--settings", JSON.stringify(hookSettings(options.toolPolicy)));
86
86
  if (runtime.reasoning) args.push("--effort", runtime.reasoning);
87
+ // The attempt's request ceiling, enforced by the CLI itself; the
88
+ // controller's monitor enforces the same number for every streaming
89
+ // harness, so this only makes the stop cleaner (a result event instead of
90
+ // a kill) for the one harness that can take it as a flag.
91
+ if (typeof options.maxTurns === "number") args.push("--max-turns", String(options.maxTurns));
87
92
  if (options.schema) args.push("--json-schema", JSON.stringify(options.schema));
88
93
  return { executable: this.executable(runtime), args, promptTransport: "stdin", input: prompt };
89
94
  },