faberun 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/faberun/references/contract.md +10 -10
- package/src/contract/assert.mjs +5 -1
- package/src/contract/index.mjs +43 -7
- package/src/contract/runtime.mjs +4 -1
- package/src/contract/snapshot.mjs +25 -1
- package/src/contract/task-packet.mjs +5 -1
- package/src/engine/backoff.mjs +2 -1
- package/src/engine/capacity.mjs +74 -0
- package/src/engine/dispatch.mjs +12 -186
- package/src/engine/lifecycle.mjs +7 -2
- package/src/engine/phase-session.mjs +217 -0
- package/src/engine/process.mjs +26 -108
- package/src/engine/scheduler.mjs +15 -3
- package/src/engine/settle.mjs +11 -2
- package/src/engine/transcript.mjs +145 -0
- package/src/harnesses/claude/index.mjs +5 -0
- package/src/harnesses/dsh/runner.mjs +38 -3
- package/src/harnesses/exec-jsonl/index.mjs +0 -499
- package/src/harnesses/index.mjs +55 -8
- package/src/harnesses/protocol.mjs +22 -2
- package/src/harnesses/session-metrics.mjs +674 -0
- package/src/host/tool-policy-decisions.mjs +78 -21
- package/src/host/tool-policy-hook.mjs +12 -3
- package/src/plan/freeze.mjs +24 -6
- package/src/plan/pipeline.mjs +232 -67
- package/src/plan/sizing.mjs +0 -0
- package/src/plan/template.mjs +110 -8
- package/src/run/operations.mjs +1 -1
- package/src/run/usage.mjs +9 -3
- package/src/util.mjs +0 -0
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which provider session a node's next invocation runs in: fresh, a
|
|
3
|
+
* continuation of a compatible earlier session in the same plan phase, or a
|
|
4
|
+
* rotation -- a fresh session that carries the prior nodes' structured
|
|
5
|
+
* summaries. Separate from dispatch.mjs, which spawns whatever this decides,
|
|
6
|
+
* because the decision reads only persisted node snapshots and the routing
|
|
7
|
+
* table, and because dispatch.mjs crossed the 800-line ceiling carrying it.
|
|
8
|
+
*/
|
|
9
|
+
import { basename, join } from "node:path";
|
|
10
|
+
import { createHash } from "node:crypto";
|
|
11
|
+
import { boundedUtf8, stableJson } from "../util.mjs";
|
|
12
|
+
import { providerCommand } from "../harnesses/index.mjs";
|
|
13
|
+
import { readJson } from "../run/store.mjs";
|
|
14
|
+
import { routeRuntimeForState } from "./failover.mjs";
|
|
15
|
+
import { validateNodeSnapshot } from "../contract/snapshot.mjs";
|
|
16
|
+
|
|
17
|
+
/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
|
|
18
|
+
/** @typedef {import("../contract/index.mjs").ValidatedNode} ValidatedNode */
|
|
19
|
+
/** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
|
|
20
|
+
/** @typedef {import("../contract/index.mjs").RuntimeSnapshot} RuntimeSnapshot */
|
|
21
|
+
/** @typedef {import("./process.mjs").Invocation} Invocation */
|
|
22
|
+
|
|
23
|
+
/** @typedef {{forceFresh?: boolean}} SessionPolicy */
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Resolve the session policy one dispatch runs under, then consume the copy the
|
|
27
|
+
* node persisted. The explicit argument comes from a caller that is dispatching
|
|
28
|
+
* on the spot; `state.sessionPolicy` is the copy a rejection decision left when
|
|
29
|
+
* it handed the node back to the scheduler, whose own `startWorker` call passes
|
|
30
|
+
* nothing at all.
|
|
31
|
+
*
|
|
32
|
+
* Persisting is the whole point: `phaseInvocationPlan` rediscovers a compatible
|
|
33
|
+
* continuation from the persisted ledger, so nulling a local continuation id at
|
|
34
|
+
* the call site would let the scheduler's later dispatch find it again. Clearing
|
|
35
|
+
* the stored policy here makes it one-shot — it governs exactly the dispatch it
|
|
36
|
+
* was recorded for, and the next unrelated attempt reuses normally.
|
|
37
|
+
*
|
|
38
|
+
* @param {{sessionPolicy?: SessionPolicy|null}} state
|
|
39
|
+
* @param {SessionPolicy} [explicit]
|
|
40
|
+
* @returns {SessionPolicy}
|
|
41
|
+
*/
|
|
42
|
+
export function forceFreshSession(state, explicit = {}) {
|
|
43
|
+
const persisted = /** @type {SessionPolicy|undefined} */ (state?.sessionPolicy ?? undefined);
|
|
44
|
+
const policy = { ...(persisted ?? {}), ...explicit };
|
|
45
|
+
if (state && state.sessionPolicy !== undefined && state.sessionPolicy !== null) state.sessionPolicy = null;
|
|
46
|
+
return policy;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Select the only continuation that is allowed for this plan phase and role.
|
|
51
|
+
* The search is intentionally limited to persisted node snapshots in this run.
|
|
52
|
+
*
|
|
53
|
+
* `policy.forceFresh` is the explicit session policy a rejection decision
|
|
54
|
+
* carries: it short-circuits the search before it can rediscover a compatible
|
|
55
|
+
* continuation, so a retry after a gate rejection starts a fresh provider
|
|
56
|
+
* session instead of re-reading the failed transcript. Nulling a local id at
|
|
57
|
+
* the call site is not enough, because this function rediscovers the prior
|
|
58
|
+
* continuation from the persisted ledger.
|
|
59
|
+
*
|
|
60
|
+
* @param {ValidatedContract} contract
|
|
61
|
+
* @param {ValidatedNode} node
|
|
62
|
+
* @param {NodeSnapshot} state
|
|
63
|
+
* @param {string} runDir
|
|
64
|
+
* @param {"worker"|"judge"} role
|
|
65
|
+
* @param {string} prompt
|
|
66
|
+
* @param {SessionPolicy} [policy]
|
|
67
|
+
* @returns {{prompt: string, continuationId: string|null, mode: "fresh"|"reuse"|"rotate"}}
|
|
68
|
+
*/
|
|
69
|
+
export function phaseInvocationPlan(contract, node, state, runDir, role, prompt, policy = {}) {
|
|
70
|
+
if (policy.forceFresh === true) {
|
|
71
|
+
return { prompt, continuationId: null, mode: "fresh" };
|
|
72
|
+
}
|
|
73
|
+
const runId = basename(runDir);
|
|
74
|
+
const session = phaseSessionCandidates(contract, node, state, runDir, role).at(-1);
|
|
75
|
+
const runtime = routeRuntimeForState(contract, node, state, role);
|
|
76
|
+
const identityMatches = session && session.invocation.runId === runId
|
|
77
|
+
&& session.invocation.campaignId === contract.campaignId
|
|
78
|
+
&& session.invocation.planPhase === node.phase
|
|
79
|
+
&& session.invocation.role === role
|
|
80
|
+
&& session.invocation.harness === runtime.harness
|
|
81
|
+
&& session.invocation.runtimeId === runtime.id
|
|
82
|
+
&& session.invocation.runtimeFingerprint === fingerprintRuntime(runtime)
|
|
83
|
+
&& session.invocation.model === runtime.model
|
|
84
|
+
&& session.invocation.reasoning === (runtime.reasoning ?? null)
|
|
85
|
+
&& session.invocation.sandbox === (runtime.sandbox ?? null);
|
|
86
|
+
const canContinue = runtime.capabilities.continuation === true;
|
|
87
|
+
// A node continuing its own earlier session keeps it. A phase sibling's
|
|
88
|
+
// session is reused only when the contract opts in (`phaseSessionReuse`):
|
|
89
|
+
// measured 2026-09-20 over 21 runs with both kinds of turn, a turn opened
|
|
90
|
+
// on a sibling's session cost 1.87x the fresh one at the same request
|
|
91
|
+
// count -- it began with 200k tokens of context instead of 45k and re-read
|
|
92
|
+
// them on every request -- while the rotation below hands the sibling's
|
|
93
|
+
// structured summary to a fresh session whose first request is already 90%
|
|
94
|
+
// served from the shared prefix cache.
|
|
95
|
+
const ownSession = session !== undefined && session.nodeId === node.id;
|
|
96
|
+
if (identityMatches && canContinue && (ownSession || contract.phaseSessionReuse === true)) {
|
|
97
|
+
return { prompt, continuationId: session.invocation.continuationId ?? null, mode: "reuse" };
|
|
98
|
+
}
|
|
99
|
+
// A harness that cannot continue at all, or a session picked up from a
|
|
100
|
+
// different phase-sibling node whose identity does not match this one, has
|
|
101
|
+
// no native continuity: the fresh attempt carries the prior nodes'
|
|
102
|
+
// structured summaries forward instead of starting blind.
|
|
103
|
+
if (session && (!canContinue || session.nodeId !== node.id)) {
|
|
104
|
+
return {
|
|
105
|
+
prompt: phaseHandoffPrompt(contract, node, state, runDir, role),
|
|
106
|
+
continuationId: null,
|
|
107
|
+
mode: "rotate",
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
// A capable harness continuing its own node whose identity merely drifted
|
|
111
|
+
// (the run directory moved, or a runtime edge) still gets the caller's own
|
|
112
|
+
// prompt — already carrying the node's bounded "Previous attempt" section —
|
|
113
|
+
// in a fresh session, never a synthesized handoff.
|
|
114
|
+
return { prompt, continuationId: null, mode: session ? "rotate" : "fresh" };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Continuation ids a live invocation is already driving, anywhere in the run.
|
|
119
|
+
*
|
|
120
|
+
* This is what makes concurrent nodes of one phase safe, and it is read from
|
|
121
|
+
* the persisted ledger rather than from an in-memory registry so a controller
|
|
122
|
+
* that took over a run inherits the claims instead of racing them.
|
|
123
|
+
*
|
|
124
|
+
* @param {ValidatedContract} contract
|
|
125
|
+
* @param {NodeSnapshot} currentState
|
|
126
|
+
* @param {string} runDir
|
|
127
|
+
* @returns {Set<string>}
|
|
128
|
+
*/
|
|
129
|
+
function claimedContinuations(contract, currentState, runDir) {
|
|
130
|
+
/** @type {Set<string>} */
|
|
131
|
+
const claimed = new Set();
|
|
132
|
+
for (const candidate of contract.nodes) {
|
|
133
|
+
let state = candidate.id === currentState.id ? currentState : null;
|
|
134
|
+
if (!state) {
|
|
135
|
+
try { state = validateNodeSnapshot(readJson(join(runDir, "nodes", `${candidate.id}.json`)), candidate); } catch { continue; }
|
|
136
|
+
}
|
|
137
|
+
for (const invocation of state.invocations ?? []) {
|
|
138
|
+
if (invocation.status === "active" && invocation.continuationId) claimed.add(invocation.continuationId);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return claimed;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* @param {ValidatedContract} contract
|
|
146
|
+
* @param {ValidatedNode} node
|
|
147
|
+
* @param {NodeSnapshot} currentState
|
|
148
|
+
* @param {string} runDir
|
|
149
|
+
* @param {"worker"|"judge"} role
|
|
150
|
+
* @returns {{nodeId: string, invocation: Invocation}[]}
|
|
151
|
+
*/
|
|
152
|
+
function phaseSessionCandidates(contract, node, currentState, runDir, role) {
|
|
153
|
+
/** @type {{nodeId: string, invocation: Invocation}[]} */
|
|
154
|
+
const candidates = [];
|
|
155
|
+
const claimed = claimedContinuations(contract, currentState, runDir);
|
|
156
|
+
for (const candidate of contract.nodes) {
|
|
157
|
+
if (candidate.phase !== node.phase) continue;
|
|
158
|
+
let state = candidate.id === currentState.id ? currentState : null;
|
|
159
|
+
if (!state) {
|
|
160
|
+
try { state = validateNodeSnapshot(readJson(join(runDir, "nodes", `${candidate.id}.json`)), candidate); } catch { continue; }
|
|
161
|
+
}
|
|
162
|
+
for (const invocation of state.invocations ?? []) {
|
|
163
|
+
if (invocation.role !== role || invocation.planPhase !== node.phase || !invocation.continuationId) continue;
|
|
164
|
+
if (invocation.nodeId !== candidate.id || invocation.attempt !== state.attempt || invocation.workspace !== state.worktree?.path) continue;
|
|
165
|
+
// One provider session, one live turn. With `maxParallel` above one,
|
|
166
|
+
// two nodes of a phase can be dispatched in the same tick, and without
|
|
167
|
+
// this both would hand the same continuation id to their own provider
|
|
168
|
+
// process. The claim is read from the persisted ledger, which the
|
|
169
|
+
// in-tick dispatch already wrote for the node that went first.
|
|
170
|
+
if (claimed.has(invocation.continuationId)) continue;
|
|
171
|
+
candidates.push({ nodeId: candidate.id, invocation });
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
return candidates.sort((left, right) => {
|
|
175
|
+
const leftStarted = Date.parse(left.invocation.startedAt);
|
|
176
|
+
const rightStarted = Date.parse(right.invocation.startedAt);
|
|
177
|
+
if (leftStarted !== rightStarted) return leftStarted - rightStarted;
|
|
178
|
+
const leftUpdated = Date.parse(left.invocation.updatedAt);
|
|
179
|
+
const rightUpdated = Date.parse(right.invocation.updatedAt);
|
|
180
|
+
if (leftUpdated !== rightUpdated) return leftUpdated - rightUpdated;
|
|
181
|
+
return left.invocation.id.localeCompare(right.invocation.id);
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** @param {RuntimeSnapshot} runtime @returns {string} */
|
|
186
|
+
export function fingerprintRuntime(runtime) {
|
|
187
|
+
const executable = providerCommand(runtime, "").executable;
|
|
188
|
+
return createHash("sha256").update(stableJson({ runtime, executable })).digest("hex");
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** @param {ValidatedContract} contract @param {ValidatedNode} node @param {NodeSnapshot} state @param {string} runDir @param {"worker"|"judge"} role @returns {string} */
|
|
192
|
+
function phaseHandoffPrompt(contract, node, state, runDir, role) {
|
|
193
|
+
const summaries = phaseSessionCandidates(contract, node, state, runDir, role)
|
|
194
|
+
.map(({ nodeId }) => {
|
|
195
|
+
const candidate = contract.nodes.find((item) => item.id === nodeId);
|
|
196
|
+
let snapshot = null;
|
|
197
|
+
try { snapshot = readJson(join(runDir, "nodes", `${nodeId}.json`)); } catch {
|
|
198
|
+
// ENOENT or unreadable snapshot: this prior node contributes no summary.
|
|
199
|
+
}
|
|
200
|
+
const result = snapshot?.result;
|
|
201
|
+
const record = result && typeof result === "object" && !Array.isArray(result)
|
|
202
|
+
? /** @type {Record<string, unknown>} */ (result)
|
|
203
|
+
: null;
|
|
204
|
+
const summary = typeof record?.summary === "string" ? record.summary : null;
|
|
205
|
+
return summary && candidate ? `${candidate.id}: ${boundedUtf8(summary, 1024)}` : null;
|
|
206
|
+
})
|
|
207
|
+
.filter(Boolean)
|
|
208
|
+
.slice(-8);
|
|
209
|
+
const handoff = [
|
|
210
|
+
`Continue phase ${node.phase} as the ${role} agent in a fresh provider session.`,
|
|
211
|
+
"Prior structured node summaries:",
|
|
212
|
+
summaries.length ? summaries.map((summary) => `- ${summary}`).join("\n") : "- (none)",
|
|
213
|
+
"Current closed task packet:",
|
|
214
|
+
boundedUtf8(node.prompt, 48 * 1024),
|
|
215
|
+
].join("\n\n");
|
|
216
|
+
return boundedUtf8(handoff, 60 * 1024);
|
|
217
|
+
}
|
package/src/engine/process.mjs
CHANGED
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
* a judge decides, or when a run is done. That separation is the point: a stuck
|
|
8
8
|
* provider is killed by the same code whatever it was asked to do.
|
|
9
9
|
*/
|
|
10
|
-
import {
|
|
11
|
-
import { closeSync, existsSync, fsyncSync, openSync,
|
|
10
|
+
import { boundedRegion, monitorInvocation } from "./transcript.mjs";
|
|
11
|
+
import { closeSync, existsSync, fsyncSync, openSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
|
|
12
12
|
import { dirname, join } from "node:path";
|
|
13
13
|
import { errorCode, errorMessage } from "../util.mjs";
|
|
14
14
|
import { fileURLToPath } from "node:url";
|
|
@@ -33,19 +33,14 @@ import { writeNodeSnapshot } from "../run/node-store.mjs";
|
|
|
33
33
|
/** @typedef {ProviderEnvelope & {costProvenance?: "priced"}} PricedEnvelope */
|
|
34
34
|
/** @typedef {import("node:child_process").ChildProcess} ChildProcess */
|
|
35
35
|
/** @typedef {{prompt: string|null, stdout: string, stderr: string}} PathSet */
|
|
36
|
-
/** @typedef {{id: string, pid: number, processGroupId: number|null, processStartToken: string|null, harness: string, runtimeId: string|null, runtimeFingerprint?: string, revision?: number, phase: string, promptPath: string|null, stdoutPath: string, stderrPath: string, startedAt: string, deadlineAt: string|null, updatedAt: string, closedAt: string|null, exitCode: number|null, signal: string|null, status: "active"|"closed"|"terminated", executable: string, snapshotPath?: string, usage?: Usage, usageEstimated?: boolean, costUsd?: number|null, costProvenance?: "priced", runId?: string, campaignId?: string, nodeId?: string, attempt?: number, workspace?: string, worktreeBranch?: string|null, worktreeBaseSha?: string|null, planPhase?: string, role?: "worker"|"judge", model?: string, reasoning?: string|null, sandbox?: string|null, continuationId?: string|null, continuationMode?: "fresh"|"reuse"|"rotate"}} Invocation */
|
|
36
|
+
/** @typedef {{id: string, pid: number, processGroupId: number|null, processStartToken: string|null, harness: string, runtimeId: string|null, runtimeFingerprint?: string, revision?: number, phase: string, promptPath: string|null, stdoutPath: string, stderrPath: string, startedAt: string, deadlineAt: string|null, updatedAt: string, closedAt: string|null, exitCode: number|null, signal: string|null, status: "active"|"closed"|"terminated", executable: string, snapshotPath?: string, usage?: Usage, usageEstimated?: boolean, costUsd?: number|null, costProvenance?: "priced", runId?: string, campaignId?: string, nodeId?: string, attempt?: number, workspace?: string, worktreeBranch?: string|null, worktreeBaseSha?: string|null, planPhase?: string, role?: "worker"|"judge", model?: string, reasoning?: string|null, sandbox?: string|null, continuationId?: string|null, continuationMode?: "fresh"|"reuse"|"rotate", session?: import("../harnesses/session-metrics.mjs").SessionLedger|null}} Invocation */
|
|
37
37
|
/** @typedef {{pid: number|null, processGroupId?: number|null, processStartToken?: string|null}} InvocationProbe */
|
|
38
|
-
/** @typedef {{child: ChildProcess, contract: ValidatedContract, node: ValidatedNode, state: NodeSnapshot, runtime: HarnessRuntime & {id: string|null}, cwd: string, paths: PathSet, phase: string, invocation: Invocation, startedAt: string, startedTicks: bigint, progressTicks: bigint, lastOutputAt: number, closed: boolean, exitCode: number|null, signal: string|null, spawnError: Error|null, terminating: Promise<void>|null, gateConfigPath: string, gateReleasePath: string, scopeBaseline?: unknown, scopeChecked?: boolean, scopeViolation?: boolean, resultMaterialization?: boolean, recoveryBaseline?: unknown, observeTimer?: ReturnType<typeof setInterval>, monitorOffset?: number, monitorParser?: import("../harnesses/
|
|
38
|
+
/** @typedef {{child: ChildProcess, contract: ValidatedContract, node: ValidatedNode, state: NodeSnapshot, runtime: HarnessRuntime & {id: string|null}, cwd: string, paths: PathSet, phase: string, invocation: Invocation, startedAt: string, startedTicks: bigint, progressTicks: bigint, lastOutputAt: number, closed: boolean, exitCode: number|null, signal: string|null, spawnError: Error|null, terminating: Promise<void>|null, gateConfigPath: string, gateReleasePath: string, scopeBaseline?: unknown, scopeChecked?: boolean, scopeViolation?: boolean, resultMaterialization?: boolean, recoveryBaseline?: unknown, observeTimer?: ReturnType<typeof setInterval>, monitorOffset?: number, monitorParser?: import("../harnesses/session-metrics.mjs").SessionMetricsParser, lastEventCount?: number, observedOnce?: boolean, onClose?: (invocation: Invocation) => void, onInvocationUpdate?: (invocation: Invocation) => void, onProgress?: (state: NodeSnapshot) => void}} Job */
|
|
39
39
|
/** @typedef {{graceMs?: number, killGraceMs?: number, escalate?: boolean, runDir?: string, kill?: (pid: number, signal: string|number) => unknown, child?: ChildProcess|null}} TerminateOptions */
|
|
40
40
|
|
|
41
41
|
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
42
42
|
const DEFAULT_GRACE_MS = 2_000;
|
|
43
43
|
const GATE_PATH = join(HERE, "gate.mjs");
|
|
44
|
-
const MAX_PROVIDER_LOG_BYTES = 512 * 1024;
|
|
45
|
-
/** Fixed-size read for incremental transcript observation. */
|
|
46
|
-
const MONITOR_CHUNK_BYTES = 64 * 1024;
|
|
47
|
-
/** Per-observation read budget: one tick never blocks on a huge backlog. */
|
|
48
|
-
const MONITOR_CALL_BUDGET_BYTES = 1024 * 1024;
|
|
49
44
|
/**
|
|
50
45
|
* @param {{contract: ValidatedContract, node: ValidatedNode, state: NodeSnapshot, runtime: HarnessRuntime & {id: string|null}, prompt: string, paths: PathSet, phase: string, workspace?: string, commandOptions?: import("../harnesses/index.mjs").CommandOptions, onInvocation: (invocation: Invocation, job: Job) => void, onInvocationUpdate?: (invocation: Invocation) => void, onProgress?: (state: NodeSnapshot) => void}} args
|
|
51
46
|
* @returns {Job}
|
|
@@ -201,47 +196,6 @@ function observeInvocation(job) {
|
|
|
201
196
|
// id must not be able to kill the job that is being observed.
|
|
202
197
|
}
|
|
203
198
|
}
|
|
204
|
-
/**
|
|
205
|
-
* Observe the transcript incrementally: read only the bytes appended since
|
|
206
|
-
* the last observation, in fixed-size chunks folded into a parser whose
|
|
207
|
-
* retained state never scales with the unread length — so the metrics
|
|
208
|
-
* survive both a transcript that outgrows any fixed window and an
|
|
209
|
-
* already-large transcript on the first call after a controller restart.
|
|
210
|
-
* The gate caps the log only at close, so byte offsets stay valid while the
|
|
211
|
-
* provider is live. Only newline-terminated records are evidence; a
|
|
212
|
-
* trailing partial record stays unconsumed for the next observation. The
|
|
213
|
-
* generic metrics are zero for a provider that does not expose them.
|
|
214
|
-
*
|
|
215
|
-
* @param {Job} job
|
|
216
|
-
* @returns {{continuationId: string|null, turns: number, cacheReadInputTokens: number, toolCalls: number, completed: boolean}}
|
|
217
|
-
*/
|
|
218
|
-
export function monitorInvocation(job) {
|
|
219
|
-
try {
|
|
220
|
-
const parser = job.monitorParser ?? (job.monitorParser = new SessionMetricsParser(job.runtime.harness));
|
|
221
|
-
const size = statSync(job.paths.stdout).size;
|
|
222
|
-
let offset = job.monitorOffset ?? 0;
|
|
223
|
-
let budget = MONITOR_CALL_BUDGET_BYTES;
|
|
224
|
-
if (size > offset) {
|
|
225
|
-
const fd = openSync(job.paths.stdout, "r");
|
|
226
|
-
try {
|
|
227
|
-
const chunk = Buffer.alloc(MONITOR_CHUNK_BYTES);
|
|
228
|
-
while (offset < size && budget > 0) {
|
|
229
|
-
const read = readSync(fd, chunk, 0, Math.min(chunk.length, size - offset, budget), offset);
|
|
230
|
-
if (read <= 0) break;
|
|
231
|
-
parser.push(chunk.subarray(0, read));
|
|
232
|
-
offset += read;
|
|
233
|
-
budget -= read;
|
|
234
|
-
}
|
|
235
|
-
} finally {
|
|
236
|
-
closeSync(fd);
|
|
237
|
-
}
|
|
238
|
-
job.monitorOffset = offset;
|
|
239
|
-
}
|
|
240
|
-
return { continuationId: parser.continuationId, ...parser.metrics() };
|
|
241
|
-
} catch {
|
|
242
|
-
return { continuationId: null, turns: 0, cacheReadInputTokens: 0, toolCalls: 0, completed: false };
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
199
|
/**
|
|
246
200
|
* @param {string} path
|
|
247
201
|
*/
|
|
@@ -321,7 +275,7 @@ export async function terminateInvocation(invocation, options = {}) {
|
|
|
321
275
|
* sealing here, a timeout parks with an empty seal and the recorded work is
|
|
322
276
|
* abandoned in a worktree the next attempt never reads.
|
|
323
277
|
*/
|
|
324
|
-
const SEAL_BEFORE_KILL_CODES = new Set(["wall_clock_timeout", "stall_timeout"]);
|
|
278
|
+
const SEAL_BEFORE_KILL_CODES = new Set(["wall_clock_timeout", "stall_timeout", "turn_limit"]);
|
|
325
279
|
|
|
326
280
|
/**
|
|
327
281
|
* How long a `SIGSTOP`ped process group is given to actually stop before the
|
|
@@ -393,8 +347,8 @@ function resumeInvocation(invocation) {
|
|
|
393
347
|
}
|
|
394
348
|
|
|
395
349
|
/**
|
|
396
|
-
* The pre-termination seam, filled: on `wall_clock_timeout
|
|
397
|
-
* quiesce the provider, seal the attempt worktree, and only then let the caller
|
|
350
|
+
* The pre-termination seam, filled: on `wall_clock_timeout`, `stall_timeout`
|
|
351
|
+
* and `turn_limit` quiesce the provider, seal the attempt worktree, and only then let the caller
|
|
398
352
|
* terminate it, so the next attempt is cut from the seal. The seal is bounded
|
|
399
353
|
* by the same git timeout every synchronous git call uses
|
|
400
354
|
* (`GIT_SYNC_TIMEOUT_MS`, overridable with `FABERUN_GIT_TIMEOUT_MS`);
|
|
@@ -487,6 +441,25 @@ export async function detectStalls(contract, running, onTimeout, onProgress, onB
|
|
|
487
441
|
// as liveness, not only for an mtime that no longer does.
|
|
488
442
|
job.lastOutputAt = Date.now();
|
|
489
443
|
}
|
|
444
|
+
// The attempt's request ceiling: a turn still making requests past it
|
|
445
|
+
// is the runaway shape, not progress. measured 2026-09-20: two 600-request
|
|
446
|
+
// turns re-sent 190M and 167M tokens of context and produced no result;
|
|
447
|
+
// the 23 turns with no result held 25% of all context spend. Ends the
|
|
448
|
+
// attempt the way a timeout does: sealed, then one automatic retry.
|
|
449
|
+
// (`turns` counts provider requests for claude, dsh and agy; codex
|
|
450
|
+
// reports whole turns, so its cap is in effect a turn count.)
|
|
451
|
+
const turnCap = job.node?.maxTurns ?? contract.maxTurns;
|
|
452
|
+
if (typeof turnCap === "number" && monitored.turns >= turnCap) {
|
|
453
|
+
const limit = {
|
|
454
|
+
code: "turn_limit",
|
|
455
|
+
message: `${job.phase} made ${monitored.turns} provider requests, the attempt's maxTurns of ${turnCap}`,
|
|
456
|
+
};
|
|
457
|
+
await onBeforeTerminate(job, limit);
|
|
458
|
+
await terminateProcess(job);
|
|
459
|
+
running.delete(nodeId);
|
|
460
|
+
await onTimeout(job, "exhausted", limit);
|
|
461
|
+
continue;
|
|
462
|
+
}
|
|
490
463
|
} else if (job.observedOnce !== true) {
|
|
491
464
|
job.progressTicks = now;
|
|
492
465
|
job.lastOutputAt = Date.now();
|
|
@@ -583,28 +556,6 @@ export function priceUsage(runtime, usage, reportedCostUsd) {
|
|
|
583
556
|
}
|
|
584
557
|
return { costUsd: total / 1_000_000, costProvenance: "priced" };
|
|
585
558
|
}
|
|
586
|
-
/**
|
|
587
|
-
* @param {string} path
|
|
588
|
-
* @param {number} maxBytes
|
|
589
|
-
* @returns {string}
|
|
590
|
-
*/
|
|
591
|
-
function boundedRegion(path, maxBytes = MAX_PROVIDER_LOG_BYTES) {
|
|
592
|
-
try {
|
|
593
|
-
return dropPartialLogLine(readFileSync(`${path}.tail`, "utf8"));
|
|
594
|
-
} catch (error) {
|
|
595
|
-
if (errorCode(error) !== "ENOENT") throw error;
|
|
596
|
-
}
|
|
597
|
-
const size = statSync(path).size;
|
|
598
|
-
if (size <= maxBytes) return readFileSync(path, "utf8");
|
|
599
|
-
const fd = openSync(path, "r");
|
|
600
|
-
try {
|
|
601
|
-
const bytes = Buffer.alloc(maxBytes);
|
|
602
|
-
readSync(fd, bytes, 0, maxBytes, size - maxBytes);
|
|
603
|
-
return dropPartialLogLine(bytes.toString("utf8"));
|
|
604
|
-
} finally {
|
|
605
|
-
closeSync(fd);
|
|
606
|
-
}
|
|
607
|
-
}
|
|
608
559
|
/**
|
|
609
560
|
* Signal the invocation's process group only when ownership is proven. Never
|
|
610
561
|
* throws: ESRCH is gone, EPERM is a group this user cannot signal and therefore
|
|
@@ -746,36 +697,3 @@ export function logPaths(runDir, nodeId, phase, attempt) {
|
|
|
746
697
|
stderr: join(runDir, "logs", `${stem}.err`),
|
|
747
698
|
};
|
|
748
699
|
}
|
|
749
|
-
/**
|
|
750
|
-
* @param {string} path
|
|
751
|
-
* @param {number} [maxBytes]
|
|
752
|
-
* @returns {string}
|
|
753
|
-
*/
|
|
754
|
-
export function readBoundedTail(path, maxBytes = 512 * 1024) {
|
|
755
|
-
try {
|
|
756
|
-
try { return dropPartialLogLine(readFileSync(`${path}.tail`, "utf8")); } catch (tailError) {
|
|
757
|
-
if (errorCode(tailError) !== "ENOENT") throw tailError;
|
|
758
|
-
}
|
|
759
|
-
const size = statSync(path).size;
|
|
760
|
-
if (size <= maxBytes) return readFileSync(path, "utf8");
|
|
761
|
-
const fd = openSync(path, "r");
|
|
762
|
-
try {
|
|
763
|
-
const bytes = Buffer.alloc(maxBytes);
|
|
764
|
-
readSync(fd, bytes, 0, maxBytes, size - maxBytes);
|
|
765
|
-
return dropPartialLogLine(bytes.toString("utf8"));
|
|
766
|
-
} finally {
|
|
767
|
-
closeSync(fd);
|
|
768
|
-
}
|
|
769
|
-
} catch (error) {
|
|
770
|
-
if (errorCode(error) === "ENOENT") return "";
|
|
771
|
-
throw error;
|
|
772
|
-
}
|
|
773
|
-
}
|
|
774
|
-
/**
|
|
775
|
-
* @param {unknown} value
|
|
776
|
-
* @returns {string}
|
|
777
|
-
*/
|
|
778
|
-
function dropPartialLogLine(value) {
|
|
779
|
-
const newline = String(value).indexOf("\n");
|
|
780
|
-
return newline < 0 ? "" : String(value).slice(newline + 1);
|
|
781
|
-
}
|
package/src/engine/scheduler.mjs
CHANGED
|
@@ -7,7 +7,8 @@ import {
|
|
|
7
7
|
applyJudgeRound,
|
|
8
8
|
} from "./review.mjs";
|
|
9
9
|
import { CONTRACT_VERSION, PROTOCOL_SCHEMA_VERSION } from "../harnesses/index.mjs";
|
|
10
|
-
import { routingBackoffActive } from "./failover.mjs";
|
|
10
|
+
import { routeRuntimeForState, routingBackoffActive } from "./failover.mjs";
|
|
11
|
+
import { quotaHeldRuntimes, runningPerRuntime, runtimeHasCapacity } from "./capacity.mjs";
|
|
11
12
|
|
|
12
13
|
import {
|
|
13
14
|
bootstrapAttemptPath,
|
|
@@ -583,7 +584,7 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
|
|
|
583
584
|
// A judge killed on its own wall clock produced no verdict. That is a
|
|
584
585
|
// judge protocol defect, not a node outcome: it earns the one bounded
|
|
585
586
|
// re-ask, and only then the review mode settles the node.
|
|
586
|
-
if (job.phase === "judge" && error.code === "wall_clock_timeout") {
|
|
587
|
+
if (job.phase === "judge" && (error.code === "wall_clock_timeout" || error.code === "turn_limit")) {
|
|
587
588
|
await applyJudgeProtocolFailure(contract, job.node, job.state, runDir, running, lock, states, campaign.path, error.message);
|
|
588
589
|
return;
|
|
589
590
|
}
|
|
@@ -614,9 +615,20 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
|
|
|
614
615
|
return state?.status === "pending" && !pendingSettlements.has(node.id)
|
|
615
616
|
&& node.dependsOn.every((id) => states.get(id)?.status === "done");
|
|
616
617
|
});
|
|
617
|
-
|
|
618
|
+
// Per-runtime capacity is judged per dispatch, not per tick: the
|
|
619
|
+
// counts include what this tick has already started, and a runtime a
|
|
620
|
+
// sibling is waiting out a quota reset on accepts nothing new.
|
|
621
|
+
const counts = runningPerRuntime(running.values());
|
|
622
|
+
const held = quotaHeldRuntimes(states.values(), Date.now());
|
|
623
|
+
let dispatched = 0;
|
|
624
|
+
for (const node of ready) {
|
|
625
|
+
if (dispatched >= slots) break;
|
|
618
626
|
const state = states.get(node.id);
|
|
619
627
|
if (!state || routingBackoffActive(state, state.phase)) continue;
|
|
628
|
+
const routed = routeRuntimeForState(contract, node, state, state.phase === "judge" ? "judge" : "worker");
|
|
629
|
+
if (!runtimeHasCapacity(routed.id, contract, counts, held)) continue;
|
|
630
|
+
counts.set(routed.id, (counts.get(routed.id) ?? 0) + 1);
|
|
631
|
+
dispatched += 1;
|
|
620
632
|
// A node recovered pending a re-ask judge (its own worker attempt
|
|
621
633
|
// already accepted, `state.result` durable) reaches `settleDone` /
|
|
622
634
|
// `integrateAttempt` exactly like a closed job's own settlement
|
package/src/engine/settle.mjs
CHANGED
|
@@ -40,11 +40,20 @@ import { verifyCandidateWorkspace } from "./verify.mjs";
|
|
|
40
40
|
/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
|
|
41
41
|
/** @typedef {import("../contract/index.mjs").ValidatedNode} ValidatedNode */
|
|
42
42
|
|
|
43
|
-
/**
|
|
43
|
+
/**
|
|
44
|
+
* Settle one worker-generation rejection: bounded revision when one remains,
|
|
45
|
+
* otherwise terminal exhausted/failed. The revision budget is the node's
|
|
46
|
+
* (`gate.maxRevisions`, default 1) whether or not the gate reviews: a red
|
|
47
|
+
* deterministic verification earns the same fresh attempt with the failure in
|
|
48
|
+
* front of the worker that a judge rejection does. Measured 2026-09-20 in the
|
|
49
|
+
* orchestration-arms campaign: with the budget behind `gate.enabled`, a node
|
|
50
|
+
* under `gate: false` died on one timing test that flaked under load, and its
|
|
51
|
+
* dependant with it, while the judged twin of the same node got its retry.
|
|
52
|
+
* @param {ValidatedContract} contract @param {ValidatedNode} node @param {NodeSnapshot} state @param {string} runDir @param {Map<string, Job>|null} running @param {LockHandle} lock @param {Map<string, NodeSnapshot>} states @param {string} campaignPath @param {JudgeVerdict} verdict @param {{code: string, label: string, phase?: "worker"|"judge", message?: string, forceFresh?: boolean}} options */
|
|
44
53
|
export function applyRejection(contract, node, state, runDir, running, lock, states, campaignPath, verdict, options) {
|
|
45
54
|
const { code, label, phase = "worker", message = verdict.summary, forceFresh = true } = options;
|
|
46
55
|
state.gate = verdict;
|
|
47
|
-
if (
|
|
56
|
+
if (state.revisions < (node.gate.maxRevisions ?? 1)) {
|
|
48
57
|
resetPhaseRouting(state);
|
|
49
58
|
state.revisions += 1;
|
|
50
59
|
// The fresh-session decision travels with the node, not just this call:
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The provider transcript as the controller reads it: bounded tail reads at
|
|
3
|
+
* settlement, incremental observation while the provider runs (the liveness
|
|
4
|
+
* signal and the per-request ledger), and the drain at close. Separate from
|
|
5
|
+
* process.mjs, which owns the process itself -- spawn, gate, kill, seal --
|
|
6
|
+
* because reading the log never touches the process, and because process.mjs
|
|
7
|
+
* crossed the 800-line ceiling carrying both jobs.
|
|
8
|
+
*/
|
|
9
|
+
import { closeSync, openSync, readFileSync, readSync, statSync } from "node:fs";
|
|
10
|
+
import { Buffer } from "node:buffer";
|
|
11
|
+
import { SessionMetricsParser } from "../harnesses/session-metrics.mjs";
|
|
12
|
+
import { errorCode } from "../util.mjs";
|
|
13
|
+
|
|
14
|
+
/** @typedef {import("./process.mjs").Job} Job */
|
|
15
|
+
|
|
16
|
+
const MAX_PROVIDER_LOG_BYTES = 512 * 1024;
|
|
17
|
+
/** Fixed-size read for incremental transcript observation. */
|
|
18
|
+
const MONITOR_CHUNK_BYTES = 64 * 1024;
|
|
19
|
+
/** Per-observation read budget: one tick never blocks on a huge backlog. */
|
|
20
|
+
const MONITOR_CALL_BUDGET_BYTES = 1024 * 1024;
|
|
21
|
+
/** Settlement drain bound: 64 budgets is 64 MiB, above the largest stored transcript (25.8 MB, measured 2026-09-20). */
|
|
22
|
+
const MONITOR_DRAIN_CALLS = 64;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Observe the transcript incrementally: read only the bytes appended since
|
|
26
|
+
* the last observation, in fixed-size chunks folded into a parser whose
|
|
27
|
+
* retained state never scales with the unread length — so the metrics
|
|
28
|
+
* survive both a transcript that outgrows any fixed window and an
|
|
29
|
+
* already-large transcript on the first call after a controller restart.
|
|
30
|
+
* The gate caps the log only at close, so byte offsets stay valid while the
|
|
31
|
+
* provider is live. Only newline-terminated records are evidence; a
|
|
32
|
+
* trailing partial record stays unconsumed for the next observation. The
|
|
33
|
+
* generic metrics are zero for a provider that does not expose them.
|
|
34
|
+
*
|
|
35
|
+
* @param {Job} job
|
|
36
|
+
* @returns {{continuationId: string|null, turns: number, cacheReadInputTokens: number, toolCalls: number, completed: boolean}}
|
|
37
|
+
*/
|
|
38
|
+
export function monitorInvocation(job) {
|
|
39
|
+
try {
|
|
40
|
+
const parser = job.monitorParser ?? (job.monitorParser = new SessionMetricsParser(job.runtime.harness));
|
|
41
|
+
const size = statSync(job.paths.stdout).size;
|
|
42
|
+
let offset = job.monitorOffset ?? 0;
|
|
43
|
+
let budget = MONITOR_CALL_BUDGET_BYTES;
|
|
44
|
+
if (size > offset) {
|
|
45
|
+
const fd = openSync(job.paths.stdout, "r");
|
|
46
|
+
try {
|
|
47
|
+
const chunk = Buffer.alloc(MONITOR_CHUNK_BYTES);
|
|
48
|
+
while (offset < size && budget > 0) {
|
|
49
|
+
const read = readSync(fd, chunk, 0, Math.min(chunk.length, size - offset, budget), offset);
|
|
50
|
+
if (read <= 0) break;
|
|
51
|
+
parser.push(chunk.subarray(0, read));
|
|
52
|
+
offset += read;
|
|
53
|
+
budget -= read;
|
|
54
|
+
}
|
|
55
|
+
} finally {
|
|
56
|
+
closeSync(fd);
|
|
57
|
+
}
|
|
58
|
+
job.monitorOffset = offset;
|
|
59
|
+
}
|
|
60
|
+
return { continuationId: parser.continuationId, ...parser.metrics() };
|
|
61
|
+
} catch {
|
|
62
|
+
return { continuationId: null, turns: 0, cacheReadInputTokens: 0, toolCalls: 0, completed: false };
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Drain what the live monitor has not read yet and return the per-request
|
|
68
|
+
* ledger the transcript proves. Called at settlement, before any outcome is
|
|
69
|
+
* decided: the gate caps the log to its last 512 KiB at close, so the head of
|
|
70
|
+
* a long turn -- the requests that show how its context grew -- survives only
|
|
71
|
+
* in what the monitor folded while the provider was alive (measured
|
|
72
|
+
* 2026-09-20: 132 of 715 stored logs sit exactly at the cap). A harness the
|
|
73
|
+
* monitor never observed (no live stream) is read here from offset zero.
|
|
74
|
+
*
|
|
75
|
+
* @param {Job} job
|
|
76
|
+
* @returns {import("../harnesses/session-metrics.mjs").SessionLedger}
|
|
77
|
+
*/
|
|
78
|
+
export function sessionLedger(job) {
|
|
79
|
+
for (let call = 0; call < MONITOR_DRAIN_CALLS; call += 1) {
|
|
80
|
+
const before = job.monitorOffset ?? 0;
|
|
81
|
+
monitorInvocation(job);
|
|
82
|
+
if ((job.monitorOffset ?? 0) === before) break;
|
|
83
|
+
}
|
|
84
|
+
const parser = job.monitorParser ?? (job.monitorParser = new SessionMetricsParser(job.runtime.harness));
|
|
85
|
+
parser.flush();
|
|
86
|
+
return parser.session();
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* @param {string} path
|
|
91
|
+
* @param {number} maxBytes
|
|
92
|
+
* @returns {string}
|
|
93
|
+
*/
|
|
94
|
+
export function boundedRegion(path, maxBytes = MAX_PROVIDER_LOG_BYTES) {
|
|
95
|
+
try {
|
|
96
|
+
return dropPartialLogLine(readFileSync(`${path}.tail`, "utf8"));
|
|
97
|
+
} catch (error) {
|
|
98
|
+
if (errorCode(error) !== "ENOENT") throw error;
|
|
99
|
+
}
|
|
100
|
+
const size = statSync(path).size;
|
|
101
|
+
if (size <= maxBytes) return readFileSync(path, "utf8");
|
|
102
|
+
const fd = openSync(path, "r");
|
|
103
|
+
try {
|
|
104
|
+
const bytes = Buffer.alloc(maxBytes);
|
|
105
|
+
readSync(fd, bytes, 0, maxBytes, size - maxBytes);
|
|
106
|
+
return dropPartialLogLine(bytes.toString("utf8"));
|
|
107
|
+
} finally {
|
|
108
|
+
closeSync(fd);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* @param {string} path
|
|
114
|
+
* @param {number} [maxBytes]
|
|
115
|
+
* @returns {string}
|
|
116
|
+
*/
|
|
117
|
+
export function readBoundedTail(path, maxBytes = 512 * 1024) {
|
|
118
|
+
try {
|
|
119
|
+
try { return dropPartialLogLine(readFileSync(`${path}.tail`, "utf8")); } catch (tailError) {
|
|
120
|
+
if (errorCode(tailError) !== "ENOENT") throw tailError;
|
|
121
|
+
}
|
|
122
|
+
const size = statSync(path).size;
|
|
123
|
+
if (size <= maxBytes) return readFileSync(path, "utf8");
|
|
124
|
+
const fd = openSync(path, "r");
|
|
125
|
+
try {
|
|
126
|
+
const bytes = Buffer.alloc(maxBytes);
|
|
127
|
+
readSync(fd, bytes, 0, maxBytes, size - maxBytes);
|
|
128
|
+
return dropPartialLogLine(bytes.toString("utf8"));
|
|
129
|
+
} finally {
|
|
130
|
+
closeSync(fd);
|
|
131
|
+
}
|
|
132
|
+
} catch (error) {
|
|
133
|
+
if (errorCode(error) === "ENOENT") return "";
|
|
134
|
+
throw error;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* @param {unknown} value
|
|
140
|
+
* @returns {string}
|
|
141
|
+
*/
|
|
142
|
+
function dropPartialLogLine(value) {
|
|
143
|
+
const newline = String(value).indexOf("\n");
|
|
144
|
+
return newline < 0 ? "" : String(value).slice(newline + 1);
|
|
145
|
+
}
|
|
@@ -84,6 +84,11 @@ export const claudeHarness = {
|
|
|
84
84
|
];
|
|
85
85
|
if (options.toolPolicy) args.push("--settings", JSON.stringify(hookSettings(options.toolPolicy)));
|
|
86
86
|
if (runtime.reasoning) args.push("--effort", runtime.reasoning);
|
|
87
|
+
// The attempt's request ceiling, enforced by the CLI itself; the
|
|
88
|
+
// controller's monitor enforces the same number for every streaming
|
|
89
|
+
// harness, so this only makes the stop cleaner (a result event instead of
|
|
90
|
+
// a kill) for the one harness that can take it as a flag.
|
|
91
|
+
if (typeof options.maxTurns === "number") args.push("--max-turns", String(options.maxTurns));
|
|
87
92
|
if (options.schema) args.push("--json-schema", JSON.stringify(options.schema));
|
|
88
93
|
return { executable: this.executable(runtime), args, promptTransport: "stdin", input: prompt };
|
|
89
94
|
},
|