faberun 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/faberun/references/contract.md +10 -10
- package/src/contract/assert.mjs +5 -1
- package/src/contract/index.mjs +43 -7
- package/src/contract/runtime.mjs +4 -1
- package/src/contract/snapshot.mjs +25 -1
- package/src/contract/task-packet.mjs +5 -1
- package/src/engine/backoff.mjs +2 -1
- package/src/engine/capacity.mjs +74 -0
- package/src/engine/dispatch.mjs +12 -186
- package/src/engine/lifecycle.mjs +7 -2
- package/src/engine/phase-session.mjs +217 -0
- package/src/engine/process.mjs +26 -108
- package/src/engine/scheduler.mjs +15 -3
- package/src/engine/settle.mjs +11 -2
- package/src/engine/transcript.mjs +145 -0
- package/src/harnesses/claude/index.mjs +5 -0
- package/src/harnesses/dsh/runner.mjs +38 -3
- package/src/harnesses/exec-jsonl/index.mjs +0 -499
- package/src/harnesses/index.mjs +55 -8
- package/src/harnesses/protocol.mjs +22 -2
- package/src/harnesses/session-metrics.mjs +674 -0
- package/src/host/tool-policy-decisions.mjs +78 -21
- package/src/host/tool-policy-hook.mjs +12 -3
- package/src/plan/freeze.mjs +24 -6
- package/src/plan/pipeline.mjs +232 -67
- package/src/plan/sizing.mjs +0 -0
- package/src/plan/template.mjs +110 -8
- package/src/run/operations.mjs +1 -1
- package/src/run/usage.mjs +9 -3
- package/src/util.mjs +0 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "faberun",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.13.0",
|
|
4
4
|
"description": "Faberun is a development orchestration system that turns intent into verified software: harness- and model-agnostic, it keeps the intent, coordinates the work, verifies the result and decides what happens next.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -297,19 +297,19 @@ offered to a second node, so one session runs one turn.
|
|
|
297
297
|
|
|
298
298
|
## Gates
|
|
299
299
|
|
|
300
|
-
`gate: false` skips review
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
`
|
|
304
|
-
`
|
|
305
|
-
|
|
300
|
+
`gate: false` skips review; the node keeps `maxRevisions` (default 1) fresh
|
|
301
|
+
attempts after a red verification, and `{ enabled: false, maxRevisions: 0 }`
|
|
302
|
+
makes the first red one final. A gate object accepts `runtime` (the judge),
|
|
303
|
+
`review` (`none`/`advisory`/`blocking`, default `advisory`), `failOn`
|
|
304
|
+
(default `["critical"]`) and `maxRevisions`. `advisory` records the verdict
|
|
305
|
+
and still settles `done` on deterministic verification alone, never
|
|
306
|
+
re-dispatching; `blocking` re-dispatches within `maxRevisions` when findings
|
|
306
307
|
reach `failOn`. Validation requires `critical` whenever `major` is in
|
|
307
308
|
`failOn`, and `major` in `failOn` for a `blocking` gate: `["critical"]` alone
|
|
308
|
-
passes every major finding
|
|
309
|
+
passes every major finding.
|
|
309
310
|
|
|
310
|
-
The revision budget counts
|
|
311
|
-
crash-restart never consumes one
|
|
312
|
-
`revisions`). Deterministic `verification` commands run once by default
|
|
311
|
+
The revision budget counts rejections, not worker starts; a resume or a
|
|
312
|
+
crash-restart never consumes one. Deterministic `verification` commands run once by default
|
|
313
313
|
before any judge and the judge reviews the recorded results, never
|
|
314
314
|
re-running them (`repeat` opts into re-running a flaky check). A judge
|
|
315
315
|
output is `pass` only with empty `findings` and `maxSeverity: none`; for
|
package/src/contract/assert.mjs
CHANGED
|
@@ -46,7 +46,11 @@ export function assertObject(value, label) {
|
|
|
46
46
|
* @returns {asserts value is string}
|
|
47
47
|
*/
|
|
48
48
|
export function requireId(value, label) {
|
|
49
|
-
|
|
49
|
+
// Absent and malformed refuse with different messages: observed 2026-09-20,
|
|
50
|
+
// a missing id answered with the character-class message and sent the reader
|
|
51
|
+
// hunting for an illegal character that was not there.
|
|
52
|
+
if (typeof value !== "string" || !value) throw new TypeError(`${label} is required`);
|
|
53
|
+
if (!/^[A-Za-z0-9._-]+$/u.test(value)) {
|
|
50
54
|
throw new TypeError(`${label} must contain only letters, numbers, dot, underscore, or dash`);
|
|
51
55
|
}
|
|
52
56
|
if (value === "." || value === "..") throw new TypeError(`${label} must not be "." or ".."`);
|
package/src/contract/index.mjs
CHANGED
|
@@ -32,16 +32,25 @@ const WRITE_FILE_LINE_WARN_MARGIN = 100;
|
|
|
32
32
|
|
|
33
33
|
const CONTRACT_FIELDS = new Set([
|
|
34
34
|
"schemaVersion", "contractVersion", "id", "campaignId", "goal", "cwd", "sourceIdentity",
|
|
35
|
-
"maxParallel", "pollIntervalMs", "stallTimeoutSec", "timeoutSec",
|
|
35
|
+
"maxParallel", "pollIntervalMs", "stallTimeoutSec", "timeoutSec", "maxTurns", "phaseSessionReuse",
|
|
36
36
|
"runtimeDefaults", "runtimes", "nodes", "warnings", "finalVerification", "sharedVerification", "nodeAdvisory",
|
|
37
37
|
]);
|
|
38
38
|
const DEFAULTS_FIELDS = new Set(["worker", "judge"]);
|
|
39
39
|
const NODE_FIELDS = new Set([
|
|
40
40
|
"id", "type", "phase", "runtime", "dependsOn", "taskPacket", "taskPacketFile", "prompt", "promptFile",
|
|
41
|
-
"definitionOfDone", "gate", "timeoutSec",
|
|
41
|
+
"definitionOfDone", "gate", "timeoutSec", "maxTurns",
|
|
42
42
|
"requiredCapabilities", "packetHash", "sourceIdentity", "replayPolicy",
|
|
43
43
|
]);
|
|
44
44
|
const REPLAY_POLICIES = new Set(["safe", "reconcile", "never"]);
|
|
45
|
+
/**
|
|
46
|
+
* Provider requests one attempt may make before the controller ends it, when
|
|
47
|
+
* neither the contract nor the node says otherwise. measured 2026-09-20 over
|
|
48
|
+
* 200 completed claude worker turns: p90 83, p95 96, p99 122, max 339; the 23
|
|
49
|
+
* turns that never produced a result held 25% of all context spend, and six
|
|
50
|
+
* of them ran past 150 (180 to 600 requests). One completed turn would have
|
|
51
|
+
* been cut and retried once.
|
|
52
|
+
*/
|
|
53
|
+
export const DEFAULT_MAX_TURNS = 150;
|
|
45
54
|
const GATE_FIELDS = new Set(["enabled", "runtime", "review", "failOn", "maxRevisions", "requiredCapabilities", "skipWhen"]);
|
|
46
55
|
const GATE_REVIEWS = new Set(["none", "advisory", "blocking"]);
|
|
47
56
|
|
|
@@ -55,13 +64,13 @@ const GATE_REVIEWS = new Set(["none", "advisory", "blocking"]);
|
|
|
55
64
|
|
|
56
65
|
/** @typedef {{mode: "execution"|"discovery"|"autonomous", objective: string, instructions: string[], readFiles: string[], writeFiles?: string[], writeRoots?: string[], symbols: string[], scopeAcknowledged?: string[], decisions: string[], nonGoals: string[], verification: VerificationCommand[]}} TaskPacket */
|
|
57
66
|
|
|
58
|
-
/** @typedef {{harness: "claude"|"codex"|"agy"|"dsh"|"zcode"|"exec-jsonl"|"replay", model: string, reasoning?: string, sandbox?: "read-only"|"workspace-write"|"danger-full-access", permissionMode?: string, config?: Record<string, unknown>, printTimeout?: string, tools?: string[], executable?: string, args?: string[], versionArgs?: string[], maxArgvPromptBytes?: number, requiredCapabilities?: CapabilityRequirements, costRank?: number, fallback?: string, vendor: string, tier?: number|string, stallTimeoutSec?: number}} ValidatedRuntime */
|
|
67
|
+
/** @typedef {{harness: "claude"|"codex"|"agy"|"dsh"|"zcode"|"exec-jsonl"|"replay", model: string, reasoning?: string, sandbox?: "read-only"|"workspace-write"|"danger-full-access", permissionMode?: string, config?: Record<string, unknown>, printTimeout?: string, tools?: string[], executable?: string, args?: string[], versionArgs?: string[], maxArgvPromptBytes?: number, requiredCapabilities?: CapabilityRequirements, costRank?: number, fallback?: string, vendor: string, tier?: number|string, stallTimeoutSec?: number, maxConcurrent?: number}} ValidatedRuntime */
|
|
59
68
|
|
|
60
69
|
/** @typedef {{enabled: boolean, review?: ("none"|"advisory"|"blocking"), runtime?: string, failOn?: ("minor"|"major"|"critical")[], maxRevisions?: number, requiredCapabilities?: CapabilityRequirements, skipWhen?: {verificationGreen: true, maxChangedPaths: number}}} ValidatedGate */
|
|
61
70
|
|
|
62
|
-
/** @typedef {{id: string, type: string, phase: string, runtime?: string, dependsOn: string[], taskPacket: TaskPacket, taskPacketFile?: string, prompt: string, definitionOfDone: import("./definition-of-done.mjs").DefinitionOfDoneItem[], gate: ValidatedGate, timeoutSec?: number, requiredCapabilities: CapabilityRequirements, packetHash: string, sourceIdentity: SourceIdentity, replayPolicy: "safe"|"reconcile"|"never"}} ValidatedNode */
|
|
71
|
+
/** @typedef {{id: string, type: string, phase: string, runtime?: string, dependsOn: string[], taskPacket: TaskPacket, taskPacketFile?: string, prompt: string, definitionOfDone: import("./definition-of-done.mjs").DefinitionOfDoneItem[], gate: ValidatedGate, timeoutSec?: number, maxTurns?: number, requiredCapabilities: CapabilityRequirements, packetHash: string, sourceIdentity: SourceIdentity, replayPolicy: "safe"|"reconcile"|"never"}} ValidatedNode */
|
|
63
72
|
|
|
64
|
-
/** @typedef {{schemaVersion: number, contractVersion: string, id: string, campaignId: string, goal: string, cwd: string, sourceIdentity: SourceIdentity, runtimes: Record<string, ValidatedRuntime>, runtimeDefaults: {worker?: string, judge?: string}, nodes: ValidatedNode[], maxParallel: number, pollIntervalMs: number, stallTimeoutSec: number, timeoutSec: number, finalVerification?: VerificationCommand[], sharedVerification?: VerificationCommand[], nodeAdvisory?: NodeAdvisoryPolicy, warnings: string[]}} ValidatedContract */
|
|
73
|
+
/** @typedef {{schemaVersion: number, contractVersion: string, id: string, campaignId: string, goal: string, cwd: string, sourceIdentity: SourceIdentity, runtimes: Record<string, ValidatedRuntime>, runtimeDefaults: {worker?: string, judge?: string}, nodes: ValidatedNode[], maxParallel: number, pollIntervalMs: number, stallTimeoutSec: number, timeoutSec: number, maxTurns: number, phaseSessionReuse: boolean, finalVerification?: VerificationCommand[], sharedVerification?: VerificationCommand[], nodeAdvisory?: NodeAdvisoryPolicy, warnings: string[]}} ValidatedContract */
|
|
65
74
|
/** @typedef {{costUsd?: number, durationSec?: number}} NodeAdvisoryPolicy */
|
|
66
75
|
|
|
67
76
|
/** @typedef {"pending"|"running"|"done"|"no-op"|"blocked"|"failed"|"exhausted"|"stalled"|"canceled"} NodeStatus */
|
|
@@ -220,6 +229,9 @@ export function validateContract(raw, contractPath, options = {}) {
|
|
|
220
229
|
const timeoutSec = node.timeoutSec === undefined
|
|
221
230
|
? undefined
|
|
222
231
|
: positiveNumber(node.timeoutSec, `nodes[${index}].timeoutSec`);
|
|
232
|
+
const maxTurns = node.maxTurns === undefined
|
|
233
|
+
? undefined
|
|
234
|
+
: positiveInteger(node.maxTurns, `nodes[${index}].maxTurns`);
|
|
223
235
|
const replayPolicy = validateReplayPolicy(node.replayPolicy, `nodes[${index}]`);
|
|
224
236
|
return /** @type {ValidatedNode} */ ({
|
|
225
237
|
...node,
|
|
@@ -232,6 +244,7 @@ export function validateContract(raw, contractPath, options = {}) {
|
|
|
232
244
|
prompt,
|
|
233
245
|
gate,
|
|
234
246
|
timeoutSec,
|
|
247
|
+
maxTurns,
|
|
235
248
|
replayPolicy,
|
|
236
249
|
});
|
|
237
250
|
});
|
|
@@ -362,6 +375,14 @@ export function validateContract(raw, contractPath, options = {}) {
|
|
|
362
375
|
pollIntervalMs: positiveInteger(raw.pollIntervalMs ?? 1_000, "contract.pollIntervalMs"),
|
|
363
376
|
stallTimeoutSec: positiveNumber(raw.stallTimeoutSec ?? 300, "contract.stallTimeoutSec"),
|
|
364
377
|
timeoutSec: positiveNumber(raw.timeoutSec ?? 2_400, "contract.timeoutSec"),
|
|
378
|
+
maxTurns: positiveInteger(raw.maxTurns ?? DEFAULT_MAX_TURNS, "contract.maxTurns"),
|
|
379
|
+
// Opt-in: a phase sibling's provider session is rotated (fresh session,
|
|
380
|
+
// structured summaries carried) unless the contract asks to reuse it.
|
|
381
|
+
// measured 2026-09-20 over 21 runs with both kinds of turn: a turn opened
|
|
382
|
+
// on a sibling's session cost 1.87x the fresh one at the same request
|
|
383
|
+
// count, because it began with 200k tokens of context instead of 45k and
|
|
384
|
+
// re-read them on every request.
|
|
385
|
+
phaseSessionReuse: booleanField(raw.phaseSessionReuse, false, "contract.phaseSessionReuse"),
|
|
365
386
|
finalVerification: validateFinalVerification(raw.finalVerification, "contract.finalVerification"),
|
|
366
387
|
sharedVerification: validateSharedVerification(raw.sharedVerification, "contract.sharedVerification"),
|
|
367
388
|
nodeAdvisory: validateNodeAdvisory(raw.nodeAdvisory),
|
|
@@ -447,8 +468,16 @@ function validateGate(gate, runtimes, index, nodeId) {
|
|
|
447
468
|
assertObject(gate, `nodes[${index}].gate`);
|
|
448
469
|
rejectUnknown(gate, GATE_FIELDS, `nodes[${index}].gate`);
|
|
449
470
|
if (gate.enabled === false) {
|
|
450
|
-
|
|
451
|
-
|
|
471
|
+
// A disabled gate reviews nothing, but the node keeps its revision budget
|
|
472
|
+
// for a red verification (default 1, as with a gate); that budget is the
|
|
473
|
+
// one field the disabled shape may carry.
|
|
474
|
+
if (Object.keys(gate).some((key) => key !== "enabled" && key !== "maxRevisions")) {
|
|
475
|
+
throw new TypeError(`nodes[${index}].gate disabled shape only allows enabled and maxRevisions`);
|
|
476
|
+
}
|
|
477
|
+
return {
|
|
478
|
+
enabled: false,
|
|
479
|
+
...(gate.maxRevisions === undefined ? {} : { maxRevisions: nonNegativeInteger(gate.maxRevisions, `nodes[${index}].gate.maxRevisions`) }),
|
|
480
|
+
};
|
|
452
481
|
}
|
|
453
482
|
if (gate.enabled !== undefined && gate.enabled !== true) {
|
|
454
483
|
throw new TypeError(`nodes[${index}].gate.enabled must be true or false`);
|
|
@@ -686,3 +715,10 @@ function validateMaxParallel(value) {
|
|
|
686
715
|
// beyond being a sane positive integer.
|
|
687
716
|
return positiveInteger(value, "contract.maxParallel");
|
|
688
717
|
}
|
|
718
|
+
|
|
719
|
+
/** @param {unknown} value @param {boolean} fallback @param {string} label @returns {boolean} */
|
|
720
|
+
function booleanField(value, fallback, label) {
|
|
721
|
+
if (value === undefined) return fallback;
|
|
722
|
+
if (typeof value !== "boolean") throw new TypeError(`${label} must be a boolean`);
|
|
723
|
+
return value;
|
|
724
|
+
}
|
package/src/contract/runtime.mjs
CHANGED
|
@@ -22,7 +22,7 @@ import { stableJson } from "../util.mjs";
|
|
|
22
22
|
const RUNTIME_FIELDS = new Set([
|
|
23
23
|
"harness", "model", "reasoning", "sandbox", "permissionMode", "config", "printTimeout", "tools",
|
|
24
24
|
"executable", "args", "versionArgs", "maxArgvPromptBytes", "requiredCapabilities", "costRank",
|
|
25
|
-
"fallback", "vendor", "tier", "pricing", "stallTimeoutSec",
|
|
25
|
+
"fallback", "vendor", "tier", "pricing", "stallTimeoutSec", "maxConcurrent",
|
|
26
26
|
]);
|
|
27
27
|
const RUNTIME_HARNESSES = new Set(["claude", "codex", "agy", "dsh", "zcode", "exec-jsonl", "replay"]);
|
|
28
28
|
|
|
@@ -98,6 +98,9 @@ function validateRuntimeValues(runtime, label, executableRequired) {
|
|
|
98
98
|
throw new TypeError(`${label}.sandbox is invalid`);
|
|
99
99
|
}
|
|
100
100
|
if (runtime.permissionMode !== undefined) requireString(runtime.permissionMode, `${label}.permissionMode`);
|
|
101
|
+
// How many attempts this runtime may run at once, below the run's
|
|
102
|
+
// maxParallel; absent leaves only maxParallel to bound it.
|
|
103
|
+
if (runtime.maxConcurrent !== undefined) positiveInteger(runtime.maxConcurrent, `${label}.maxConcurrent`);
|
|
101
104
|
if (runtime.config !== undefined && (!runtime.config || typeof runtime.config !== "object" || Array.isArray(runtime.config))) {
|
|
102
105
|
throw new TypeError(`${label}.config must be an object`);
|
|
103
106
|
}
|
|
@@ -246,7 +246,7 @@ function validateInvocations(value, label) {
|
|
|
246
246
|
"promptPath", "stdoutPath", "stderrPath", "startedAt", "updatedAt", "closedAt", "deadlineAt",
|
|
247
247
|
"exitCode", "signal", "status", "executable", "usage", "usageEstimated", "costUsd", "costProvenance", "snapshotPath", "revision", "cycle",
|
|
248
248
|
"runId", "campaignId", "planPhase", "role", "runtimeFingerprint", "model", "reasoning", "sandbox", "continuationId", "continuationMode",
|
|
249
|
-
"nodeId", "attempt", "workspace", "worktreeBranch", "worktreeBaseSha",
|
|
249
|
+
"nodeId", "attempt", "workspace", "worktreeBranch", "worktreeBaseSha", "session",
|
|
250
250
|
]);
|
|
251
251
|
rejectUnknown(invocation, allowed, `${label}[${index}]`);
|
|
252
252
|
requireString(invocation.id, `${label}[${index}].id`);
|
|
@@ -272,6 +272,7 @@ function validateInvocations(value, label) {
|
|
|
272
272
|
if (!['fresh', 'reuse', 'rotate'].includes(/** @type {string} */ (invocation.continuationMode))) {
|
|
273
273
|
throw new TypeError(`${label}[${index}].continuationMode is invalid`);
|
|
274
274
|
}
|
|
275
|
+
if (invocation.session !== undefined && invocation.session !== null) validateSessionLedger(invocation.session, `${label}[${index}].session`);
|
|
275
276
|
if (invocation.revision !== undefined) nonNegativeInteger(invocation.revision, `${label}[${index}].revision`);
|
|
276
277
|
if (invocation.cycle !== undefined) nonNegativeInteger(invocation.cycle, `${label}[${index}].cycle`);
|
|
277
278
|
for (const key of ["promptPath", "stdoutPath", "stderrPath", "executable"]) {
|
|
@@ -643,3 +644,26 @@ function validateScopeBoundarySnapshot(value, label) {
|
|
|
643
644
|
}
|
|
644
645
|
}
|
|
645
646
|
}
|
|
647
|
+
|
|
648
|
+
const SESSION_LEDGER_FIELDS = new Set(["turns", "toolCalls", "requests", "contextFirst", "contextMax", "contextLast", "contextSum", "completed"]);
|
|
649
|
+
|
|
650
|
+
/**
|
|
651
|
+
* The per-request ledger an invocation carries (`session-metrics.mjs`'s
|
|
652
|
+
* SessionLedger): turns and tool calls are non-negative integers; requests
|
|
653
|
+
* and the context fields are non-negative integers, or null when the stream
|
|
654
|
+
* carried no usage per request (codex reports per turn, zcode emits one
|
|
655
|
+
* document), which is unknown, not zero.
|
|
656
|
+
*
|
|
657
|
+
* @param {unknown} value
|
|
658
|
+
* @param {string} label
|
|
659
|
+
*/
|
|
660
|
+
function validateSessionLedger(value, label) {
|
|
661
|
+
assertObject(value, label);
|
|
662
|
+
const record = /** @type {Record<string, unknown>} */ (value);
|
|
663
|
+
rejectUnknown(record, SESSION_LEDGER_FIELDS, label);
|
|
664
|
+
for (const key of ["turns", "toolCalls"]) nonNegativeInteger(record[key], `${label}.${key}`);
|
|
665
|
+
for (const key of ["requests", "contextFirst", "contextMax", "contextLast", "contextSum"]) {
|
|
666
|
+
if (record[key] !== null) nonNegativeInteger(record[key], `${label}.${key}`);
|
|
667
|
+
}
|
|
668
|
+
if (typeof record.completed !== "boolean") throw new TypeError(`${label}.completed must be a boolean`);
|
|
669
|
+
}
|
|
@@ -25,7 +25,11 @@ const PROMPT_MAX_BYTES = 64 * 1024;
|
|
|
25
25
|
* process-free command, so a sandbox that cannot signal processes never hangs
|
|
26
26
|
* on the node's own verification.
|
|
27
27
|
*/
|
|
28
|
-
|
|
28
|
+
// measured 2026-09-20 over 228 stored claude worker turns: tool results are
|
|
29
|
+
// 73% of what enters the context after the packet (Read 38%, Bash 32%), edits
|
|
30
|
+
// and whole-file writes 24%, and each byte is re-read by a median of 49 later
|
|
31
|
+
// requests of the same turn. Hence the last sentence.
|
|
32
|
+
const VERIFICATION_PARAGRAPH = "The controller runs every command below after you report; its recorded results are the proof of this node. Running a command yourself is optional and only for one that finishes in seconds and spawns no long-lived process. Keep output bounded (pipe through `| tail -n 200`). Never wait on a background job, never run the whole test suite, and never run tests that start and terminate other processes. Prefer a targeted edit over rewriting a whole file, and read with an offset and limit rather than whole files: every byte you read or write is re-read by every later request of this turn.";
|
|
29
33
|
|
|
30
34
|
/**
|
|
31
35
|
* The `## Required output` schema every mode states: literally every key
|
package/src/engine/backoff.mjs
CHANGED
|
@@ -321,7 +321,8 @@ export function networkTransition(contract, node, state, role, envelope, exitCod
|
|
|
321
321
|
* @returns {boolean}
|
|
322
322
|
*/
|
|
323
323
|
export function isRepairable(node, state) {
|
|
324
|
-
|
|
324
|
+
// The budget is the node's, gate or not: see settle.mjs applyRejection.
|
|
325
|
+
return (state.revisions ?? 0) < (node.gate.maxRevisions ?? 1);
|
|
325
326
|
}
|
|
326
327
|
|
|
327
328
|
/**
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-runtime dispatch capacity: how many attempts one runtime may run at
|
|
3
|
+
* once (`maxConcurrent`; absent means only `maxParallel` bounds it) and which
|
|
4
|
+
* runtimes are on a quota hold -- a node of theirs is waiting out a provider
|
|
5
|
+
* exhaustion backoff on that same runtime, so dispatching a sibling to it
|
|
6
|
+
* would spend the next quota window on a refusal the run already knows
|
|
7
|
+
* about. Separate from the scheduler because it is pure over the running set
|
|
8
|
+
* and the persisted snapshots, and from failover.mjs because that decides one
|
|
9
|
+
* node's route while this decides whether the tick may start another.
|
|
10
|
+
*
|
|
11
|
+
* measured 2026-09-20: `maxParallel` was the one concurrency knob, global to
|
|
12
|
+
* the run; nothing reduced dispatch to a provider that had just refused a
|
|
13
|
+
* sibling on quota, and 14 of 58 stored contracts ran with maxParallel 2.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
|
|
17
|
+
/** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Live attempts per runtime id, from the jobs a tick is holding.
|
|
21
|
+
*
|
|
22
|
+
* @param {Iterable<{runtime: {id: string|null}}>} running
|
|
23
|
+
* @returns {Map<string, number>}
|
|
24
|
+
*/
|
|
25
|
+
export function runningPerRuntime(running) {
|
|
26
|
+
/** @type {Map<string, number>} */
|
|
27
|
+
const counts = new Map();
|
|
28
|
+
for (const job of running) {
|
|
29
|
+
const id = job.runtime.id;
|
|
30
|
+
if (typeof id !== "string") continue;
|
|
31
|
+
counts.set(id, (counts.get(id) ?? 0) + 1);
|
|
32
|
+
}
|
|
33
|
+
return counts;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Runtimes some node is waiting to use again after a provider exhaustion:
|
|
38
|
+
* the node's current routing override still has a `backoffUntil` in the
|
|
39
|
+
* future, it points back at the runtime that was exhausted (a reset wait, not
|
|
40
|
+
* a hop to another runtime), and the exhaustion was the last routing event.
|
|
41
|
+
* A network wait or an ordinary provider failure holds nothing.
|
|
42
|
+
*
|
|
43
|
+
* @param {Iterable<NodeSnapshot>} states
|
|
44
|
+
* @param {number} now epoch milliseconds
|
|
45
|
+
* @returns {Set<string>}
|
|
46
|
+
*/
|
|
47
|
+
export function quotaHeldRuntimes(states, now) {
|
|
48
|
+
/** @type {Set<string>} */
|
|
49
|
+
const held = new Set();
|
|
50
|
+
for (const state of states) {
|
|
51
|
+
const override = state.routing?.currentOverride;
|
|
52
|
+
const last = state.routing?.history?.at(-1);
|
|
53
|
+
if (!override?.backoffUntil || Date.parse(override.backoffUntil) <= now) continue;
|
|
54
|
+
if (!override.runtime || !last || last.runtime !== override.runtime) continue;
|
|
55
|
+
if (last.status !== "exhausted") continue;
|
|
56
|
+
held.add(override.runtime);
|
|
57
|
+
}
|
|
58
|
+
return held;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Whether one more attempt may start on `runtimeId` right now.
|
|
63
|
+
*
|
|
64
|
+
* @param {string} runtimeId
|
|
65
|
+
* @param {ValidatedContract} contract
|
|
66
|
+
* @param {Map<string, number>} counts live attempts per runtime, `runningPerRuntime`'s shape
|
|
67
|
+
* @param {Set<string>} held `quotaHeldRuntimes`'s result
|
|
68
|
+
* @returns {boolean}
|
|
69
|
+
*/
|
|
70
|
+
export function runtimeHasCapacity(runtimeId, contract, counts, held) {
|
|
71
|
+
if (held.has(runtimeId)) return false;
|
|
72
|
+
const limit = contract.runtimes[runtimeId]?.maxConcurrent;
|
|
73
|
+
return typeof limit !== "number" || (counts.get(runtimeId) ?? 0) < limit;
|
|
74
|
+
}
|
package/src/engine/dispatch.mjs
CHANGED
|
@@ -24,21 +24,23 @@ import {
|
|
|
24
24
|
import { attemptWorkspace, createAttemptWorktree, sealAttempt } from "../repo/worktree.mjs";
|
|
25
25
|
import { attemptWorktreePath } from "../run/paths.mjs";
|
|
26
26
|
import { basename, dirname, join } from "node:path";
|
|
27
|
-
import {
|
|
27
|
+
import { errorCode, errorMessage } from "../util.mjs";
|
|
28
28
|
import { captureWorkspaceScope, captureWorkspaceSnapshot } from "../repo/workspace.mjs";
|
|
29
|
-
import { createHash } from "node:crypto";
|
|
30
29
|
import { deterministicGate, judgeReaskReason, judgeRequired, judgeSkippedByScope } from "./judge-gate.mjs";
|
|
31
30
|
import { emptyScope, persistedScopeBoundary, workerScope } from "./scope.mjs";
|
|
32
31
|
import { hasOperationIntent, hasOperationSettlement, operationNeedsRecovery, operationNextState, persistInvocationIntent, providerReceipts, settleInvocation } from "../run/operations.mjs";
|
|
33
32
|
import { invocationCost, invocationUsage } from "../run/usage.mjs";
|
|
34
|
-
import { logPaths,
|
|
33
|
+
import { logPaths, startProcess } from "./process.mjs";
|
|
34
|
+
import { readBoundedTail } from "./transcript.mjs";
|
|
35
35
|
import { mkdirSync, statSync } from "node:fs";
|
|
36
|
-
import { READ_LINE_LIMIT, normalizeProviderResult
|
|
37
|
-
import {
|
|
36
|
+
import { READ_BYTE_LIMIT, READ_LINE_LIMIT, normalizeProviderResult } from "../harnesses/index.mjs";
|
|
37
|
+
import { writeJsonAtomic } from "../run/store.mjs";
|
|
38
38
|
import { judgeReaskInstruction, reviewMode } from "../contract/review-modes.mjs";
|
|
39
39
|
import { routeRuntimeForState, runtimeSnapshot } from "./failover.mjs";
|
|
40
|
+
import { fingerprintRuntime, forceFreshSession, phaseInvocationPlan } from "./phase-session.mjs";
|
|
41
|
+
|
|
42
|
+
/** @typedef {import("./phase-session.mjs").SessionPolicy} SessionPolicy */
|
|
40
43
|
import { transition, writeNode } from "./state.mjs";
|
|
41
|
-
import { validateNodeSnapshot } from "../contract/snapshot.mjs";
|
|
42
44
|
|
|
43
45
|
/**
|
|
44
46
|
* What a judge round decided, for the caller to act on.
|
|
@@ -63,156 +65,8 @@ import { validateNodeSnapshot } from "../contract/snapshot.mjs";
|
|
|
63
65
|
/** @typedef {import("../contract/index.mjs").VerificationState} VerificationState */
|
|
64
66
|
/** @typedef {import("../contract/index.mjs").WorkspaceScopeBoundary} WorkspaceScopeBoundary */
|
|
65
67
|
|
|
66
|
-
/** @typedef {{forceFresh?: boolean}} SessionPolicy */
|
|
67
68
|
|
|
68
|
-
/**
|
|
69
|
-
* Resolve the session policy one dispatch runs under, then consume the copy the
|
|
70
|
-
* node persisted. The explicit argument comes from a caller that is dispatching
|
|
71
|
-
* on the spot; `state.sessionPolicy` is the copy a rejection decision left when
|
|
72
|
-
* it handed the node back to the scheduler, whose own `startWorker` call passes
|
|
73
|
-
* nothing at all.
|
|
74
|
-
*
|
|
75
|
-
* Persisting is the whole point: `phaseInvocationPlan` rediscovers a compatible
|
|
76
|
-
* continuation from the persisted ledger, so nulling a local continuation id at
|
|
77
|
-
* the call site would let the scheduler's later dispatch find it again. Clearing
|
|
78
|
-
* the stored policy here makes it one-shot — it governs exactly the dispatch it
|
|
79
|
-
* was recorded for, and the next unrelated attempt reuses normally.
|
|
80
|
-
*
|
|
81
|
-
* @param {{sessionPolicy?: SessionPolicy|null}} state
|
|
82
|
-
* @param {SessionPolicy} [explicit]
|
|
83
|
-
* @returns {SessionPolicy}
|
|
84
|
-
*/
|
|
85
|
-
export function forceFreshSession(state, explicit = {}) {
|
|
86
|
-
const persisted = /** @type {SessionPolicy|undefined} */ (state?.sessionPolicy ?? undefined);
|
|
87
|
-
const policy = { ...(persisted ?? {}), ...explicit };
|
|
88
|
-
if (state && state.sessionPolicy !== undefined && state.sessionPolicy !== null) state.sessionPolicy = null;
|
|
89
|
-
return policy;
|
|
90
|
-
}
|
|
91
69
|
|
|
92
|
-
/**
|
|
93
|
-
* Select the only continuation that is allowed for this plan phase and role.
|
|
94
|
-
* The search is intentionally limited to persisted node snapshots in this run.
|
|
95
|
-
*
|
|
96
|
-
* `policy.forceFresh` is the explicit session policy a rejection decision
|
|
97
|
-
* carries: it short-circuits the search before it can rediscover a compatible
|
|
98
|
-
* continuation, so a retry after a gate rejection starts a fresh provider
|
|
99
|
-
* session instead of re-reading the failed transcript. Nulling a local id at
|
|
100
|
-
* the call site is not enough, because this function rediscovers the prior
|
|
101
|
-
* continuation from the persisted ledger.
|
|
102
|
-
*
|
|
103
|
-
* @param {ValidatedContract} contract
|
|
104
|
-
* @param {ValidatedNode} node
|
|
105
|
-
* @param {NodeSnapshot} state
|
|
106
|
-
* @param {string} runDir
|
|
107
|
-
* @param {"worker"|"judge"} role
|
|
108
|
-
* @param {string} prompt
|
|
109
|
-
* @param {SessionPolicy} [policy]
|
|
110
|
-
* @returns {{prompt: string, continuationId: string|null, mode: "fresh"|"reuse"|"rotate"}}
|
|
111
|
-
*/
|
|
112
|
-
function phaseInvocationPlan(contract, node, state, runDir, role, prompt, policy = {}) {
|
|
113
|
-
if (policy.forceFresh === true) {
|
|
114
|
-
return { prompt, continuationId: null, mode: "fresh" };
|
|
115
|
-
}
|
|
116
|
-
const runId = basename(runDir);
|
|
117
|
-
const session = phaseSessionCandidates(contract, node, state, runDir, role).at(-1);
|
|
118
|
-
const runtime = routeRuntimeForState(contract, node, state, role);
|
|
119
|
-
const identityMatches = session && session.invocation.runId === runId
|
|
120
|
-
&& session.invocation.campaignId === contract.campaignId
|
|
121
|
-
&& session.invocation.planPhase === node.phase
|
|
122
|
-
&& session.invocation.role === role
|
|
123
|
-
&& session.invocation.harness === runtime.harness
|
|
124
|
-
&& session.invocation.runtimeId === runtime.id
|
|
125
|
-
&& session.invocation.runtimeFingerprint === fingerprintRuntime(runtime)
|
|
126
|
-
&& session.invocation.model === runtime.model
|
|
127
|
-
&& session.invocation.reasoning === (runtime.reasoning ?? null)
|
|
128
|
-
&& session.invocation.sandbox === (runtime.sandbox ?? null);
|
|
129
|
-
const canContinue = runtime.capabilities.continuation === true;
|
|
130
|
-
if (identityMatches && canContinue) {
|
|
131
|
-
return { prompt, continuationId: session.invocation.continuationId ?? null, mode: "reuse" };
|
|
132
|
-
}
|
|
133
|
-
// A harness that cannot continue at all, or a session picked up from a
|
|
134
|
-
// different phase-sibling node whose identity does not match this one, has
|
|
135
|
-
// no native continuity: the fresh attempt carries the prior nodes'
|
|
136
|
-
// structured summaries forward instead of starting blind.
|
|
137
|
-
if (session && (!canContinue || session.nodeId !== node.id)) {
|
|
138
|
-
return {
|
|
139
|
-
prompt: phaseHandoffPrompt(contract, node, state, runDir, role),
|
|
140
|
-
continuationId: null,
|
|
141
|
-
mode: "rotate",
|
|
142
|
-
};
|
|
143
|
-
}
|
|
144
|
-
// A capable harness continuing its own node whose identity merely drifted
|
|
145
|
-
// (the run directory moved, or a runtime edge) still gets the caller's own
|
|
146
|
-
// prompt — already carrying the node's bounded "Previous attempt" section —
|
|
147
|
-
// in a fresh session, never a synthesized handoff.
|
|
148
|
-
return { prompt, continuationId: null, mode: session ? "rotate" : "fresh" };
|
|
149
|
-
}
|
|
150
|
-
/**
|
|
151
|
-
* Continuation ids a live invocation is already driving, anywhere in the run.
|
|
152
|
-
*
|
|
153
|
-
* This is what makes concurrent nodes of one phase safe, and it is read from
|
|
154
|
-
* the persisted ledger rather than from an in-memory registry so a controller
|
|
155
|
-
* that took over a run inherits the claims instead of racing them.
|
|
156
|
-
*
|
|
157
|
-
* @param {ValidatedContract} contract
|
|
158
|
-
* @param {NodeSnapshot} currentState
|
|
159
|
-
* @param {string} runDir
|
|
160
|
-
* @returns {Set<string>}
|
|
161
|
-
*/
|
|
162
|
-
function claimedContinuations(contract, currentState, runDir) {
|
|
163
|
-
/** @type {Set<string>} */
|
|
164
|
-
const claimed = new Set();
|
|
165
|
-
for (const candidate of contract.nodes) {
|
|
166
|
-
let state = candidate.id === currentState.id ? currentState : null;
|
|
167
|
-
if (!state) {
|
|
168
|
-
try { state = validateNodeSnapshot(readJson(join(runDir, "nodes", `${candidate.id}.json`)), candidate); } catch { continue; }
|
|
169
|
-
}
|
|
170
|
-
for (const invocation of state.invocations ?? []) {
|
|
171
|
-
if (invocation.status === "active" && invocation.continuationId) claimed.add(invocation.continuationId);
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
return claimed;
|
|
175
|
-
}
|
|
176
|
-
/**
|
|
177
|
-
* @param {ValidatedContract} contract
|
|
178
|
-
* @param {ValidatedNode} node
|
|
179
|
-
* @param {NodeSnapshot} currentState
|
|
180
|
-
* @param {string} runDir
|
|
181
|
-
* @param {"worker"|"judge"} role
|
|
182
|
-
* @returns {{nodeId: string, invocation: Invocation}[]}
|
|
183
|
-
*/
|
|
184
|
-
function phaseSessionCandidates(contract, node, currentState, runDir, role) {
|
|
185
|
-
/** @type {{nodeId: string, invocation: Invocation}[]} */
|
|
186
|
-
const candidates = [];
|
|
187
|
-
const claimed = claimedContinuations(contract, currentState, runDir);
|
|
188
|
-
for (const candidate of contract.nodes) {
|
|
189
|
-
if (candidate.phase !== node.phase) continue;
|
|
190
|
-
let state = candidate.id === currentState.id ? currentState : null;
|
|
191
|
-
if (!state) {
|
|
192
|
-
try { state = validateNodeSnapshot(readJson(join(runDir, "nodes", `${candidate.id}.json`)), candidate); } catch { continue; }
|
|
193
|
-
}
|
|
194
|
-
for (const invocation of state.invocations ?? []) {
|
|
195
|
-
if (invocation.role !== role || invocation.planPhase !== node.phase || !invocation.continuationId) continue;
|
|
196
|
-
if (invocation.nodeId !== candidate.id || invocation.attempt !== state.attempt || invocation.workspace !== state.worktree?.path) continue;
|
|
197
|
-
// One provider session, one live turn. With `maxParallel` above one,
|
|
198
|
-
// two nodes of a phase can be dispatched in the same tick, and without
|
|
199
|
-
// this both would hand the same continuation id to their own provider
|
|
200
|
-
// process. The claim is read from the persisted ledger, which the
|
|
201
|
-
// in-tick dispatch already wrote for the node that went first.
|
|
202
|
-
if (claimed.has(invocation.continuationId)) continue;
|
|
203
|
-
candidates.push({ nodeId: candidate.id, invocation });
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
return candidates.sort((left, right) => {
|
|
207
|
-
const leftStarted = Date.parse(left.invocation.startedAt);
|
|
208
|
-
const rightStarted = Date.parse(right.invocation.startedAt);
|
|
209
|
-
if (leftStarted !== rightStarted) return leftStarted - rightStarted;
|
|
210
|
-
const leftUpdated = Date.parse(left.invocation.updatedAt);
|
|
211
|
-
const rightUpdated = Date.parse(right.invocation.updatedAt);
|
|
212
|
-
if (leftUpdated !== rightUpdated) return leftUpdated - rightUpdated;
|
|
213
|
-
return left.invocation.id.localeCompare(right.invocation.id);
|
|
214
|
-
});
|
|
215
|
-
}
|
|
216
70
|
/** @param {Invocation} invocation @param {ValidatedContract} contract @param {ValidatedNode} node @param {RuntimeSnapshot} runtime @param {NodeSnapshot} state @param {string} runDir @param {"worker"|"judge"} role @param {"fresh"|"reuse"|"rotate"} mode @param {string|null} continuationId */
|
|
217
71
|
function stampInvocation(invocation, contract, node, runtime, state, runDir, role, mode, continuationId) {
|
|
218
72
|
invocation.runId = basename(runDir);
|
|
@@ -235,38 +89,6 @@ function stampInvocation(invocation, contract, node, runtime, state, runDir, rol
|
|
|
235
89
|
// generation across a controller crash.
|
|
236
90
|
/** @type {{cycle?: number}} */ (invocation).cycle = state.routing?.tierExhaustionCycle ?? 0;
|
|
237
91
|
}
|
|
238
|
-
/** @param {RuntimeSnapshot} runtime @returns {string} */
|
|
239
|
-
function fingerprintRuntime(runtime) {
|
|
240
|
-
const executable = providerCommand(runtime, "").executable;
|
|
241
|
-
return createHash("sha256").update(stableJson({ runtime, executable })).digest("hex");
|
|
242
|
-
}
|
|
243
|
-
/** @param {ValidatedContract} contract @param {ValidatedNode} node @param {NodeSnapshot} state @param {string} runDir @param {"worker"|"judge"} role @returns {string} */
|
|
244
|
-
function phaseHandoffPrompt(contract, node, state, runDir, role) {
|
|
245
|
-
const summaries = phaseSessionCandidates(contract, node, state, runDir, role)
|
|
246
|
-
.map(({ nodeId }) => {
|
|
247
|
-
const candidate = contract.nodes.find((item) => item.id === nodeId);
|
|
248
|
-
let snapshot = null;
|
|
249
|
-
try { snapshot = readJson(join(runDir, "nodes", `${nodeId}.json`)); } catch {
|
|
250
|
-
// ENOENT or unreadable snapshot: this prior node contributes no summary.
|
|
251
|
-
}
|
|
252
|
-
const result = snapshot?.result;
|
|
253
|
-
const record = result && typeof result === "object" && !Array.isArray(result)
|
|
254
|
-
? /** @type {Record<string, unknown>} */ (result)
|
|
255
|
-
: null;
|
|
256
|
-
const summary = typeof record?.summary === "string" ? record.summary : null;
|
|
257
|
-
return summary && candidate ? `${candidate.id}: ${boundedUtf8(summary, 1024)}` : null;
|
|
258
|
-
})
|
|
259
|
-
.filter(Boolean)
|
|
260
|
-
.slice(-8);
|
|
261
|
-
const handoff = [
|
|
262
|
-
`Continue phase ${node.phase} as the ${role} agent in a fresh provider session.`,
|
|
263
|
-
"Prior structured node summaries:",
|
|
264
|
-
summaries.length ? summaries.map((summary) => `- ${summary}`).join("\n") : "- (none)",
|
|
265
|
-
"Current closed task packet:",
|
|
266
|
-
boundedUtf8(node.prompt, 48 * 1024),
|
|
267
|
-
].join("\n\n");
|
|
268
|
-
return boundedUtf8(handoff, 60 * 1024);
|
|
269
|
-
}
|
|
270
92
|
/**
|
|
271
93
|
* The declared weight of a node's readFiles at dispatch time: the sum of the
|
|
272
94
|
* byte sizes of the files that exist in the attempt workspace. This is the
|
|
@@ -311,6 +133,7 @@ function workerToolPolicy(runtime, node, workspace) {
|
|
|
311
133
|
writeFiles: node.taskPacket.writeFiles ?? [],
|
|
312
134
|
writeRoots: node.taskPacket.writeRoots ?? [],
|
|
313
135
|
maxReadLines: READ_LINE_LIMIT,
|
|
136
|
+
maxReadBytes: READ_BYTE_LIMIT,
|
|
314
137
|
};
|
|
315
138
|
}
|
|
316
139
|
/**
|
|
@@ -332,6 +155,9 @@ function invocationCommandOptions(contract, node, state, runtime, phasePlan, run
|
|
|
332
155
|
return {
|
|
333
156
|
...extra,
|
|
334
157
|
continuationId: runtime.capabilities.continuation === true ? phasePlan.continuationId : null,
|
|
158
|
+
// The attempt's request ceiling. An adapter that can enforce it natively
|
|
159
|
+
// takes it as a flag; the monitor enforces it for every streaming harness.
|
|
160
|
+
maxTurns: node.maxTurns ?? contract.maxTurns,
|
|
335
161
|
};
|
|
336
162
|
}
|
|
337
163
|
/**
|
package/src/engine/lifecycle.mjs
CHANGED
|
@@ -142,8 +142,13 @@ export function terminalErrorCode(state) {
|
|
|
142
142
|
return typeof error.code === "string" && error.code ? error.code : null;
|
|
143
143
|
}
|
|
144
144
|
|
|
145
|
-
/**
|
|
146
|
-
|
|
145
|
+
/**
|
|
146
|
+
* Error codes that earn exactly one automatic retry before parking.
|
|
147
|
+
* `turn_limit` is here and not among the timeout codes below: a turn the CLI
|
|
148
|
+
* stopped itself at `--max-turns` exits cleanly with no seal yet, and the
|
|
149
|
+
* next dispatch seals its worktree as it does for any previous attempt.
|
|
150
|
+
*/
|
|
151
|
+
export const AUTO_RETRY_CODES = new Set(["judge_unavailable", "provider_error", "stall_timeout", "wall_clock_timeout", "turn_limit"]);
|
|
147
152
|
|
|
148
153
|
/**
|
|
149
154
|
* Timeout codes earn their automatic retry only when phase 5b sealed work
|