faberun 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +193 -31
- package/package.json +1 -1
- package/src/campaign/chain.mjs +19 -0
- package/src/campaign/journal.mjs +48 -0
- package/src/cli/brand.mjs +1 -0
- package/src/cli/campaign.mjs +15 -2
- package/src/cli/plan.mjs +142 -0
- package/src/cli.mjs +19 -3
- package/src/contract/final-verification.mjs +41 -10
- package/src/contract/task-packet.mjs +1 -1
- package/src/contract/worker-result.mjs +40 -8
- package/src/engine/lifecycle.mjs +11 -0
- package/src/engine/result-file.mjs +18 -4
- package/src/engine/scheduler.mjs +199 -26
- package/src/engine/settle.mjs +1 -1
- package/src/engine/verify.mjs +57 -4
- package/src/harnesses/protocol.mjs +91 -11
- package/src/plan/freeze.mjs +7 -3
- package/src/plan/pipeline.mjs +371 -0
- package/src/plan/template.mjs +278 -0
- package/src/repo/integrate.mjs +6 -1
- package/src/report/render.mjs +58 -58
- package/src/seat/allowance.mjs +177 -0
- package/src/web/index.html +3 -3
- package/src/web/server.mjs +2 -1
package/src/engine/verify.mjs
CHANGED
|
@@ -11,8 +11,10 @@
|
|
|
11
11
|
import { attemptWorkspace } from "../repo/worktree.mjs";
|
|
12
12
|
import { boundedUtf8, errorMessage } from "../util.mjs";
|
|
13
13
|
import { compactVerification } from "../contract/verification.mjs";
|
|
14
|
-
import { finalVerificationCommands, sharedVerificationCommands } from "../contract/final-verification.mjs";
|
|
14
|
+
import { finalVerificationCommands, phaseTerminalNode, sharedVerificationCommands } from "../contract/final-verification.mjs";
|
|
15
15
|
import { join } from "node:path";
|
|
16
|
+
import { listNodeSnapshots, readNodeSnapshot } from "../run/node-store.mjs";
|
|
17
|
+
import { SETTLED } from "./prompts.mjs";
|
|
16
18
|
import { terminateInvocation } from "./process.mjs";
|
|
17
19
|
import { writeNode } from "./state.mjs";
|
|
18
20
|
import { runVerification } from "./run-command.mjs";
|
|
@@ -28,6 +30,34 @@ import { runVerification } from "./run-command.mjs";
|
|
|
28
30
|
/** @typedef {import("../contract/index.mjs").VerificationState} VerificationState */
|
|
29
31
|
/** @typedef {{index: number, total: number, argv: string}} VerificationProgress */
|
|
30
32
|
|
|
33
|
+
/**
|
|
34
|
+
* The sibling phase-terminal node ids already settled, read straight off
|
|
35
|
+
* disk: every node snapshot is persisted from the run's first tick (see
|
|
36
|
+
* `runContract`), so a node still `pending` reads back honestly, not as
|
|
37
|
+
* missing. `finalVerificationCommands` uses this to decide, at the moment a
|
|
38
|
+
* phase-terminal node's own verification runs, whether it is the one that
|
|
39
|
+
* closes the phase -- recomputed fresh on every call, so a retried node sees
|
|
40
|
+
* its siblings' current status each time, not a decision frozen from an
|
|
41
|
+
* earlier attempt.
|
|
42
|
+
*
|
|
43
|
+
* @param {string} runDir
|
|
44
|
+
* @param {ValidatedContract} contract
|
|
45
|
+
* @param {ValidatedNode} node
|
|
46
|
+
* @returns {Set<string>}
|
|
47
|
+
*/
|
|
48
|
+
function settledSiblingIds(runDir, contract, node) {
|
|
49
|
+
const ids = new Set();
|
|
50
|
+
for (const name of listNodeSnapshots(runDir)) {
|
|
51
|
+
const id = name.slice(0, -".json".length);
|
|
52
|
+
if (id === node.id) continue;
|
|
53
|
+
const sibling = contract.nodes.find((candidate) => candidate.id === id);
|
|
54
|
+
if (!sibling || !phaseTerminalNode(contract, sibling)) continue;
|
|
55
|
+
const snapshot = /** @type {{status?: string}} */ (readNodeSnapshot(runDir, id));
|
|
56
|
+
if (SETTLED.has(/** @type {string} */ (snapshot.status))) ids.add(id);
|
|
57
|
+
}
|
|
58
|
+
return ids;
|
|
59
|
+
}
|
|
60
|
+
|
|
31
61
|
/**
|
|
32
62
|
* The bounded `k/n · argv` shape a running node's status surfaces while a
|
|
33
63
|
* verification command is in flight: the command's 1-based position among
|
|
@@ -108,7 +138,7 @@ export async function executeControllerVerification(contract, runDir, node, stat
|
|
|
108
138
|
};
|
|
109
139
|
writeNode(runDir, state, lock);
|
|
110
140
|
const workspace = attemptWorkspace(state) ?? contract.cwd;
|
|
111
|
-
const commands = [...node.taskPacket.verification, ...sharedVerificationCommands(contract), ...finalVerificationCommands(contract, node)];
|
|
141
|
+
const commands = [...node.taskPacket.verification, ...sharedVerificationCommands(contract), ...finalVerificationCommands(contract, node, settledSiblingIds(runDir, contract, node))];
|
|
112
142
|
/** @param {VerificationAttempt} attempt @returns {VerificationProgress} */
|
|
113
143
|
const progressFor = (attempt) => verificationProgress(attempt.commandIndex + 1, commands.length, /** @type {VerificationCommand|undefined} */ (commands[attempt.commandIndex])?.argv);
|
|
114
144
|
try {
|
|
@@ -221,16 +251,37 @@ function argvEqual(a, b) {
|
|
|
221
251
|
if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false;
|
|
222
252
|
return a.every((item, index) => item === b[index]);
|
|
223
253
|
}
|
|
254
|
+
/**
|
|
255
|
+
* Marks whether the integration candidate's own verification pass is running
|
|
256
|
+
* right now, nested inside the attempt's own already-completed verification
|
|
257
|
+
* record rather than as a new node-snapshot field: that record's validator
|
|
258
|
+
* (`validateVerificationSnapshot`) accepts extra keys, where the node
|
|
259
|
+
* snapshot's own strict field list would reject one. Status surfaces read it
|
|
260
|
+
* to tell "still verifying the sealed candidate" apart from "waiting on the
|
|
261
|
+
* judge", which otherwise both read as whatever phase the attempt last
|
|
262
|
+
* dispatched under.
|
|
263
|
+
*
|
|
264
|
+
* @param {string} runDir
|
|
265
|
+
* @param {NodeSnapshot} state
|
|
266
|
+
* @param {LockHandle|null} lock
|
|
267
|
+
* @param {boolean} active
|
|
268
|
+
*/
|
|
269
|
+
function markCandidateVerification(runDir, state, lock, active) {
|
|
270
|
+
state.verification = /** @type {VerificationState} */ ({ ...(state.verification ?? { passed: false, commands: [], completed: false }), candidate: active });
|
|
271
|
+
writeNode(runDir, state, lock);
|
|
272
|
+
}
|
|
224
273
|
/**
|
|
225
274
|
* @param {ValidatedContract} contract
|
|
226
275
|
* @param {ValidatedNode} node
|
|
227
276
|
* @param {NodeSnapshot} state
|
|
228
277
|
* @param {string} runDir
|
|
229
278
|
* @param {string} workspace
|
|
279
|
+
* @param {LockHandle|null} [lock]
|
|
230
280
|
* @returns {Promise<import("../repo/integrate.mjs").CandidateEvidence>}
|
|
231
281
|
*/
|
|
232
|
-
export async function verifyCandidateWorkspace(contract, node, state, runDir, workspace) {
|
|
233
|
-
const commands = [...node.taskPacket.verification, ...sharedVerificationCommands(contract), ...finalVerificationCommands(contract, node)];
|
|
282
|
+
export async function verifyCandidateWorkspace(contract, node, state, runDir, workspace, lock = null) {
|
|
283
|
+
const commands = [...node.taskPacket.verification, ...sharedVerificationCommands(contract), ...finalVerificationCommands(contract, node, settledSiblingIds(runDir, contract, node))];
|
|
284
|
+
markCandidateVerification(runDir, state, lock, true);
|
|
234
285
|
try {
|
|
235
286
|
const result = await runVerification(commands, workspace, {
|
|
236
287
|
logDir: join(runDir, "logs", `${node.id}.${state.attempt}.candidate-verification`),
|
|
@@ -252,5 +303,7 @@ export async function verifyCandidateWorkspace(contract, node, state, runDir, wo
|
|
|
252
303
|
return settled.retried ? { ...compacted, retried: settled.retried } : compacted;
|
|
253
304
|
} catch (error) {
|
|
254
305
|
return { passed: false, error: boundedUtf8(errorMessage(error), 4 * 1024) };
|
|
306
|
+
} finally {
|
|
307
|
+
markCandidateVerification(runDir, state, lock, false);
|
|
255
308
|
}
|
|
256
309
|
}
|
|
@@ -5,6 +5,8 @@ import { finite } from "../util.mjs";
|
|
|
5
5
|
* `zcode.mjs`, `replay.mjs` and `exec-jsonl.mjs` itself all depend on it.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
+
/** @typedef {{remaining: number|null, limit: number|null, resetsAt: string|null, window: string|null}} ClaudeAllowance */
|
|
9
|
+
|
|
8
10
|
/**
|
|
9
11
|
* Parse newline-delimited JSON without accepting provider prose.
|
|
10
12
|
*
|
|
@@ -75,33 +77,110 @@ function withStartupReason(message, options = {}) {
|
|
|
75
77
|
return reason ? `${message}: ${boundedMessage(reason, 512)}` : message;
|
|
76
78
|
}
|
|
77
79
|
|
|
80
|
+
/**
|
|
81
|
+
* Claude's account-wide rate-limit signal, reported on its own `rate_limit_event`
|
|
82
|
+
* stream line, never on the terminal `result` event. Measured 2026-09-17
|
|
83
|
+
* against `claude -p 'Reply with exactly OK and use no tools.' --output-format
|
|
84
|
+
* stream-json --verbose`, cost `total_cost_usd: 0.27848` for that one probe
|
|
85
|
+
* (opus-tier default model, most of it `cache_creation_input_tokens` for this
|
|
86
|
+
* project's session context -- a probe against a smaller/cheaper model would
|
|
87
|
+
* cost far less, but the seat sampled is whichever model the operator's own
|
|
88
|
+
* session already has configured, so this is the honest per-probe figure for
|
|
89
|
+
* that seat). No `rate_limits` or `rate_limit_status` field appears anywhere
|
|
90
|
+
* in the stream. The measured line, verbatim:
|
|
91
|
+
* `{"type":"rate_limit_event","rate_limit_info":{"status":"allowed_warning",
|
|
92
|
+
* "resetsAt":1789837200,"rateLimitType":"seven_day","utilization":0.77,
|
|
93
|
+
* "isUsingOverage":false,"surpassedThreshold":0.75,"unifiedWindows":
|
|
94
|
+
* {"five_hour":{"utilization":0.1,"resetsAt":1789663200},"seven_day":
|
|
95
|
+
* {"utilization":0.77,"resetsAt":1789837200}}}}`. `utilization` is a 0..1
|
|
96
|
+
* fraction of the window named by `rateLimitType`, confirmed (not merely
|
|
97
|
+
* assumed) by `surpassedThreshold: 0.75` sitting just below the measured
|
|
98
|
+
* `0.77` at `status: "allowed_warning"` -- a 0-100 percentage scale would put
|
|
99
|
+
* 0.77 nowhere near a 75-scaled threshold. This is a different field, on a
|
|
100
|
+
* different scale, from the `used_percentage` (0-100) the statusline
|
|
101
|
+
* integration already reads off `rate_limits.five_hour` (see
|
|
102
|
+
* `integrations/claude-code/statusline.sh`); the two must not be confused.
|
|
103
|
+
* `remaining` here is the fraction left (`1 - utilization`) and `limit` is
|
|
104
|
+
* the fixed ceiling `1` -- a coarse proxy, declared as such. A `utilization`
|
|
105
|
+
* outside `[0, 1]` is treated as no signal (null) rather than silently
|
|
106
|
+
* clamped, so a future scale change on the stream fails visibly instead of
|
|
107
|
+
* pinning every sample to 0.
|
|
108
|
+
*
|
|
109
|
+
* `unifiedWindows` carries every window's own utilization side by side
|
|
110
|
+
* (`five_hour: 0.1` next to `seven_day: 0.77` in the measured line above): the
|
|
111
|
+
* top-level `utilization` is only ever the figure for the window `rateLimitType`
|
|
112
|
+
* names, so a sample must read the named window out of `unifiedWindows` (falling
|
|
113
|
+
* back to the top-level field only when `unifiedWindows` carries no entry for
|
|
114
|
+
* it) and record which window that was. Two samples of different windows are
|
|
115
|
+
* not comparable -- a `five_hour` utilization minus a `seven_day` one is not a
|
|
116
|
+
* delta -- so the window travels with the sample for `allowanceDelta` to pin.
|
|
117
|
+
*
|
|
118
|
+
* @param {Record<string, unknown>[]} events
|
|
119
|
+
* @returns {ClaudeAllowance}
|
|
120
|
+
*/
|
|
121
|
+
function extractClaudeAllowance(events) {
|
|
122
|
+
const event = events.findLast((candidate) => candidate.type === "rate_limit_event");
|
|
123
|
+
const info = event?.rate_limit_info;
|
|
124
|
+
const record = info && typeof info === "object" && !Array.isArray(info) ? /** @type {Record<string, unknown>} */ (info) : null;
|
|
125
|
+
const window = record && typeof record.rateLimitType === "string" ? record.rateLimitType : null;
|
|
126
|
+
const unifiedWindows = record?.unifiedWindows && typeof record.unifiedWindows === "object" && !Array.isArray(record.unifiedWindows)
|
|
127
|
+
? /** @type {Record<string, unknown>} */ (record.unifiedWindows)
|
|
128
|
+
: null;
|
|
129
|
+
const namedWindowEntry = window && unifiedWindows?.[window] && typeof unifiedWindows[window] === "object" && !Array.isArray(unifiedWindows[window])
|
|
130
|
+
? /** @type {Record<string, unknown>} */ (unifiedWindows[window])
|
|
131
|
+
: null;
|
|
132
|
+
const rawUtilization = finite(namedWindowEntry ? namedWindowEntry.utilization : record?.utilization);
|
|
133
|
+
const utilization = rawUtilization !== null && rawUtilization >= 0 && rawUtilization <= 1 ? rawUtilization : null;
|
|
134
|
+
const resetsAtSeconds = finite(namedWindowEntry ? namedWindowEntry.resetsAt : record?.resetsAt);
|
|
135
|
+
return {
|
|
136
|
+
remaining: utilization === null ? null : 1 - utilization,
|
|
137
|
+
limit: utilization === null ? null : 1,
|
|
138
|
+
resetsAt: resetsAtSeconds === null ? null : new Date(resetsAtSeconds * 1000).toISOString(),
|
|
139
|
+
window,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** The claude allowance shape with every member null: no signal was read. */
|
|
144
|
+
const NULL_CLAUDE_ALLOWANCE = { remaining: null, limit: null, resetsAt: null, window: null };
|
|
145
|
+
|
|
78
146
|
/**
|
|
79
147
|
* @param {string} stdout
|
|
80
148
|
* @param {number|null} exitCode
|
|
81
149
|
* @param {string|null} signal
|
|
82
150
|
* @param {import("./index.mjs").NormalizeOptions} [options]
|
|
83
|
-
* @returns {import("./index.mjs").ProviderEnvelope}
|
|
151
|
+
* @returns {import("./index.mjs").ProviderEnvelope & {allowance: ClaudeAllowance}}
|
|
84
152
|
*/
|
|
85
153
|
export function normalizeClaudeResult(stdout, exitCode, signal, options = {}) {
|
|
86
|
-
|
|
154
|
+
// The signal check must stay the first statement, byte-identical to before
|
|
155
|
+
// the allowance signal existed: `parseJsonLines` throws on any unparseable
|
|
156
|
+
// line past the first, and a killed process routinely leaves a truncated
|
|
157
|
+
// *later* line (the bounded-tail tolerance only covers the first). Parsing
|
|
158
|
+
// stdout before checking `signal` would turn a clean cancellation into a
|
|
159
|
+
// thrown error, which callers (`src/run/usage.mjs`, `src/engine/process.mjs`)
|
|
160
|
+
// catch into `invalid_output` or `null`, losing the envelope entirely.
|
|
161
|
+
if (signal) return { ...failed("canceled", `provider ended after ${signal}`, "canceled"), allowance: NULL_CLAUDE_ALLOWANCE };
|
|
87
162
|
const events = parseJsonLines(stdout, "claude");
|
|
163
|
+
const allowance = extractClaudeAllowance(events);
|
|
88
164
|
const resultEvent = events.findLast((event) => event.type === "result");
|
|
89
|
-
if (!resultEvent) return failed("incomplete_stream", withStartupReason("Claude emitted no result event", options));
|
|
165
|
+
if (!resultEvent) return { ...failed("incomplete_stream", withStartupReason("Claude emitted no result event", options)), allowance };
|
|
90
166
|
const result = typeof resultEvent.result === "string" ? resultEvent.result : null;
|
|
91
167
|
// A provider-reported quota stop is exhaustion: the declared failover edge
|
|
92
168
|
// must fire instead of settling the node as an ordinary provider failure.
|
|
93
169
|
const quotaText = claudeQuotaText(resultEvent, events);
|
|
94
170
|
if (quotaText) {
|
|
95
|
-
return
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
171
|
+
return {
|
|
172
|
+
...failed(
|
|
173
|
+
"quota_exhausted",
|
|
174
|
+
boundedMessage(quotaText, 512),
|
|
175
|
+
"exhausted",
|
|
176
|
+
typeof resultEvent.session_id === "string" ? resultEvent.session_id : null,
|
|
177
|
+
canonicalUsage(resultEvent.usage),
|
|
178
|
+
),
|
|
179
|
+
allowance,
|
|
180
|
+
};
|
|
102
181
|
}
|
|
103
182
|
if (resultEvent.is_error || exitCode !== 0) {
|
|
104
|
-
return failed("provider_error", result ?? `Claude exited with code ${exitCode}`);
|
|
183
|
+
return { ...failed("provider_error", result ?? `Claude exited with code ${exitCode}`), allowance };
|
|
105
184
|
}
|
|
106
185
|
return {
|
|
107
186
|
status: result?.trim() ? "done" : "no-op",
|
|
@@ -110,6 +189,7 @@ export function normalizeClaudeResult(stdout, exitCode, signal, options = {}) {
|
|
|
110
189
|
usage: canonicalUsage(resultEvent.usage),
|
|
111
190
|
costUsd: finite(resultEvent.total_cost_usd),
|
|
112
191
|
error: null,
|
|
192
|
+
allowance,
|
|
113
193
|
};
|
|
114
194
|
}
|
|
115
195
|
|
package/src/plan/freeze.mjs
CHANGED
|
@@ -9,10 +9,11 @@
|
|
|
9
9
|
* Nothing here invokes a model or the engine; it only writes and hashes
|
|
10
10
|
* bytes, so freezing a plan can never be mistaken for starting a run.
|
|
11
11
|
*/
|
|
12
|
-
import { mkdirSync, readFileSync, rmSync
|
|
12
|
+
import { mkdirSync, readFileSync, rmSync } from "node:fs";
|
|
13
13
|
import { join } from "node:path";
|
|
14
14
|
import { fileURLToPath } from "node:url";
|
|
15
15
|
import { CONTRACT_VERSION, PROTOCOL_SCHEMA_VERSION, contractDigest, validateContract } from "../contract/index.mjs";
|
|
16
|
+
import { writeJsonAtomic } from "../run/store.mjs";
|
|
16
17
|
|
|
17
18
|
/** @typedef {import("../contract/index.mjs").JsonObject} JsonObject */
|
|
18
19
|
|
|
@@ -55,7 +56,7 @@ export function freezePlan(plan, { outDir, provenance }) {
|
|
|
55
56
|
contractVersion: CONTRACT_VERSION,
|
|
56
57
|
...plan,
|
|
57
58
|
});
|
|
58
|
-
|
|
59
|
+
writeJsonAtomic(contractPath, raw);
|
|
59
60
|
try {
|
|
60
61
|
validateContract(raw, contractPath);
|
|
61
62
|
} catch (error) {
|
|
@@ -76,7 +77,10 @@ export function freezePlan(plan, { outDir, provenance }) {
|
|
|
76
77
|
findings: provenance.findings,
|
|
77
78
|
},
|
|
78
79
|
});
|
|
79
|
-
|
|
80
|
+
// Atomic: the pipeline rewrites this file with its status straight after, and a
|
|
81
|
+
// reader polling for the frozen plan must never see a torn or half-written one
|
|
82
|
+
// (measured 2026-09-17: eval case D25 read a statusless plan.json on a slow runner).
|
|
83
|
+
writeJsonAtomic(join(outDir, "plan.json"), frozen);
|
|
80
84
|
return frozen;
|
|
81
85
|
}
|
|
82
86
|
|
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The planning pipeline: `faberun plan` as successive ordinary runs (draft,
|
|
3
|
+
* review, revise up to a round budget) followed by the deterministic stages
|
|
4
|
+
* (sizing, routing, freeze), never as one long-lived process. Separate from
|
|
5
|
+
* `template.mjs` (which only builds the one-node contracts) and from
|
|
6
|
+
* `freeze.mjs` (which only turns a plan into a validated contract on disk):
|
|
7
|
+
* this module is the one place that sequences those runs, decides when a
|
|
8
|
+
* plan is contested instead of frozen, and records the operator-approval
|
|
9
|
+
* open-question. `launch` and `wait` are the only two seams that touch a
|
|
10
|
+
* process or the wall clock, so a test drives the whole pipeline through
|
|
11
|
+
* `runContract` in-process, deterministically.
|
|
12
|
+
*/
|
|
13
|
+
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
14
|
+
import { dirname, join, relative, resolve } from "node:path";
|
|
15
|
+
import { validateContract } from "../contract/index.mjs";
|
|
16
|
+
import { discoveryOutput } from "../contract/worker-result.mjs";
|
|
17
|
+
import { readWorkerResultFile } from "../engine/result-file.mjs";
|
|
18
|
+
import { classifyRunProgress } from "../campaign/chain.mjs";
|
|
19
|
+
import { campaignDir } from "../campaign/layout.mjs";
|
|
20
|
+
import { appendSeatAllowanceEvent, readJournal } from "../campaign/journal.mjs";
|
|
21
|
+
import { readCampaign } from "../campaign/record.mjs";
|
|
22
|
+
import { campaignCli } from "../cli/campaign.mjs";
|
|
23
|
+
import { appendJsonl, writeJsonAtomic } from "../run/store.mjs";
|
|
24
|
+
import { allowanceDelta, allowanceEventFields, sampleAllowance } from "../seat/allowance.mjs";
|
|
25
|
+
import { validateSpec } from "./spec.mjs";
|
|
26
|
+
import { collectRepoFacts } from "./repo-facts.mjs";
|
|
27
|
+
import { RISK_TIERS, buildPlanningContract, validateFindings, validatePlanOutput } from "./template.mjs";
|
|
28
|
+
import { applySizingRules } from "./sizing.mjs";
|
|
29
|
+
import { resolveRuntimes } from "./routing.mjs";
|
|
30
|
+
import { freezePlan } from "./freeze.mjs";
|
|
31
|
+
|
|
32
|
+
/** @typedef {import("../contract/index.mjs").JsonObject} JsonObject */
|
|
33
|
+
/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
|
|
34
|
+
/** @typedef {import("./template.mjs").PlanOutput} PlanOutput */
|
|
35
|
+
/** @typedef {import("./template.mjs").PlanFindingOutput} PlanFindingOutput */
|
|
36
|
+
/** @typedef {import("./sizing.mjs").PlanNode & {objective: string}} SizedPlanNode */
|
|
37
|
+
/** @typedef {"standard"|"high"|"none"} ApproveBelow */
|
|
38
|
+
/** @typedef {(contractPath: string, contract: ValidatedContract) => Promise<void>|void} LaunchFn */
|
|
39
|
+
/** @typedef {(runDir: string) => Promise<import("../engine/supervise.mjs").RunProgress>|import("../engine/supervise.mjs").RunProgress} WaitFn */
|
|
40
|
+
/** @typedef {{status: "frozen", plansDir: string, planPath: string, contractPath: string, approved: boolean, findings: PlanFindingOutput[]}} FrozenPipelineResult */
|
|
41
|
+
/** @typedef {{status: "contested", plansDir: string, planPath: string, findings: PlanFindingOutput[], round: number}} ContestedPipelineResult */
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The session id every automated journal entry this pipeline writes carries.
|
|
45
|
+
* There is no human session behind a `plan` invocation, so a fixed id names
|
|
46
|
+
* the writer the same way `src/web/api.mjs`'s `WEB_SESSION_ID` names the web
|
|
47
|
+
* surface's own automated writes.
|
|
48
|
+
*/
|
|
49
|
+
export const PLANNER_SESSION_ID = "planner";
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* No taskKind/riskTier row is opinionated by default: absent an operator
|
|
53
|
+
* `--runtime-defaults` instruction, every sized node routes through plain
|
|
54
|
+
* availability discovery (`resolveRuntimes`'s cheapest worker, strongest
|
|
55
|
+
* cross-vendor judge). A default table cannot safely name a `prefer` runtime
|
|
56
|
+
* id without knowing the operator's own catalogue, so "small default" here
|
|
57
|
+
* means empty rather than guessed.
|
|
58
|
+
*/
|
|
59
|
+
export const DEFAULT_ROUTING_TABLE = /** @type {import("./routing.mjs").RoutingRule[]} */ ([]);
|
|
60
|
+
|
|
61
|
+
/** The sizing budget a frozen node's verification is measured against, absent a project-specific one. */
|
|
62
|
+
export const DEFAULT_NODE_BUDGET_MS = 600_000;
|
|
63
|
+
|
|
64
|
+
const APPROVE_BELOW_VALUES = new Set(["standard", "high", "none"]);
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* @param {{specPath: string, campaignId: string, phase: string, cwd?: string, reviewRounds?: number, approveBelow?: ApproveBelow, runtimeDefaults?: {worker?: string, judge?: string}, runtimes: Record<string, JsonObject>, launch: LaunchFn, wait: WaitFn}} options
|
|
68
|
+
* @returns {Promise<FrozenPipelineResult|ContestedPipelineResult>}
|
|
69
|
+
*/
|
|
70
|
+
export async function runPlanningPipeline(options) {
|
|
71
|
+
const {
|
|
72
|
+
specPath, campaignId, phase, runtimes, launch, wait,
|
|
73
|
+
reviewRounds = 2, runtimeDefaults = {},
|
|
74
|
+
} = options;
|
|
75
|
+
const approveBelow = /** @type {ApproveBelow} */ (options.approveBelow ?? "standard");
|
|
76
|
+
if (!APPROVE_BELOW_VALUES.has(approveBelow)) throw new TypeError(`approveBelow must be one of ${[...APPROVE_BELOW_VALUES].join(", ")}`);
|
|
77
|
+
if (typeof launch !== "function") throw new TypeError("runPlanningPipeline requires a launch seam");
|
|
78
|
+
if (typeof wait !== "function") throw new TypeError("runPlanningPipeline requires a wait seam");
|
|
79
|
+
const cwd = resolve(options.cwd ?? ".");
|
|
80
|
+
|
|
81
|
+
const campaignPath = campaignDir(join(cwd, ".runs"), campaignId);
|
|
82
|
+
const campaign = readCampaign(campaignPath);
|
|
83
|
+
if (campaign.status !== "active") throw new Error(`campaign is closed: ${campaignId}`);
|
|
84
|
+
|
|
85
|
+
const relativeSpecPath = repoRelativePath(cwd, specPath, "specPath");
|
|
86
|
+
const specText = readFileSync(resolve(cwd, relativeSpecPath), "utf8");
|
|
87
|
+
const specValidation = validateSpec(specText, { cwd, strict: true });
|
|
88
|
+
if (specValidation.class === "structured" && !specValidation.ok) {
|
|
89
|
+
const detail = specValidation.findings.map((finding) => `${finding.rule}: ${finding.message}`).join("; ");
|
|
90
|
+
throw new Error(`spec ${specPath} fails strict traceability: ${detail}`);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const plansDir = join(cwd, ".runs", "campaigns", campaignId, "plans", phase);
|
|
94
|
+
mkdirSync(plansDir, { recursive: true });
|
|
95
|
+
const pipelineLog = join(plansDir, "pipeline.jsonl");
|
|
96
|
+
/** @param {string} stage @param {Record<string, unknown>} [extra] */
|
|
97
|
+
const logStage = (stage, extra = {}) => appendJsonl(pipelineLog, {
|
|
98
|
+
type: "plan.stage", at: new Date().toISOString(), campaignId, phase, stage, ...extra,
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
const repoFacts = collectRepoFacts(cwd);
|
|
102
|
+
const repoFactsPath = join(plansDir, "repo-facts.json");
|
|
103
|
+
writeFileSync(repoFactsPath, `${JSON.stringify(repoFacts, null, 2)}\n`);
|
|
104
|
+
const relativeRepoFactsPath = relative(cwd, repoFactsPath);
|
|
105
|
+
logStage("repo-facts", { gitHead: repoFacts.gitHead });
|
|
106
|
+
|
|
107
|
+
let n = 0;
|
|
108
|
+
const nextN = () => { n += 1; return n; };
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Build, validate, persist, launch and wait for one planning contract, and
|
|
112
|
+
* return the discovery `output` its worker recorded. Every stage the
|
|
113
|
+
* pipeline runs is an ordinary run: it lands in `usage.jsonl` exactly like
|
|
114
|
+
* any other node, and this is the only place that reads its result back.
|
|
115
|
+
*
|
|
116
|
+
* @param {import("./template.mjs").PlanningKind} kind
|
|
117
|
+
* @param {Record<string, unknown>} inputs
|
|
118
|
+
* @returns {Promise<{contract: ValidatedContract, output: Record<string, unknown>}>}
|
|
119
|
+
*/
|
|
120
|
+
const runStage = async (kind, inputs) => {
|
|
121
|
+
const stageN = nextN();
|
|
122
|
+
const contractPath = join(plansDir, "nodes", `${kind}-${stageN}.contract.json`);
|
|
123
|
+
mkdirSync(dirname(contractPath), { recursive: true });
|
|
124
|
+
const relativeCwd = relative(dirname(contractPath), cwd) || ".";
|
|
125
|
+
const raw = buildPlanningContract(kind, {
|
|
126
|
+
campaignId, phase, n: stageN, runtimes, runtimeDefaults, cwd: relativeCwd, ...inputs,
|
|
127
|
+
});
|
|
128
|
+
const validated = validateContract(raw, contractPath);
|
|
129
|
+
writeFileSync(contractPath, `${JSON.stringify(raw, null, 2)}\n`);
|
|
130
|
+
await launch(contractPath, validated);
|
|
131
|
+
const runDir = join(validated.cwd, ".runs", validated.id);
|
|
132
|
+
const progress = await wait(runDir);
|
|
133
|
+
const classification = classifyRunProgress(progress);
|
|
134
|
+
if (classification !== "succeeded") {
|
|
135
|
+
throw new Error(`planning stage ${kind} did not succeed: run ${validated.id} ${classification}`);
|
|
136
|
+
}
|
|
137
|
+
const result = readWorkerResultFile(runDir, kind);
|
|
138
|
+
const output = result ? discoveryOutput(result) : null;
|
|
139
|
+
if (!output) throw new Error(`planning stage ${kind}: run ${validated.id} recorded no discovery output`);
|
|
140
|
+
return { contract: validated, output };
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
const draft = await runStage("draft", { specPath: relativeSpecPath, repoFactsPath: relativeRepoFactsPath });
|
|
144
|
+
let plan = validatePlanOutput(draft.output.plan);
|
|
145
|
+
logStage("draft", { runId: draft.contract.id, nodeCount: plan.nodes.length });
|
|
146
|
+
|
|
147
|
+
const workingPlanPath = join(plansDir, "plan.working.json");
|
|
148
|
+
writeJsonAtomic(workingPlanPath, plan);
|
|
149
|
+
const relativeWorkingPlanPath = relative(cwd, workingPlanPath);
|
|
150
|
+
|
|
151
|
+
/** @type {PlanFindingOutput[]} */
|
|
152
|
+
let findings = [];
|
|
153
|
+
for (let round = 1; round <= reviewRounds; round += 1) {
|
|
154
|
+
const review = await runStage("review", { specPath: relativeSpecPath, repoFactsPath: relativeRepoFactsPath, planPath: relativeWorkingPlanPath });
|
|
155
|
+
findings = validateFindings(review.output.findings);
|
|
156
|
+
const criticalFindings = findings.filter((finding) => finding.severity === "critical");
|
|
157
|
+
logStage("review", { round, runId: review.contract.id, findingsCount: findings.length, criticalCount: criticalFindings.length });
|
|
158
|
+
if (criticalFindings.length === 0) break;
|
|
159
|
+
if (round === reviewRounds) {
|
|
160
|
+
const planPath = join(plansDir, "plan.json");
|
|
161
|
+
writeJsonAtomic(planPath, { formatVersion: 1, status: "contested", rounds: round, findings });
|
|
162
|
+
logStage("contested", { round, criticalCount: criticalFindings.length });
|
|
163
|
+
await campaignCli([
|
|
164
|
+
"note", campaignId, "--cwd", cwd, "--session-id", PLANNER_SESSION_ID,
|
|
165
|
+
"--kind", "open-question", "--question-id", `plan-${phase}-contested`,
|
|
166
|
+
"--text", `Plan for phase ${phase} is contested after ${round} review round(s): ${criticalFindings.map((finding) => finding.text).join("; ")}`,
|
|
167
|
+
]);
|
|
168
|
+
return { status: "contested", plansDir, planPath, findings, round };
|
|
169
|
+
}
|
|
170
|
+
const findingsPath = join(plansDir, `findings-round-${round}.json`);
|
|
171
|
+
writeJsonAtomic(findingsPath, findings);
|
|
172
|
+
const revise = await runStage("revise", {
|
|
173
|
+
specPath: relativeSpecPath, repoFactsPath: relativeRepoFactsPath, findingsPath: relative(cwd, findingsPath),
|
|
174
|
+
});
|
|
175
|
+
plan = validatePlanOutput(revise.output.plan);
|
|
176
|
+
writeJsonAtomic(workingPlanPath, plan);
|
|
177
|
+
logStage("revise", { round, runId: revise.contract.id });
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const sizing = applySizingRules(
|
|
181
|
+
{ nodes: plan.nodes.map(toSizingNode), justification: plan.justification },
|
|
182
|
+
{ nodeBudgetMs: DEFAULT_NODE_BUDGET_MS, facts: repoFacts },
|
|
183
|
+
);
|
|
184
|
+
logStage("sizing", { transformations: sizing.transformations.length, nodeCount: sizing.plan.nodes.length });
|
|
185
|
+
|
|
186
|
+
const routingRuntimes = /** @type {Record<string, import("./routing.mjs").RoutingRuntime>} */ (runtimes);
|
|
187
|
+
const routing = resolveRuntimes(sizing.plan.nodes, {
|
|
188
|
+
table: [...DEFAULT_ROUTING_TABLE],
|
|
189
|
+
runtimes: routingRuntimes,
|
|
190
|
+
availability: availabilityOf(runtimes),
|
|
191
|
+
runtimeDefaults,
|
|
192
|
+
});
|
|
193
|
+
logStage("routing", { assignments: Object.keys(routing.assignments).length });
|
|
194
|
+
|
|
195
|
+
const highestRiskTier = highestOf(sizing.plan.nodes.map((node) => node.riskTier ?? RISK_TIERS[0]));
|
|
196
|
+
const nodes = sizing.plan.nodes.map((node) => toContractNode(/** @type {SizedPlanNode} */ (node), phase, routing.assignments[node.id]));
|
|
197
|
+
const frozen = freezePlan({
|
|
198
|
+
id: `${campaignId}-${phase}`,
|
|
199
|
+
campaignId,
|
|
200
|
+
goal: campaign.goal,
|
|
201
|
+
// `freezePlan` writes contract.json inside `outDir` (`plansDir`), so `cwd`
|
|
202
|
+
// has to point back at the repo root from there, exactly like `runStage`
|
|
203
|
+
// computes it for the nodes it writes under `plansDir/nodes/`.
|
|
204
|
+
cwd: relative(plansDir, cwd) || ".",
|
|
205
|
+
runtimes,
|
|
206
|
+
runtimeDefaults,
|
|
207
|
+
nodes,
|
|
208
|
+
}, {
|
|
209
|
+
outDir: plansDir,
|
|
210
|
+
provenance: {
|
|
211
|
+
targetGitHead: repoFacts.gitHead,
|
|
212
|
+
planner: { runtimeId: runtimeDefaults.worker ?? "", model: modelOf(runtimes, runtimeDefaults.worker) },
|
|
213
|
+
reviewer: { runtimeId: runtimeDefaults.judge ?? "", model: modelOf(runtimes, runtimeDefaults.judge) },
|
|
214
|
+
sizing: sizing.transformations,
|
|
215
|
+
findings,
|
|
216
|
+
},
|
|
217
|
+
});
|
|
218
|
+
logStage("freeze", { contractId: `${campaignId}-${phase}`, highestRiskTier });
|
|
219
|
+
|
|
220
|
+
// A delta only means something between two samples of the same seat: freeze
|
|
221
|
+
// re-samples the exact harness `campaign init` recorded at `sample: "start"`
|
|
222
|
+
// (the operator's own seat), not the plan's worker runtime, which is very
|
|
223
|
+
// often a different harness entirely (codex, dsh, agy, zcode workers under
|
|
224
|
+
// a claude operator) and would make the delta null in the common case
|
|
225
|
+
// instead of the rare one. With no start entry at all (a pipeline run with
|
|
226
|
+
// no preceding `campaign init`, as in every replay-driven pipeline test)
|
|
227
|
+
// there is no seat to re-sample, so freeze samples nothing and spends no
|
|
228
|
+
// call.
|
|
229
|
+
const journalEntries = /** @type {any[]} */ (readJournal(campaignPath));
|
|
230
|
+
const startEntry = journalEntries.findLast((entry) => entry.type === "seat.allowance" && entry.sample === "start");
|
|
231
|
+
const freezeHarness = startEntry?.harness ?? null;
|
|
232
|
+
const freezeAllowance = await sampleAllowance({ harness: freezeHarness });
|
|
233
|
+
const startAllowance = startEntry
|
|
234
|
+
? { remaining: startEntry.remaining ?? null, limit: startEntry.limit ?? null, resetsAt: startEntry.resetsAt ?? null, window: startEntry.window ?? null }
|
|
235
|
+
: null;
|
|
236
|
+
appendSeatAllowanceEvent(campaignPath, {
|
|
237
|
+
sample: "freeze",
|
|
238
|
+
harness: freezeHarness,
|
|
239
|
+
delta: allowanceDelta(startAllowance, freezeAllowance),
|
|
240
|
+
...allowanceEventFields(freezeAllowance),
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
const approved = approveBelow === "high" ? true : approveBelow === "none" ? false : highestRiskTier !== "high";
|
|
244
|
+
const planPath = join(plansDir, "plan.json");
|
|
245
|
+
writeJsonAtomic(planPath, { ...frozen, status: "frozen", approved });
|
|
246
|
+
if (!approved) {
|
|
247
|
+
await campaignCli([
|
|
248
|
+
"note", campaignId, "--cwd", cwd, "--session-id", PLANNER_SESSION_ID,
|
|
249
|
+
"--kind", "open-question", "--question-id", `plan-${phase}-approval`,
|
|
250
|
+
"--text", `Plan for phase ${phase} carries a ${highestRiskTier}-risk node; approval is required under --approve-below ${approveBelow}.`,
|
|
251
|
+
]);
|
|
252
|
+
}
|
|
253
|
+
logStage("approval", { approved, approveBelow, highestRiskTier });
|
|
254
|
+
|
|
255
|
+
return { status: "frozen", plansDir, planPath, contractPath: join(plansDir, "contract.json"), approved, findings };
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* `path` made relative to `cwd`, refused when it escapes it: every planning
|
|
260
|
+
* contract's readFiles must resolve inside the same cwd a run validates
|
|
261
|
+
* against, so a spec outside the target repository can never be named there.
|
|
262
|
+
*
|
|
263
|
+
* @param {string} cwd
|
|
264
|
+
* @param {string} path
|
|
265
|
+
* @param {string} label
|
|
266
|
+
* @returns {string}
|
|
267
|
+
*/
|
|
268
|
+
function repoRelativePath(cwd, path, label) {
|
|
269
|
+
const relativePath = relative(cwd, resolve(cwd, path));
|
|
270
|
+
if (relativePath.startsWith("..")) throw new Error(`${label} must be inside ${cwd}: ${path}`);
|
|
271
|
+
return relativePath;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Every declared runtime treated as available. Live discovery (probing a
|
|
276
|
+
* harness for real exhaustion) is a separate concern this pipeline does not
|
|
277
|
+
* take on; a campaign that needs it can inject a table row and prune its
|
|
278
|
+
* `runtimes` catalogue instead.
|
|
279
|
+
*
|
|
280
|
+
* @param {Record<string, JsonObject>} runtimes
|
|
281
|
+
* @returns {Record<string, {available: true, exhaustedUntil: null}>}
|
|
282
|
+
*/
|
|
283
|
+
function availabilityOf(runtimes) {
|
|
284
|
+
return Object.fromEntries(Object.keys(runtimes).map((id) => [id, { available: true, exhaustedUntil: null }]));
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* @param {Record<string, JsonObject>} runtimes
|
|
289
|
+
* @param {string|undefined} id
|
|
290
|
+
* @returns {string}
|
|
291
|
+
*/
|
|
292
|
+
function modelOf(runtimes, id) {
|
|
293
|
+
const model = id ? runtimes[id]?.model : undefined;
|
|
294
|
+
return typeof model === "string" ? model : "";
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/**
|
|
298
|
+
* @param {string[]} riskTiers
|
|
299
|
+
* @returns {string}
|
|
300
|
+
*/
|
|
301
|
+
function highestOf(riskTiers) {
|
|
302
|
+
return riskTiers.reduce((highest, tier) => (RISK_TIERS.indexOf(tier) > RISK_TIERS.indexOf(highest) ? tier : highest), RISK_TIERS[0]);
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* A draft or revise output node (flat `readFiles`/`writeFiles`/`verification`)
|
|
307
|
+
* turned into the shape `applySizingRules` merges and splits: those fields move
|
|
308
|
+
* under `taskPacket`, alongside `sizing.mjs`'s own `writeFiles`/`verification`
|
|
309
|
+
* expectations, while `objective` rides along as a passthrough field a merge
|
|
310
|
+
* never touches.
|
|
311
|
+
*
|
|
312
|
+
* @param {import("./template.mjs").PlanOutputNode} node
|
|
313
|
+
* @returns {SizedPlanNode}
|
|
314
|
+
*/
|
|
315
|
+
function toSizingNode(node) {
|
|
316
|
+
return /** @type {SizedPlanNode} */ ({
|
|
317
|
+
id: node.id,
|
|
318
|
+
dependsOn: node.dependsOn,
|
|
319
|
+
taskKind: node.taskKind,
|
|
320
|
+
riskTier: node.riskTier,
|
|
321
|
+
objective: node.objective,
|
|
322
|
+
definitionOfDone: node.definitionOfDone,
|
|
323
|
+
taskPacket: {
|
|
324
|
+
readFiles: node.readFiles,
|
|
325
|
+
writeFiles: node.writeFiles,
|
|
326
|
+
verification: node.verification,
|
|
327
|
+
},
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* A sized plan node's classification and shape, turned into the contract node
|
|
333
|
+
* `freezePlan` validates. `riskTier: "low"` gets no gate; `standard` an
|
|
334
|
+
* advisory one; `high` a blocking one, which `validateGate` requires `major`
|
|
335
|
+
* in `failOn` for.
|
|
336
|
+
*
|
|
337
|
+
* @param {SizedPlanNode} node
|
|
338
|
+
* @param {string} phase
|
|
339
|
+
* @param {{worker: string|null, judge: string|null}|undefined} assignment
|
|
340
|
+
* @returns {JsonObject}
|
|
341
|
+
*/
|
|
342
|
+
function toContractNode(node, phase, assignment) {
|
|
343
|
+
const riskTier = /** @type {string} */ (node.riskTier);
|
|
344
|
+
const gate = riskTier === "low"
|
|
345
|
+
? false
|
|
346
|
+
: {
|
|
347
|
+
review: riskTier === "high" ? "blocking" : "advisory",
|
|
348
|
+
failOn: riskTier === "high" ? ["major", "critical"] : ["critical"],
|
|
349
|
+
...(assignment?.judge ? { runtime: assignment.judge } : {}),
|
|
350
|
+
};
|
|
351
|
+
return {
|
|
352
|
+
id: node.id,
|
|
353
|
+
type: node.taskKind,
|
|
354
|
+
phase,
|
|
355
|
+
dependsOn: node.dependsOn ?? [],
|
|
356
|
+
...(assignment?.worker ? { runtime: assignment.worker } : {}),
|
|
357
|
+
taskPacket: {
|
|
358
|
+
mode: "execution",
|
|
359
|
+
objective: node.objective,
|
|
360
|
+
instructions: [node.objective],
|
|
361
|
+
readFiles: node.taskPacket.readFiles ?? [],
|
|
362
|
+
writeFiles: node.taskPacket.writeFiles ?? [],
|
|
363
|
+
symbols: [],
|
|
364
|
+
decisions: [],
|
|
365
|
+
nonGoals: [],
|
|
366
|
+
verification: node.taskPacket.verification,
|
|
367
|
+
},
|
|
368
|
+
definitionOfDone: node.definitionOfDone ?? [],
|
|
369
|
+
gate,
|
|
370
|
+
};
|
|
371
|
+
}
|