faberun 0.30.0 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/campaign/campaign-brief.mjs +17 -6
- package/src/campaign/chain.mjs +124 -8
- package/src/campaign/ledger.mjs +19 -7
- package/src/campaign/metrics-command.mjs +68 -9
- package/src/campaign/metrics.mjs +75 -45
- package/src/campaign/record.mjs +8 -1
- package/src/campaign/reserve.mjs +340 -0
- package/src/campaign/spend.mjs +229 -0
- package/src/campaign/unpark.mjs +54 -1
- package/src/campaign/watch.mjs +314 -37
- package/src/cli/brand.mjs +2 -1
- package/src/cli.mjs +52 -2
- package/src/contract/judge-envelope.mjs +17 -2
- package/src/contract/scope-findings.mjs +3 -1
- package/src/contract/task-packet.mjs +40 -5
- package/src/contract/verification.mjs +67 -1
- package/src/engine/aux-admission.mjs +124 -0
- package/src/engine/capacity.mjs +78 -0
- package/src/engine/dispatch.mjs +116 -23
- package/src/engine/failure-cause.mjs +59 -0
- package/src/engine/lifecycle.mjs +3 -2
- package/src/engine/phase-session.mjs +56 -10
- package/src/engine/prompts.mjs +57 -2
- package/src/engine/reserve-admission.mjs +99 -0
- package/src/engine/resume.mjs +3 -1
- package/src/engine/review.mjs +21 -0
- package/src/engine/run-command.mjs +166 -11
- package/src/engine/scheduler.mjs +92 -88
- package/src/engine/settle.mjs +15 -8
- package/src/engine/settlement-overlap.mjs +143 -0
- package/src/engine/verify.mjs +131 -7
- package/src/harnesses/session-metrics.mjs +112 -20
- package/src/notify/index.mjs +86 -10
- package/src/plan/freeze.mjs +7 -2
- package/src/plan/human-step.mjs +36 -0
- package/src/plan/pipeline.mjs +53 -22
- package/src/plan/preflight.mjs +81 -5
- package/src/plan/progress.mjs +101 -1
- package/src/plan/repo-facts.mjs +403 -49
- package/src/plan/resolve.mjs +7 -0
- package/src/plan/revise-context.mjs +108 -0
- package/src/plan/rounds.mjs +180 -45
- package/src/plan/sizing.mjs +33 -1
- package/src/plan/template.mjs +134 -22
- package/src/repo/integrate.mjs +47 -5
- package/src/report/campaign-brief-estimate.mjs +47 -1
- package/src/report/campaign-brief.mjs +31 -2
- package/src/report/locale.mjs +20 -0
- package/src/report/message.mjs +13 -1
- package/src/report/metrics-report.mjs +38 -0
- package/src/report/progress.mjs +194 -10
- package/src/report/render.mjs +1 -1
- package/src/report/report-html.mjs +235 -0
- package/src/report/report-video.mjs +323 -0
- package/src/run/usage-windows.mjs +99 -4
- package/src/run/usage.mjs +69 -7
- package/src/web/api.mjs +179 -24
- package/src/web/app.mjs +27 -2
- package/src/web/server.mjs +32 -0
package/src/engine/verify.mjs
CHANGED
|
@@ -13,12 +13,14 @@ import { boundedUtf8, errorMessage } from "../util.mjs";
|
|
|
13
13
|
import { compactVerification } from "../contract/verification.mjs";
|
|
14
14
|
import { MAX_SCOPE_FINDING_PATHS } from "../contract/scope-findings.mjs";
|
|
15
15
|
import { finalVerificationCommands, phaseTerminalNode, sharedVerificationCommands } from "../contract/final-verification.mjs";
|
|
16
|
-
import {
|
|
16
|
+
import { realpathSync } from "node:fs";
|
|
17
|
+
import { join, resolve } from "node:path";
|
|
17
18
|
import { listNodeSnapshots, readNodeSnapshot } from "../run/node-store.mjs";
|
|
18
19
|
import { SETTLED } from "./prompts.mjs";
|
|
19
20
|
import { terminateInvocation } from "./process.mjs";
|
|
20
21
|
import { writeNode } from "./state.mjs";
|
|
21
22
|
import { runVerification } from "./run-command.mjs";
|
|
23
|
+
import { parkDuringProof } from "./settlement-overlap.mjs";
|
|
22
24
|
|
|
23
25
|
/** @typedef {import("../repo/integrate.mjs").CandidateEvidence} CandidateEvidence */
|
|
24
26
|
/** @typedef {import("../cli.mjs").LockHandle} LockHandle */
|
|
@@ -31,6 +33,119 @@ import { runVerification } from "./run-command.mjs";
|
|
|
31
33
|
/** @typedef {import("../contract/index.mjs").VerificationState} VerificationState */
|
|
32
34
|
/** @typedef {{index: number, total: number, argv: string}} VerificationProgress */
|
|
33
35
|
|
|
36
|
+
/**
|
|
37
|
+
* The proof gate: the verification layer's own admission for concurrent
|
|
38
|
+
* passes. R7 (phase 5) allows proofs in independent worktrees to advance
|
|
39
|
+
* together, bounded by the contract's `maxParallel` — the ceiling the planner
|
|
40
|
+
* already sized from this machine, and nothing higher (phase-5 restriction:
|
|
41
|
+
* the host is measured before concurrency is raised). Passes that share a
|
|
42
|
+
* tree can interfere, so they never overlap no matter what the budget allows:
|
|
43
|
+
* the base checkout (`contract.cwd`, where a node without an attempt worktree
|
|
44
|
+
* runs) and the run's one candidate worktree are each serialized under their
|
|
45
|
+
* own key. The settlement overlap (`engine/settlement-overlap.mjs`) is the
|
|
46
|
+
* caller that puts the budget to work: it parks a settlement for the length
|
|
47
|
+
* of each pass, so two nodes' independent worktree proofs overlap through a
|
|
48
|
+
* real run while this gate holds the bound, shared-tree passes, and the
|
|
49
|
+
* candidate worktree's pass serialized.
|
|
50
|
+
*
|
|
51
|
+
* @typedef {{limit: number, active: number, held: Set<string>, waiters: {key: string, satisfy: () => void}[]}} ProofGate
|
|
52
|
+
*/
|
|
53
|
+
|
|
54
|
+
/** @type {WeakMap<ValidatedContract, ProofGate>} */
|
|
55
|
+
const proofGates = new WeakMap();
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* @param {ValidatedContract} contract
|
|
59
|
+
* @returns {ProofGate}
|
|
60
|
+
*/
|
|
61
|
+
function proofGateFor(contract) {
|
|
62
|
+
let gate = proofGates.get(contract);
|
|
63
|
+
if (!gate) {
|
|
64
|
+
gate = { limit: Math.max(1, contract.maxParallel ?? 1), active: 0, held: new Set(), waiters: [] };
|
|
65
|
+
proofGates.set(contract, gate);
|
|
66
|
+
}
|
|
67
|
+
return gate;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* The tree a pass runs in. `realpath` because a worktree path reached through
|
|
72
|
+
* a symlinked parent (/var vs /private/var on darwin) must read as the same
|
|
73
|
+
* key on every call.
|
|
74
|
+
*
|
|
75
|
+
* @param {string} workspace
|
|
76
|
+
* @returns {string}
|
|
77
|
+
*/
|
|
78
|
+
function proofWorkspaceKey(workspace) {
|
|
79
|
+
try {
|
|
80
|
+
return realpathSync(workspace);
|
|
81
|
+
} catch {
|
|
82
|
+
// An absent workspace has no realpath; nothing can run in it anyway, so
|
|
83
|
+
// the lexical resolution is as good a key as any.
|
|
84
|
+
return resolve(workspace);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** @param {ProofGate} gate @returns {void} */
|
|
89
|
+
function drainProofWaiters(gate) {
|
|
90
|
+
for (let index = 0; index < gate.waiters.length && gate.active < gate.limit;) {
|
|
91
|
+
const waiter = /** @type {{key: string, satisfy: () => void}} */ (gate.waiters[index]);
|
|
92
|
+
if (gate.held.has(waiter.key)) {
|
|
93
|
+
index += 1;
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
gate.waiters.splice(index, 1);
|
|
97
|
+
waiter.satisfy();
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* @param {ProofGate} gate
|
|
103
|
+
* @param {string} key
|
|
104
|
+
* @returns {Promise<() => void>}
|
|
105
|
+
*/
|
|
106
|
+
function acquireProofSlot(gate, key) {
|
|
107
|
+
return new Promise((resolveSlot) => {
|
|
108
|
+
/** @returns {void} */
|
|
109
|
+
const satisfy = () => {
|
|
110
|
+
gate.active += 1;
|
|
111
|
+
gate.held.add(key);
|
|
112
|
+
let released = false;
|
|
113
|
+
resolveSlot(() => {
|
|
114
|
+
if (released) return;
|
|
115
|
+
released = true;
|
|
116
|
+
gate.active -= 1;
|
|
117
|
+
gate.held.delete(key);
|
|
118
|
+
drainProofWaiters(gate);
|
|
119
|
+
});
|
|
120
|
+
};
|
|
121
|
+
if (gate.active < gate.limit && !gate.held.has(key)) {
|
|
122
|
+
satisfy();
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
gate.waiters.push({ key, satisfy });
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Run one verification pass holding its proof slot: the pass overlaps other
|
|
131
|
+
* passes only in worktrees independent of its own, and only while the run's
|
|
132
|
+
* budget has room.
|
|
133
|
+
*
|
|
134
|
+
* @template T
|
|
135
|
+
* @param {ValidatedContract} contract
|
|
136
|
+
* @param {string} workspace
|
|
137
|
+
* @param {() => Promise<T>} run
|
|
138
|
+
* @returns {Promise<T>}
|
|
139
|
+
*/
|
|
140
|
+
async function withProofSlot(contract, workspace, run) {
|
|
141
|
+
const release = await acquireProofSlot(proofGateFor(contract), proofWorkspaceKey(workspace));
|
|
142
|
+
try {
|
|
143
|
+
return await run();
|
|
144
|
+
} finally {
|
|
145
|
+
release();
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
34
149
|
/**
|
|
35
150
|
* The sibling phase-terminal node ids already settled, read straight off
|
|
36
151
|
* disk: every node snapshot is persisted from the run's first tick (see
|
|
@@ -147,7 +262,16 @@ export async function executeControllerVerification(contract, runDir, node, stat
|
|
|
147
262
|
/** @param {VerificationAttempt} attempt @returns {VerificationProgress} */
|
|
148
263
|
const progressFor = (attempt) => verificationProgress(attempt.commandIndex + 1, commands.length, /** @type {VerificationCommand|undefined} */ (commands[attempt.commandIndex])?.argv);
|
|
149
264
|
try {
|
|
150
|
-
|
|
265
|
+
// Parked for the length of the pass: the settlement's section slot goes
|
|
266
|
+
// back to the overlap while these child processes run, so a sibling's
|
|
267
|
+
// settlement -- and its own proof -- advances. Everything after this
|
|
268
|
+
// await (judge dispatch, gate verdicts, integration) runs in the
|
|
269
|
+
// re-acquired section; the proof gate below is what bounds the overlap.
|
|
270
|
+
// The candidate pass in `verifyCandidateWorkspace` is deliberately not
|
|
271
|
+
// parked: it runs inside the integration transaction's candidate lock,
|
|
272
|
+
// and parking it would wait to re-acquire a section a sibling holds
|
|
273
|
+
// while that sibling waits on this same candidate lock.
|
|
274
|
+
const result = await parkDuringProof(() => withProofSlot(contract, workspace, () => runVerification(commands, workspace, {
|
|
151
275
|
logDir: join(runDir, "logs", `${node.id}.${state.attempt}.verification`),
|
|
152
276
|
writeFiles: node.taskPacket.writeFiles ?? [],
|
|
153
277
|
onAttemptStart: (attempt) => persistVerificationAttempt(runDir, state, lock, attempt, progressFor(attempt)),
|
|
@@ -156,7 +280,7 @@ export async function executeControllerVerification(contract, runDir, node, stat
|
|
|
156
280
|
...attempt,
|
|
157
281
|
result: boundedVerificationAttemptResult(attempt.result),
|
|
158
282
|
}),
|
|
159
|
-
});
|
|
283
|
+
})));
|
|
160
284
|
state.verification = {
|
|
161
285
|
...compactVerification(result),
|
|
162
286
|
completed: true,
|
|
@@ -318,19 +442,19 @@ export async function verifyCandidateWorkspace(contract, node, state, runDir, wo
|
|
|
318
442
|
const commands = [...node.taskPacket.verification, ...sharedVerificationCommands(contract), ...finalVerificationCommands(contract, node, settledSiblingIds(runDir, contract, node))];
|
|
319
443
|
markCandidateVerification(runDir, state, lock, true);
|
|
320
444
|
try {
|
|
321
|
-
const result = await runVerification(commands, workspace, {
|
|
445
|
+
const result = await withProofSlot(contract, workspace, () => runVerification(commands, workspace, {
|
|
322
446
|
logDir: join(runDir, "logs", `${node.id}.${state.attempt}.candidate-verification`),
|
|
323
447
|
writeFiles: node.taskPacket.writeFiles ?? [],
|
|
324
|
-
});
|
|
448
|
+
}));
|
|
325
449
|
/** @type {import("../contract/verification.mjs").VerificationResult & {retried?: number[]}} */
|
|
326
450
|
let settled = result;
|
|
327
451
|
if (!result.passed) {
|
|
328
452
|
settled = await retryDivergentCandidateCommands(result, state.verification, async (indexes) => {
|
|
329
453
|
const subset = indexes.map((index) => commands[index]);
|
|
330
|
-
const rerun = await runVerification(subset, workspace, {
|
|
454
|
+
const rerun = await withProofSlot(contract, workspace, () => runVerification(subset, workspace, {
|
|
331
455
|
logDir: join(runDir, "logs", `${node.id}.${state.attempt}.candidate-retry`),
|
|
332
456
|
writeFiles: node.taskPacket.writeFiles ?? [],
|
|
333
|
-
});
|
|
457
|
+
}));
|
|
334
458
|
return rerun.commands;
|
|
335
459
|
});
|
|
336
460
|
}
|
|
@@ -11,7 +11,14 @@
|
|
|
11
11
|
import { canonicalUsage, eventItem, extractJson } from "./protocol.mjs";
|
|
12
12
|
import { finite } from "../util.mjs";
|
|
13
13
|
|
|
14
|
-
/** @typedef {{turns: number, cacheReadInputTokens: number, toolCalls: number, completed: boolean, requests: number, contextFirst: number|null, contextMax: number|null, contextLast: number|null, contextSum: number}} SessionTotals */
|
|
14
|
+
/** @typedef {{turns: number, cacheReadInputTokens: number, cacheWriteInputTokens: number|null, outputTokens: number|null, toolCalls: number, completed: boolean, requests: number, contextFirst: number|null, contextMax: number|null, contextLast: number|null, contextSum: number}} SessionTotals */
|
|
15
|
+
/**
|
|
16
|
+
* The five measured counters one invocation proves, mapped from the
|
|
17
|
+
* harness-reported session usage. `bytes` is never stream-derived -- the
|
|
18
|
+
* ledger measures the dispatched prompt file -- so it is always null here;
|
|
19
|
+
* a counter the harness never reported stays null, never a plausible zero.
|
|
20
|
+
* @typedef {{bytes: number|null, inputTokens: number|null, cacheReadInputTokens: number|null, cacheWriteInputTokens: number|null, outputTokens: number|null}} UsageCounters
|
|
21
|
+
*/
|
|
15
22
|
/**
|
|
16
23
|
* The per-request ledger one transcript proves: provider requests that
|
|
17
24
|
* reported usage, and the context each re-sent -- uncached input (cache
|
|
@@ -155,6 +162,34 @@ function sumRequestUsage(usages) {
|
|
|
155
162
|
return inputTokens === null ? { inputTokens: null, cacheReadInputTokens: null } : { inputTokens, cacheReadInputTokens };
|
|
156
163
|
}
|
|
157
164
|
|
|
165
|
+
/** Sum two counters of which either may be unmeasured; null only when neither measured. @param {number|null} left @param {number|null} right @returns {number|null} */
|
|
166
|
+
export function addMeasured(left, right) {
|
|
167
|
+
return left === null && right === null ? null : (left ?? 0) + (right ?? 0);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* The five usage counters one bounded transcript proves: uncached input and
|
|
172
|
+
* cache reads from the observation the live meter reads, cache writes and
|
|
173
|
+
* output from the session totals the parser folds.
|
|
174
|
+
*
|
|
175
|
+
* @param {string} harness
|
|
176
|
+
* @param {string} stdout bounded transcript tail
|
|
177
|
+
* @returns {UsageCounters}
|
|
178
|
+
*/
|
|
179
|
+
export function liveUsageCounters(harness, stdout) {
|
|
180
|
+
const parser = new SessionMetricsParser(harness);
|
|
181
|
+
parser.push(String(stdout));
|
|
182
|
+
parser.flush();
|
|
183
|
+
const usage = liveUsage(harness, stdout);
|
|
184
|
+
return {
|
|
185
|
+
bytes: null,
|
|
186
|
+
inputTokens: usage.inputTokens,
|
|
187
|
+
cacheReadInputTokens: usage.cacheReadInputTokens,
|
|
188
|
+
cacheWriteInputTokens: parser.totals.cacheWriteInputTokens,
|
|
189
|
+
outputTokens: parser.totals.outputTokens,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
158
193
|
/**
|
|
159
194
|
* Budgeted live meter: one number, with cache reads weighted by the campaign
|
|
160
195
|
* policy so it is comparable with the persisted ledger. A provider that only
|
|
@@ -221,6 +256,9 @@ const TOOL_USE_NEEDLE = Buffer.from('"type":"tool_use"', "utf8");
|
|
|
221
256
|
/** Cache-read evidence spellings across harness streams. */
|
|
222
257
|
const CACHE_READ_PATTERN = /"(?:cache_read_input_tokens|cached_input_tokens|cacheReadInputTokens)":(\d+)/gu;
|
|
223
258
|
|
|
259
|
+
/** Cache-write evidence spellings across harness streams. */
|
|
260
|
+
const CACHE_WRITE_PATTERN = /"(?:cache_creation_input_tokens|cacheWriteInputTokens|cache_write_input_tokens|cache_creation_tokens|cache_write_tokens)":(\d+)/gu;
|
|
261
|
+
|
|
224
262
|
/**
|
|
225
263
|
* Bounded incremental session-metrics parser: fold fixed-size chunks into
|
|
226
264
|
* running rotation totals without ever holding a buffer that scales with the
|
|
@@ -240,6 +278,8 @@ export class SessionMetricsParser {
|
|
|
240
278
|
this.totals = {
|
|
241
279
|
turns: previous.turns ?? 0,
|
|
242
280
|
cacheReadInputTokens: previous.cacheReadInputTokens ?? 0,
|
|
281
|
+
cacheWriteInputTokens: null,
|
|
282
|
+
outputTokens: null,
|
|
243
283
|
toolCalls: previous.toolCalls ?? 0,
|
|
244
284
|
completed: previous.completed === true,
|
|
245
285
|
requests: 0,
|
|
@@ -455,7 +495,11 @@ function foldRecord(harness, totals, record) {
|
|
|
455
495
|
if (harness === "codex") {
|
|
456
496
|
if (record.type === "turn.completed") {
|
|
457
497
|
totals.turns += 1;
|
|
458
|
-
|
|
498
|
+
const usage = canonicalUsage(record.usage);
|
|
499
|
+
totals.cacheReadInputTokens = Math.max(totals.cacheReadInputTokens, usage.cacheReadInputTokens ?? 0);
|
|
500
|
+
// Codex counts input and output cumulatively per turn; it reports no
|
|
501
|
+
// cache-write counter at all, so the write total stays unmeasured.
|
|
502
|
+
if (usage.outputTokens !== null) totals.outputTokens = Math.max(totals.outputTokens ?? 0, usage.outputTokens);
|
|
459
503
|
} else if (record.type === "item.completed" && CODEX_TOOL_ITEM_TYPES.has(String(eventItem(record)?.type))) {
|
|
460
504
|
totals.toolCalls += 1;
|
|
461
505
|
}
|
|
@@ -465,14 +509,16 @@ function foldRecord(harness, totals, record) {
|
|
|
465
509
|
if (record.type === "assistant") {
|
|
466
510
|
const message = /** @type {Record<string, unknown>} */ (record.message ?? {});
|
|
467
511
|
totals.turns += 1;
|
|
468
|
-
|
|
469
|
-
|
|
512
|
+
const usage = canonicalUsage(message.usage);
|
|
513
|
+
totals.cacheReadInputTokens += usage.cacheReadInputTokens ?? 0;
|
|
514
|
+
totals.cacheWriteInputTokens = addMeasured(totals.cacheWriteInputTokens, cacheWriteTokensOf(message.usage));
|
|
515
|
+
totals.outputTokens = addMeasured(totals.outputTokens, usage.outputTokens);
|
|
516
|
+
foldRequestUsage(totals, usage);
|
|
470
517
|
totals.toolCalls += Array.isArray(message.content)
|
|
471
518
|
? message.content.filter((/** @type {{type?: unknown}} */ block) => block?.type === "tool_use").length
|
|
472
519
|
: 0;
|
|
473
520
|
} else if (record.type === "result") {
|
|
474
|
-
|
|
475
|
-
if (sessionTotal !== null) totals.cacheReadInputTokens = sessionTotal;
|
|
521
|
+
foldSessionTotal(totals, record.usage);
|
|
476
522
|
}
|
|
477
523
|
return;
|
|
478
524
|
}
|
|
@@ -487,13 +533,15 @@ function foldRecord(harness, totals, record) {
|
|
|
487
533
|
// activity -- three kills on 2026-09-16 at 903s against a 900s limit.
|
|
488
534
|
if (record.type === transcript.usage) {
|
|
489
535
|
totals.turns += 1;
|
|
490
|
-
|
|
491
|
-
|
|
536
|
+
const usage = canonicalUsage(record.usage);
|
|
537
|
+
totals.cacheReadInputTokens += usage.cacheReadInputTokens ?? 0;
|
|
538
|
+
totals.cacheWriteInputTokens = addMeasured(totals.cacheWriteInputTokens, cacheWriteTokensOf(record.usage));
|
|
539
|
+
totals.outputTokens = addMeasured(totals.outputTokens, usage.outputTokens);
|
|
540
|
+
foldRequestUsage(totals, usage);
|
|
492
541
|
} else if (record.type === transcript.tool) {
|
|
493
542
|
totals.toolCalls += 1;
|
|
494
543
|
} else if (record.type === transcript.completed || record.type === transcript.failed) {
|
|
495
|
-
|
|
496
|
-
if (sessionTotal !== null) totals.cacheReadInputTokens = sessionTotal;
|
|
544
|
+
foldSessionTotal(totals, record.usage);
|
|
497
545
|
}
|
|
498
546
|
return;
|
|
499
547
|
}
|
|
@@ -507,32 +555,57 @@ function foldRecord(harness, totals, record) {
|
|
|
507
555
|
if (step) {
|
|
508
556
|
if (step.step_type === "agent_response" && step.state === "DONE") {
|
|
509
557
|
totals.turns += 1;
|
|
510
|
-
|
|
511
|
-
|
|
558
|
+
const usage = canonicalUsage(step.usage);
|
|
559
|
+
totals.cacheReadInputTokens += usage.cacheReadInputTokens ?? 0;
|
|
560
|
+
totals.cacheWriteInputTokens = addMeasured(totals.cacheWriteInputTokens, cacheWriteTokensOf(step.usage));
|
|
561
|
+
totals.outputTokens = addMeasured(totals.outputTokens, usage.outputTokens);
|
|
562
|
+
foldRequestUsage(totals, usage);
|
|
512
563
|
} else if (step.step_type === "tool" && (step.state === "DONE" || step.state === "ERROR")) {
|
|
513
564
|
totals.toolCalls += 1;
|
|
514
565
|
}
|
|
515
566
|
} else if (record.event === "result") {
|
|
516
|
-
|
|
517
|
-
if (sessionTotal !== null) totals.cacheReadInputTokens = sessionTotal;
|
|
567
|
+
foldSessionTotal(totals, agyResult(record)?.usage);
|
|
518
568
|
}
|
|
519
569
|
return;
|
|
520
570
|
}
|
|
521
571
|
if (harness === "exec-jsonl" && record.type === "run.completed") {
|
|
522
572
|
// The protocol carries no tool events; only a completed run proves a turn.
|
|
523
573
|
totals.turns += 1;
|
|
524
|
-
|
|
525
|
-
|
|
574
|
+
const usage = canonicalUsage(record.usage);
|
|
575
|
+
totals.cacheReadInputTokens += usage.cacheReadInputTokens ?? 0;
|
|
576
|
+
totals.cacheWriteInputTokens = addMeasured(totals.cacheWriteInputTokens, cacheWriteTokensOf(record.usage));
|
|
577
|
+
totals.outputTokens = addMeasured(totals.outputTokens, usage.outputTokens);
|
|
578
|
+
foldRequestUsage(totals, usage);
|
|
526
579
|
}
|
|
527
580
|
if (harness === "replay" && typeof record.status === "string") {
|
|
528
581
|
// A replayed envelope is the whole invocation: one completed turn, no
|
|
529
582
|
// tool events, usage only in the terminal record.
|
|
530
583
|
totals.turns += 1;
|
|
531
|
-
|
|
532
|
-
|
|
584
|
+
const usage = canonicalUsage(record.usage);
|
|
585
|
+
totals.cacheReadInputTokens += usage.cacheReadInputTokens ?? 0;
|
|
586
|
+
totals.cacheWriteInputTokens = addMeasured(totals.cacheWriteInputTokens, cacheWriteTokensOf(record.usage));
|
|
587
|
+
totals.outputTokens = addMeasured(totals.outputTokens, usage.outputTokens);
|
|
588
|
+
foldRequestUsage(totals, usage);
|
|
533
589
|
}
|
|
534
590
|
}
|
|
535
591
|
|
|
592
|
+
/** Fold a terminal record's session totals over the mid-run sums; an unreported total leaves the running sum alone. @param {SessionTotals} totals @param {unknown} raw */
|
|
593
|
+
function foldSessionTotal(totals, raw) {
|
|
594
|
+
const usage = canonicalUsage(raw);
|
|
595
|
+
if (usage.cacheReadInputTokens !== null) totals.cacheReadInputTokens = usage.cacheReadInputTokens;
|
|
596
|
+
const writeTotal = cacheWriteTokensOf(raw);
|
|
597
|
+
if (writeTotal !== null) totals.cacheWriteInputTokens = writeTotal;
|
|
598
|
+
if (usage.outputTokens !== null) totals.outputTokens = usage.outputTokens;
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
/** The cache-write tokens one raw usage object reports, across the harness spellings; null when none. @param {unknown} raw @returns {number|null} */
|
|
602
|
+
function cacheWriteTokensOf(raw) {
|
|
603
|
+
if (!raw || typeof raw !== "object") return null;
|
|
604
|
+
const usage = /** @type {Record<string, unknown>} */ (raw);
|
|
605
|
+
return finite(usage.cache_creation_input_tokens ?? usage.cacheWriteInputTokens
|
|
606
|
+
?? usage.cache_write_input_tokens ?? usage.cache_creation_tokens ?? usage.cache_write_tokens);
|
|
607
|
+
}
|
|
608
|
+
|
|
536
609
|
/**
|
|
537
610
|
* Fold one provider request's usage into the per-request ledger. A request
|
|
538
611
|
* with no measured input is not one the ledger can count.
|
|
@@ -575,15 +648,21 @@ function agyResult(record) {
|
|
|
575
648
|
*/
|
|
576
649
|
function foldFragmentRecord(harness, totals, fragments) {
|
|
577
650
|
const text = `${fragments.head}\n${fragments.tail}`;
|
|
651
|
+
// Output tokens have no fragment scanner: a record that outgrew the
|
|
652
|
+
// retention bound leaves the output total unmeasured rather than guessed.
|
|
578
653
|
if (harness === "claude") {
|
|
579
654
|
if (text.includes('"type":"assistant"')) {
|
|
580
655
|
totals.turns += 1;
|
|
581
656
|
totals.toolCalls += fragments.toolUse;
|
|
582
657
|
const cacheRead = lastCacheRead(text);
|
|
583
658
|
if (cacheRead !== null) totals.cacheReadInputTokens += cacheRead;
|
|
659
|
+
const cacheWrite = lastCacheWrite(text);
|
|
660
|
+
if (cacheWrite !== null) totals.cacheWriteInputTokens = addMeasured(totals.cacheWriteInputTokens, cacheWrite);
|
|
584
661
|
} else if (text.includes('"type":"result"')) {
|
|
585
|
-
const
|
|
586
|
-
if (
|
|
662
|
+
const cacheRead = lastCacheRead(text);
|
|
663
|
+
if (cacheRead !== null) totals.cacheReadInputTokens = cacheRead;
|
|
664
|
+
const cacheWrite = lastCacheWrite(text);
|
|
665
|
+
if (cacheWrite !== null) totals.cacheWriteInputTokens = cacheWrite;
|
|
587
666
|
totals.completed = true;
|
|
588
667
|
}
|
|
589
668
|
return;
|
|
@@ -607,6 +686,8 @@ function foldFragmentRecord(harness, totals, fragments) {
|
|
|
607
686
|
totals.turns += 1;
|
|
608
687
|
const cacheRead = lastCacheRead(text);
|
|
609
688
|
if (cacheRead !== null) totals.cacheReadInputTokens += cacheRead;
|
|
689
|
+
const cacheWrite = lastCacheWrite(text);
|
|
690
|
+
if (cacheWrite !== null) totals.cacheWriteInputTokens = addMeasured(totals.cacheWriteInputTokens, cacheWrite);
|
|
610
691
|
totals.completed = true;
|
|
611
692
|
}
|
|
612
693
|
}
|
|
@@ -697,3 +778,14 @@ function lastCacheRead(text) {
|
|
|
697
778
|
const matches = [...text.matchAll(CACHE_READ_PATTERN)];
|
|
698
779
|
return matches.length > 0 ? Number(matches.at(-1)?.[1]) : null;
|
|
699
780
|
}
|
|
781
|
+
|
|
782
|
+
/**
|
|
783
|
+
* The last cache-write number in a fragment text, or null.
|
|
784
|
+
*
|
|
785
|
+
* @param {string} text
|
|
786
|
+
* @returns {number|null}
|
|
787
|
+
*/
|
|
788
|
+
function lastCacheWrite(text) {
|
|
789
|
+
const matches = [...text.matchAll(CACHE_WRITE_PATTERN)];
|
|
790
|
+
return matches.length > 0 ? Number(matches.at(-1)?.[1]) : null;
|
|
791
|
+
}
|
package/src/notify/index.mjs
CHANGED
|
@@ -17,6 +17,18 @@
|
|
|
17
17
|
* fallback. The macOS notifier is reachable only by setting
|
|
18
18
|
* `FABERUN_NOTIFY_BIN=os-macos`, an explicit opt-in, never a default.
|
|
19
19
|
*
|
|
20
|
+
* A transport may also answer with a receipt: the last line it prints, as
|
|
21
|
+
* JSON carrying the `messageId` it created or edited. The campaign watcher's
|
|
22
|
+
* automatic progress update is the consumer -- its text is rendered from the
|
|
23
|
+
* shared projection (`buildCampaignProgress`), never a model, and when a
|
|
24
|
+
* previous progress message of the same campaign is still inside the
|
|
25
|
+
* fifteen-minute window (`progressEditTarget`) the update travels as an edit
|
|
26
|
+
* of that message. The anchor naming the editable message is written only
|
|
27
|
+
* after the transport's receipt returns, so a delivery is claimed only on
|
|
28
|
+
* evidence; a transport that prints no receipt simply always gets a new
|
|
29
|
+
* message. An attention event never consults the anchor: it is always a new
|
|
30
|
+
* message.
|
|
31
|
+
*
|
|
20
32
|
* Measured 2026-09-16: `test/cli/cli.test.mjs`'s two notifier fixtures were
|
|
21
33
|
* instrumented with `{at, phase}` timelines (spawned, stdin-end, exit) and run
|
|
22
34
|
* over 80 times (targeted loops, four-way parallel full-file bursts, and a
|
|
@@ -92,7 +104,7 @@ export const NOTIFY_LANG_ENV = "FABERUN_NOTIFY_LANG";
|
|
|
92
104
|
export const NOTIFY_ENV_NAMES = Object.freeze([NOTIFY_BIN_ENV, NOTIFY_SESSION_ENV, NOTIFY_EVENTS_ENV, NOTIFY_LANG_ENV]);
|
|
93
105
|
|
|
94
106
|
/** Every event type the dispatcher can be asked to deliver. */
|
|
95
|
-
export const NOTIFY_EVENT_TYPES = Object.freeze(["node.terminal", "run.terminal", "attention", "advisory"]);
|
|
107
|
+
export const NOTIFY_EVENT_TYPES = Object.freeze(["node.terminal", "run.terminal", "attention", "advisory", "progress"]);
|
|
96
108
|
|
|
97
109
|
/**
|
|
98
110
|
* What leaves the controller when `FABERUN_NOTIFY_EVENTS` is unset: a phase
|
|
@@ -107,6 +119,30 @@ export const NOTIFY_EVENT_TYPES = Object.freeze(["node.terminal", "run.terminal"
|
|
|
107
119
|
*/
|
|
108
120
|
export const DEFAULT_NOTIFY_EVENTS = Object.freeze(["run.terminal", "attention", "advisory"]);
|
|
109
121
|
|
|
122
|
+
/**
|
|
123
|
+
* How long a delivered progress message stays editable, from the phase spec's
|
|
124
|
+
* constraint: a progress message of the same campaign inside this window is
|
|
125
|
+
* edited in place, an event that demands attention is always a new message.
|
|
126
|
+
*/
|
|
127
|
+
export const PROGRESS_EDIT_WINDOW_MS = 15 * 60_000;
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* The message id an automatic progress update may edit, from the receipt-
|
|
131
|
+
* backed anchor of the last one: an anchor inside the window names its
|
|
132
|
+
* message; an expired, foreign, malformed or absent one edits nothing and the
|
|
133
|
+
* next update is a new message. An attention event never consults this --
|
|
134
|
+
* attention is always a new message.
|
|
135
|
+
*
|
|
136
|
+
* @param {{messageId: string, at: number}|null} anchor the anchor the last receipt wrote
|
|
137
|
+
* @param {number} nowMs epoch milliseconds
|
|
138
|
+
* @returns {string|null}
|
|
139
|
+
*/
|
|
140
|
+
export function progressEditTarget(anchor, nowMs) {
|
|
141
|
+
if (!anchor || typeof anchor.messageId !== "string" || !anchor.messageId) return null;
|
|
142
|
+
if (typeof anchor.at !== "number" || !Number.isFinite(anchor.at)) return null;
|
|
143
|
+
return nowMs - anchor.at < PROGRESS_EDIT_WINDOW_MS ? anchor.messageId : null;
|
|
144
|
+
}
|
|
145
|
+
|
|
110
146
|
/** The two languages the message renders its wording in; `FABERUN_NOTIFY_LANG` may name either. */
|
|
111
147
|
export const NOTIFY_LANGUAGES = Object.freeze(["en", "pt"]);
|
|
112
148
|
|
|
@@ -209,9 +245,9 @@ export const NOTIFY_NO_TRANSPORT_WARNING = "no human notification transport is c
|
|
|
209
245
|
/** @typedef {Record<string, unknown>} JsonObject */
|
|
210
246
|
/** @typedef {{schemaVersion: number, eventId: string, at: string, type: string, campaignId: string|null, runId: string|null, nodeId: string|null, status: string|null, errorCode: string|null, dedupeKey: string, summary: string}} InboxEntry */
|
|
211
247
|
/** @typedef {{type: string, dedupeKey: string, summary: string, at?: string, campaignId?: string|null, runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}} InboxEvent */
|
|
212
|
-
/** @typedef {{type: "node.terminal"|"run.terminal"|"attention"|"advisory", runId: string|null, campaignId?: string|null, nodeId?: string|null, status?: string|null, attempt?: number|null, errorCode?: string|null, done?: number|null, total?: number|null, dedupeKey?: string|null, runDir?: string|null, costUsd?: number|null, summary?: string|null, eventId?: string}} NotifyEvent */
|
|
213
|
-
/** @typedef {{id: string, ok: boolean, error?: string}} TransportOutcome one transport's own outcome, named so the receipt says which took the message */
|
|
214
|
-
/** @typedef {{ok: boolean, error?: string, noTransport?: boolean, transports?: TransportOutcome[]}} DeliveryResult */
|
|
248
|
+
/** @typedef {{type: "node.terminal"|"run.terminal"|"attention"|"advisory"|"progress", runId: string|null, campaignId?: string|null, nodeId?: string|null, status?: string|null, attempt?: number|null, errorCode?: string|null, done?: number|null, total?: number|null, dedupeKey?: string|null, runDir?: string|null, costUsd?: number|null, summary?: string|null, eventId?: string, editOfMessageId?: string|null, artifacts?: {kind: "html"|"video", path: string}[]}} NotifyEvent */
|
|
249
|
+
/** @typedef {{id: string, ok: boolean, error?: string, messageId?: string}} TransportOutcome one transport's own outcome, named so the receipt says which took the message; `messageId` is the id a transport receipt named for the message it created or edited */
|
|
250
|
+
/** @typedef {{ok: boolean, error?: string, noTransport?: boolean, messageId?: string, transports?: TransportOutcome[]}} DeliveryResult */
|
|
215
251
|
|
|
216
252
|
/**
|
|
217
253
|
* Render the message for an event. For `node.terminal`, `run.terminal` and
|
|
@@ -219,7 +255,10 @@ export const NOTIFY_NO_TRANSPORT_WARNING = "no human notification transport is c
|
|
|
219
255
|
* own persisted state (its contract, its node snapshots, its usage) from
|
|
220
256
|
* `event.runDir` -- so whatever calls this function once gets exactly the
|
|
221
257
|
* string every audience of that event sees: the inbox entry's `summary` and
|
|
222
|
-
* the transport's stdin are never rendered separately and never drift.
|
|
258
|
+
* the transport's stdin are never rendered separately and never drift. A
|
|
259
|
+
* `progress` event is never rendered here: the campaign watcher renders it
|
|
260
|
+
* from the shared projection and passes it pre-rendered, so a progress event
|
|
261
|
+
* without a summary is a caller bug and throws.
|
|
223
262
|
*
|
|
224
263
|
* `runDir` is absent, or names a directory with no readable run scaffold
|
|
225
264
|
* (no `contract.json`, no `run.json`, a node snapshot that fails validation),
|
|
@@ -438,13 +477,16 @@ export function appendInbox(runsDir, event) {
|
|
|
438
477
|
* text. The result is `ok` when any one of them took the message, carries
|
|
439
478
|
* every transport's own outcome for the receipt, and is
|
|
440
479
|
* `{ok: false, noTransport: true}` when nothing at all is bound -- without
|
|
441
|
-
* spawning or connecting anything.
|
|
480
|
+
* spawning or connecting anything. When a transport answered with a receipt
|
|
481
|
+
* naming the message it created or edited, that id is surfaced as `messageId`
|
|
482
|
+
* so the caller can claim exactly what the transport acknowledged -- never
|
|
483
|
+
* more.
|
|
442
484
|
*
|
|
443
485
|
* @param {{type: string, summary: string, campaignId?: string|null, [key: string]: unknown}} event
|
|
444
486
|
* @param {{bin?: string, env?: NodeJS.ProcessEnv, spawn?: typeof defaultSpawn, timeoutMs?: number}} [options]
|
|
445
487
|
* @returns {Promise<DeliveryResult>}
|
|
446
488
|
*/
|
|
447
|
-
async function deliverNotification(event, options = {}) {
|
|
489
|
+
export async function deliverNotification(event, options = {}) {
|
|
448
490
|
const env = options.env ?? process.env;
|
|
449
491
|
const bin = options.bin ?? env[NOTIFY_BIN_ENV];
|
|
450
492
|
const targets = resolveSessionTargets(env);
|
|
@@ -461,20 +503,50 @@ async function deliverNotification(event, options = {}) {
|
|
|
461
503
|
if (!attempts.length && !targets.length) return { ok: false, noTransport: true, transports: [] };
|
|
462
504
|
const transports = [...(await Promise.all(attempts)), ...(await sessions)];
|
|
463
505
|
const failures = transports.filter((outcome) => !outcome.ok).map((outcome) => `${outcome.id}: ${outcome.error ?? "failed"}`);
|
|
506
|
+
const messageId = transports.map((outcome) => outcome.messageId).find((id) => typeof id === "string" && id !== "");
|
|
464
507
|
return {
|
|
465
508
|
ok: transports.some((outcome) => outcome.ok),
|
|
509
|
+
...(messageId ? { messageId } : {}),
|
|
466
510
|
...(failures.length ? { error: failures.join("; ") } : {}),
|
|
467
511
|
transports,
|
|
468
512
|
};
|
|
469
513
|
}
|
|
470
514
|
|
|
515
|
+
/**
|
|
516
|
+
* The transport's delivery receipt, read from the last non-empty line it
|
|
517
|
+
* printed: a JSON object whose `messageId` names the message it created or
|
|
518
|
+
* edited. A transport that prints nothing else, prints non-JSON, or exits
|
|
519
|
+
* non-zero yields no receipt, and the caller claims no editable message from
|
|
520
|
+
* it.
|
|
521
|
+
*
|
|
522
|
+
* @param {string} stdout
|
|
523
|
+
* @returns {{messageId?: string}}
|
|
524
|
+
*/
|
|
525
|
+
function transportReceipt(stdout) {
|
|
526
|
+
const lines = stdout.split("\n");
|
|
527
|
+
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
|
528
|
+
const trimmed = lines[index].trim();
|
|
529
|
+
if (!trimmed) continue;
|
|
530
|
+
try {
|
|
531
|
+
const parsed = JSON.parse(trimmed);
|
|
532
|
+
const messageId = parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed.messageId : undefined;
|
|
533
|
+
return typeof messageId === "string" && messageId ? { messageId } : {};
|
|
534
|
+
} catch {
|
|
535
|
+
return {};
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
return {};
|
|
539
|
+
}
|
|
540
|
+
|
|
471
541
|
/**
|
|
472
542
|
* Resolves on the child's own `exit`, not `close`: `close` waits for every
|
|
473
543
|
* stdio stream to finish closing, and a piped `stderr` fd can be held open
|
|
474
544
|
* by a lingering grandchild or be slow to flush under load well after the
|
|
475
545
|
* notifier process itself has terminated. Gating delivery on that stream
|
|
476
546
|
* closing (as the previous implementation did) could stall a healthy,
|
|
477
|
-
* already-exited delivery until the timeout fired.
|
|
547
|
+
* already-exited delivery until the timeout fired. A zero exit is the
|
|
548
|
+
* delivery; whatever the transport printed on stdout is its receipt, read
|
|
549
|
+
* by `transportReceipt`.
|
|
478
550
|
*
|
|
479
551
|
* @param {string} bin
|
|
480
552
|
* @param {JsonObject} event
|
|
@@ -490,7 +562,7 @@ function spawnDeliver(bin, event, { spawn = defaultSpawn, timeoutMs = notificati
|
|
|
490
562
|
// names its interpreter. Either spawns as EFTYPE unhandled, which is a
|
|
491
563
|
// notification that silently never arrives.
|
|
492
564
|
const invocation = spawnInvocation(bin, []);
|
|
493
|
-
child = spawn(invocation.command, invocation.args, { stdio: ["pipe", "
|
|
565
|
+
child = spawn(invocation.command, invocation.args, { stdio: ["pipe", "pipe", "pipe"], env: process.env, ...invocation.options });
|
|
494
566
|
} catch (error) {
|
|
495
567
|
resolveDelivery({ ok: false, error: errorMessage(error) });
|
|
496
568
|
return;
|
|
@@ -504,11 +576,15 @@ function spawnDeliver(bin, event, { spawn = defaultSpawn, timeoutMs = notificati
|
|
|
504
576
|
resolveDelivery(result);
|
|
505
577
|
};
|
|
506
578
|
let stderr = "";
|
|
579
|
+
let stdout = "";
|
|
507
580
|
child.stderr?.on("data", (chunk) => {
|
|
508
581
|
stderr = `${stderr}${chunk}`.slice(-1024);
|
|
509
582
|
});
|
|
583
|
+
child.stdout?.on("data", (chunk) => {
|
|
584
|
+
stdout = `${stdout}${chunk}`.slice(-2048);
|
|
585
|
+
});
|
|
510
586
|
child.once("error", (error) => finish({ ok: false, error: errorMessage(error) }));
|
|
511
|
-
child.once("exit", (code) => finish(code === 0 ? { ok: true } : { ok: false, error: stderr || `notification exited ${code}` }));
|
|
587
|
+
child.once("exit", (code) => finish(code === 0 ? { ok: true, ...transportReceipt(stdout) } : { ok: false, error: stderr || `notification exited ${code}` }));
|
|
512
588
|
const timer = setTimeout(() => {
|
|
513
589
|
try {
|
|
514
590
|
child.kill("SIGTERM");
|
package/src/plan/freeze.mjs
CHANGED
|
@@ -35,7 +35,8 @@ import { NODE_VALUE_OPTIONS, assertFilteredProofsNameTheirTest } from "./proof-s
|
|
|
35
35
|
/** @typedef {{path: string, digest: string}} PlanSpecIdentity */
|
|
36
36
|
/** @typedef {{formatVersion: number, contractDigest: string, spec?: PlanSpecIdentity, phases?: PlanPhase[], humanSteps?: PlanHumanStep[], provenance: PlanProvenance}} FrozenPlan */
|
|
37
37
|
/** @typedef {{ok: boolean, digest: string, expectedDigest: string}} FrozenPlanVerdict */
|
|
38
|
-
/** @typedef {{
|
|
38
|
+
/** @typedef {{argv: string[], measuredMs: number|null, eligible: boolean}} MeasuredCandidate */
|
|
39
|
+
/** @typedef {{scripts?: Record<string, string>, verificationCandidates: MeasuredCandidate[]}} MeasuredFacts */
|
|
39
40
|
|
|
40
41
|
const PLAN_FORMAT_VERSION = 1;
|
|
41
42
|
|
|
@@ -74,13 +75,17 @@ function includedDirectories(arg, directories) {
|
|
|
74
75
|
* the sum of the `node --test <dir>` candidates it includes, `npm test`
|
|
75
76
|
* resolved through the `test` script. A lower bound when the command also
|
|
76
77
|
* runs files no candidate measured; null when it includes nothing measured.
|
|
78
|
+
* Only an eligible candidate with a numeric duration counts, exact or
|
|
79
|
+
* aggregated: a candidate killed at its timeout persists measuredMs null
|
|
80
|
+
* with eligible false, so neither an exact read nor a directory sum may
|
|
81
|
+
* plan from it.
|
|
77
82
|
*
|
|
78
83
|
* @param {string[]} argv
|
|
79
84
|
* @param {MeasuredFacts} facts
|
|
80
85
|
* @returns {number|null}
|
|
81
86
|
*/
|
|
82
87
|
function measuredMsFor(argv, facts) {
|
|
83
|
-
const candidates = facts.verificationCandidates;
|
|
88
|
+
const candidates = facts.verificationCandidates.filter((candidate) => candidate.eligible === true && typeof candidate.measuredMs === "number");
|
|
84
89
|
const exact = candidates.find((candidate) => candidate.argv.join(" ") === argv.join(" "));
|
|
85
90
|
if (exact) return exact.measuredMs;
|
|
86
91
|
const npmTest = argv[0] === "npm" && ["test", "run test"].includes(argv.slice(1).join(" "));
|