faberun 0.3.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +152 -100
- package/package.json +10 -2
- package/skills/faberun/SKILL.md +6 -5
- package/skills/faberun/references/contract.md +23 -11
- package/skills/faberun/references/engineering.md +3 -1
- package/skills/faberun/references/operations.md +19 -12
- package/skills/faberun/references/rules.md +3 -1
- package/src/campaign/chain.mjs +6 -2
- package/src/campaign/index.mjs +17 -1
- package/src/campaign/metrics.mjs +3 -3
- package/src/cli/brand.mjs +2 -1
- package/src/cli/campaign.mjs +2 -0
- package/src/cli/contract.mjs +2 -0
- package/src/cli/manual.mjs +341 -0
- package/src/cli/seat.mjs +2 -0
- package/src/cli/setup.mjs +109 -30
- package/src/cli/skills.mjs +310 -8
- package/src/cli.mjs +3 -2
- package/src/contract/final-verification.mjs +31 -2
- package/src/contract/index.mjs +28 -25
- package/src/contract/runtime.mjs +5 -1
- package/src/contract/snapshot.mjs +7 -1
- package/src/contract/task-packet.mjs +20 -9
- package/src/contract/verification.mjs +1 -1
- package/src/engine/backoff.mjs +1 -1
- package/src/engine/dispatch.mjs +31 -4
- package/src/engine/gate.mjs +12 -0
- package/src/engine/process-identity.mjs +39 -0
- package/src/engine/prompts.mjs +18 -0
- package/src/engine/resume.mjs +2 -2
- package/src/engine/review.mjs +9 -1
- package/src/engine/run-command.mjs +23 -2
- package/src/engine/run-identity.mjs +14 -0
- package/src/engine/scheduler.mjs +45 -12
- package/src/engine/settle.mjs +29 -0
- package/src/engine/supervise.mjs +32 -6
- package/src/engine/verify.mjs +98 -9
- package/src/harnesses/agy/index.mjs +3 -0
- package/src/harnesses/claude/index.mjs +5 -0
- package/src/harnesses/codex/index.mjs +3 -0
- package/src/harnesses/dsh/index.mjs +26 -0
- package/src/harnesses/exec-jsonl/index.mjs +2 -0
- package/src/harnesses/index.mjs +10 -3
- package/src/harnesses/replay/index.mjs +2 -0
- package/src/harnesses/zcode/index.mjs +3 -0
- package/src/host/preflight.mjs +5 -1
- package/src/notify/index.mjs +45 -2
- package/src/repo/source-identity.mjs +4 -3
- package/src/report/final.mjs +3 -2
- package/src/report/render.mjs +134 -51
- package/src/web/index.html +1 -1
package/src/engine/verify.mjs
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
import { attemptWorkspace } from "../repo/worktree.mjs";
|
|
12
12
|
import { boundedUtf8, errorMessage } from "../util.mjs";
|
|
13
13
|
import { compactVerification } from "../contract/verification.mjs";
|
|
14
|
-
import { finalVerificationCommands } from "../contract/final-verification.mjs";
|
|
14
|
+
import { finalVerificationCommands, sharedVerificationCommands } from "../contract/final-verification.mjs";
|
|
15
15
|
import { join } from "node:path";
|
|
16
16
|
import { terminateInvocation } from "./process.mjs";
|
|
17
17
|
import { writeNode } from "./state.mjs";
|
|
@@ -24,8 +24,24 @@ import { runVerification } from "./run-command.mjs";
|
|
|
24
24
|
/** @typedef {import("../contract/index.mjs").ValidatedNode} ValidatedNode */
|
|
25
25
|
/** @typedef {import("../contract/verification.mjs").VerificationAttempt} VerificationAttempt */
|
|
26
26
|
/** @typedef {import("../contract/verification.mjs").VerificationAttemptResult} VerificationAttemptResult */
|
|
27
|
+
/** @typedef {import("../contract/verification.mjs").VerificationCommand} VerificationCommand */
|
|
27
28
|
/** @typedef {import("../contract/index.mjs").VerificationState} VerificationState */
|
|
29
|
+
/** @typedef {{index: number, total: number, argv: string}} VerificationProgress */
|
|
28
30
|
|
|
31
|
+
/**
|
|
32
|
+
* The bounded `k/n · argv` shape a running node's status surfaces while a
|
|
33
|
+
* verification command is in flight: the command's 1-based position among
|
|
34
|
+
* every command this pass runs, and its argv joined and bounded so a long
|
|
35
|
+
* command line can never threaten the node snapshot's byte ceiling.
|
|
36
|
+
*
|
|
37
|
+
* @param {number} index 1-based position of the command now running
|
|
38
|
+
* @param {number} total command count in this verification pass
|
|
39
|
+
* @param {string[]|undefined} argv
|
|
40
|
+
* @returns {VerificationProgress}
|
|
41
|
+
*/
|
|
42
|
+
export function verificationProgress(index, total, argv) {
|
|
43
|
+
return { index, total, argv: boundedUtf8((argv ?? []).join(" "), 120) };
|
|
44
|
+
}
|
|
29
45
|
/**
|
|
30
46
|
* @param {VerificationAttemptResult|null|undefined} result
|
|
31
47
|
* @returns {VerificationAttemptResult}
|
|
@@ -57,14 +73,21 @@ function verificationAttemptRecords(state) {
|
|
|
57
73
|
* @param {NodeSnapshot} state
|
|
58
74
|
* @param {LockHandle} lock
|
|
59
75
|
* @param {VerificationAttempt} attempt
|
|
76
|
+
* @param {VerificationProgress} [progress] the running command's `k/n · argv`
|
|
77
|
+
* shape; omitted on completion, since a following `onAttemptStart` replaces
|
|
78
|
+
* it or the pass's final rewrite of `state.verification` drops it
|
|
60
79
|
*/
|
|
61
|
-
function persistVerificationAttempt(runDir, state, lock, attempt) {
|
|
80
|
+
function persistVerificationAttempt(runDir, state, lock, attempt, progress) {
|
|
62
81
|
const attempts = verificationAttemptRecords(state);
|
|
63
82
|
const index = attempts.findIndex((item) => item.invocationId === attempt.invocationId);
|
|
64
83
|
if (index >= 0) attempts[index] = { ...attempts[index], ...attempt };
|
|
65
84
|
else attempts.push({ ...attempt, completedAt: attempt.completedAt ?? null, result: attempt.result ?? null });
|
|
66
85
|
state.verification ??= { passed: false, commands: [], completed: false, attempts: [] };
|
|
67
86
|
state.verification.attempts = attempts.slice(-16);
|
|
87
|
+
if (progress) {
|
|
88
|
+
/** @type {VerificationState & {progress?: VerificationProgress}} */
|
|
89
|
+
(state.verification).progress = progress;
|
|
90
|
+
}
|
|
68
91
|
writeNode(runDir, state, lock);
|
|
69
92
|
}
|
|
70
93
|
/**
|
|
@@ -85,12 +108,15 @@ export async function executeControllerVerification(contract, runDir, node, stat
|
|
|
85
108
|
};
|
|
86
109
|
writeNode(runDir, state, lock);
|
|
87
110
|
const workspace = attemptWorkspace(state) ?? contract.cwd;
|
|
111
|
+
const commands = [...node.taskPacket.verification, ...sharedVerificationCommands(contract), ...finalVerificationCommands(contract, node)];
|
|
112
|
+
/** @param {VerificationAttempt} attempt @returns {VerificationProgress} */
|
|
113
|
+
const progressFor = (attempt) => verificationProgress(attempt.commandIndex + 1, commands.length, /** @type {VerificationCommand|undefined} */ (commands[attempt.commandIndex])?.argv);
|
|
88
114
|
try {
|
|
89
|
-
const result = await runVerification(
|
|
115
|
+
const result = await runVerification(commands, workspace, {
|
|
90
116
|
logDir: join(runDir, "logs", `${node.id}.${state.attempt}.verification`),
|
|
91
117
|
writeFiles: node.taskPacket.writeFiles ?? [],
|
|
92
|
-
onAttemptStart: (attempt) => persistVerificationAttempt(runDir, state, lock, attempt),
|
|
93
|
-
onAttemptSpawn: (attempt) => persistVerificationAttempt(runDir, state, lock, attempt),
|
|
118
|
+
onAttemptStart: (attempt) => persistVerificationAttempt(runDir, state, lock, attempt, progressFor(attempt)),
|
|
119
|
+
onAttemptSpawn: (attempt) => persistVerificationAttempt(runDir, state, lock, attempt, progressFor(attempt)),
|
|
94
120
|
onAttemptComplete: (attempt) => persistVerificationAttempt(runDir, state, lock, {
|
|
95
121
|
...attempt,
|
|
96
122
|
result: boundedVerificationAttemptResult(attempt.result),
|
|
@@ -102,10 +128,13 @@ export async function executeControllerVerification(contract, runDir, node, stat
|
|
|
102
128
|
attempts: verificationAttemptRecords(state),
|
|
103
129
|
};
|
|
104
130
|
} catch (error) {
|
|
131
|
+
// A rebuilt object, not a spread of the prior one: a thrown error can land
|
|
132
|
+
// between an `onAttemptStart` and the matching `onAttemptComplete`, and the
|
|
133
|
+
// stale `progress` that start wrote must not survive into the terminal record.
|
|
105
134
|
state.verification = {
|
|
106
|
-
...state.verification,
|
|
107
|
-
completed: true,
|
|
108
135
|
passed: false,
|
|
136
|
+
commands: state.verification?.commands ?? [],
|
|
137
|
+
completed: true,
|
|
109
138
|
error: boundedUtf8(errorMessage(error), 4 * 1024),
|
|
110
139
|
attempts: verificationAttemptRecords(state),
|
|
111
140
|
};
|
|
@@ -146,6 +175,52 @@ export async function recoverVerificationAttempts(runDir, state, lock) {
|
|
|
146
175
|
delete state.verification.error;
|
|
147
176
|
writeNode(runDir, state, lock);
|
|
148
177
|
}
|
|
178
|
+
/**
|
|
179
|
+
* Re-run, once, exactly the candidate commands that failed here but passed in
|
|
180
|
+
* the attempt's own recorded verification, same argv and same position. A
|
|
181
|
+
* candidate whose *every* failure disagrees with the attempt this way is
|
|
182
|
+
* evidence about the two worktrees' environment rather than about the work --
|
|
183
|
+
* `candidateOnlyFailures` (judge-gate.mjs) already names that disagreement in
|
|
184
|
+
* the rejection it phrases, and this is what earns the candidate one
|
|
185
|
+
* independent confirmation before the node pays for a defect that may not be
|
|
186
|
+
* its own. A candidate with even one failure that also failed in the attempt
|
|
187
|
+
* is not purely divergent, so nothing is retried and the failure stands.
|
|
188
|
+
*
|
|
189
|
+
* `run` is the one side-effecting seam, injected so this stays unit-testable
|
|
190
|
+
* without a workspace or a git repository.
|
|
191
|
+
*
|
|
192
|
+
* @param {import("../contract/verification.mjs").VerificationResult} result the candidate's verification result
|
|
193
|
+
* @param {{commands?: Array<{argv?: string[], passed?: boolean}>}|null|undefined} attemptEvidence the attempt's own recorded verification
|
|
194
|
+
* @param {(indexes: number[]) => Promise<import("../contract/verification.mjs").VerificationCommandResult[]>} run re-runs exactly the commands at `indexes`, returning their results in that order
|
|
195
|
+
* @returns {Promise<import("../contract/verification.mjs").VerificationResult & {retried?: number[]}>}
|
|
196
|
+
*/
|
|
197
|
+
export async function retryDivergentCandidateCommands(result, attemptEvidence, run) {
|
|
198
|
+
const commands = result?.commands ?? [];
|
|
199
|
+
const failedIndexes = commands.reduce((indexes, command, index) => {
|
|
200
|
+
if (!command.passed) indexes.push(index);
|
|
201
|
+
return indexes;
|
|
202
|
+
}, /** @type {number[]} */ ([]));
|
|
203
|
+
if (!failedIndexes.length) return result;
|
|
204
|
+
const attemptCommands = attemptEvidence?.commands ?? [];
|
|
205
|
+
const divergent = failedIndexes.every((index) => {
|
|
206
|
+
const counterpart = attemptCommands[index];
|
|
207
|
+
return counterpart?.passed === true && argvEqual(commands[index]?.argv, counterpart.argv);
|
|
208
|
+
});
|
|
209
|
+
if (!divergent) return result;
|
|
210
|
+
const retried = await run(failedIndexes);
|
|
211
|
+
const merged = [...commands];
|
|
212
|
+
failedIndexes.forEach((index, position) => { merged[index] = retried[position]; });
|
|
213
|
+
return { ...result, commands: merged, passed: merged.every((command) => command.passed), retried: failedIndexes };
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* @param {unknown} a
|
|
217
|
+
* @param {unknown} b
|
|
218
|
+
* @returns {boolean}
|
|
219
|
+
*/
|
|
220
|
+
function argvEqual(a, b) {
|
|
221
|
+
if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false;
|
|
222
|
+
return a.every((item, index) => item === b[index]);
|
|
223
|
+
}
|
|
149
224
|
/**
|
|
150
225
|
* @param {ValidatedContract} contract
|
|
151
226
|
* @param {ValidatedNode} node
|
|
@@ -155,12 +230,26 @@ export async function recoverVerificationAttempts(runDir, state, lock) {
|
|
|
155
230
|
* @returns {Promise<import("../repo/integrate.mjs").CandidateEvidence>}
|
|
156
231
|
*/
|
|
157
232
|
export async function verifyCandidateWorkspace(contract, node, state, runDir, workspace) {
|
|
233
|
+
const commands = [...node.taskPacket.verification, ...sharedVerificationCommands(contract), ...finalVerificationCommands(contract, node)];
|
|
158
234
|
try {
|
|
159
|
-
const result = await runVerification(
|
|
235
|
+
const result = await runVerification(commands, workspace, {
|
|
160
236
|
logDir: join(runDir, "logs", `${node.id}.${state.attempt}.candidate-verification`),
|
|
161
237
|
writeFiles: node.taskPacket.writeFiles ?? [],
|
|
162
238
|
});
|
|
163
|
-
|
|
239
|
+
/** @type {import("../contract/verification.mjs").VerificationResult & {retried?: number[]}} */
|
|
240
|
+
let settled = result;
|
|
241
|
+
if (!result.passed) {
|
|
242
|
+
settled = await retryDivergentCandidateCommands(result, state.verification, async (indexes) => {
|
|
243
|
+
const subset = indexes.map((index) => commands[index]);
|
|
244
|
+
const rerun = await runVerification(subset, workspace, {
|
|
245
|
+
logDir: join(runDir, "logs", `${node.id}.${state.attempt}.candidate-retry`),
|
|
246
|
+
writeFiles: node.taskPacket.writeFiles ?? [],
|
|
247
|
+
});
|
|
248
|
+
return rerun.commands;
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
const compacted = compactVerification(settled);
|
|
252
|
+
return settled.retried ? { ...compacted, retried: settled.retried } : compacted;
|
|
164
253
|
} catch (error) {
|
|
165
254
|
return { passed: false, error: boundedUtf8(errorMessage(error), 4 * 1024) };
|
|
166
255
|
}
|
|
@@ -19,6 +19,9 @@ export const agyHarness = {
|
|
|
19
19
|
// `--output-format=stream-json` writes one JSON line per event as the
|
|
20
20
|
// turn runs, not one dump at exit.
|
|
21
21
|
streamsOutput: true,
|
|
22
|
+
// Unmeasured: no run has proven whether agy's sandbox can signal child
|
|
23
|
+
// processes or read the process table.
|
|
24
|
+
signalsProcesses: null,
|
|
22
25
|
},
|
|
23
26
|
|
|
24
27
|
// command() always passes --dangerously-skip-permissions.
|
|
@@ -45,6 +45,11 @@ export const claudeHarness = {
|
|
|
45
45
|
// `--output-format stream-json --verbose` writes one JSON line per event
|
|
46
46
|
// as the turn runs, not one dump at exit.
|
|
47
47
|
streamsOutput: true,
|
|
48
|
+
// Measured 2026-09-16: `bypassPermissions` runs an unsandboxed Bash that
|
|
49
|
+
// signals child processes and reads the process table; headless
|
|
50
|
+
// `acceptEdits` cannot run commands at all, so the flag describes the
|
|
51
|
+
// executing mode.
|
|
52
|
+
signalsProcesses: true,
|
|
48
53
|
},
|
|
49
54
|
|
|
50
55
|
// Headless acceptEdits denies Bash; bypassPermissions executes commands.
|
|
@@ -41,6 +41,9 @@ export const codexHarness = {
|
|
|
41
41
|
// `--json` writes one JSONL event per item/turn as it happens, not one
|
|
42
42
|
// dump at exit.
|
|
43
43
|
streamsOutput: true,
|
|
44
|
+
// Unmeasured: no run has proven whether codex's sandbox can signal child
|
|
45
|
+
// processes or read the process table.
|
|
46
|
+
signalsProcesses: null,
|
|
44
47
|
},
|
|
45
48
|
|
|
46
49
|
// Every sandbox mode executes commands; sandbox only bounds their effects.
|
|
@@ -51,6 +51,11 @@ export const dshHarness = {
|
|
|
51
51
|
// deepseek-official/deepseek-flash grew the redirected stdout file from
|
|
52
52
|
// 67 to 1,380 to 1,441 to 2,084 bytes across a 14s turn.
|
|
53
53
|
streamsOutput: true,
|
|
54
|
+
// Measured 2026-09-16 in the controller's workspace-write sandbox: a test
|
|
55
|
+
// that starts and terminates a child process cannot signal it or read the
|
|
56
|
+
// process table, so it hangs until the executor's cap. `danger-full-access`
|
|
57
|
+
// was not measured.
|
|
58
|
+
signalsProcesses: false,
|
|
54
59
|
},
|
|
55
60
|
|
|
56
61
|
// sandbox maps to DSH_PERMISSION_MODE, which is the harness's file-effect
|
|
@@ -98,6 +103,7 @@ export const dshHarness = {
|
|
|
98
103
|
args,
|
|
99
104
|
promptTransport: "stdin",
|
|
100
105
|
input: withSchema(prompt, options.schema),
|
|
106
|
+
env: dshEnvironmentOverlay(process.env),
|
|
101
107
|
};
|
|
102
108
|
},
|
|
103
109
|
|
|
@@ -156,6 +162,26 @@ export const dshHarness = {
|
|
|
156
162
|
},
|
|
157
163
|
};
|
|
158
164
|
|
|
165
|
+
/**
|
|
166
|
+
* The environment overlay every dsh turn spawns with. The Claude Code shell
|
|
167
|
+
* exports `GIT_CONFIG_COUNT` with the VALUE half of each `GIT_CONFIG_{KEY,VALUE}_<n>`
|
|
168
|
+
* pair but not the key half, so `git init` inside the worker fails with status
|
|
169
|
+
* 128; the overlay removes the whole family. `GIT_TERMINAL_PROMPT=0` takes over
|
|
170
|
+
* the prompting that family was for. A null value removes the ambient variable
|
|
171
|
+
* when the gate merges the overlay over the runner environment.
|
|
172
|
+
*
|
|
173
|
+
* @param {NodeJS.ProcessEnv} env
|
|
174
|
+
* @returns {Record<string, string|null>}
|
|
175
|
+
*/
|
|
176
|
+
function dshEnvironmentOverlay(env) {
|
|
177
|
+
/** @type {Record<string, string|null>} */
|
|
178
|
+
const overlay = { GIT_TERMINAL_PROMPT: "0" };
|
|
179
|
+
for (const key of Object.keys(env)) {
|
|
180
|
+
if (key === "GIT_CONFIG_COUNT" || /^GIT_CONFIG_(?:KEY|VALUE)_\d+$/u.test(key)) overlay[key] = null;
|
|
181
|
+
}
|
|
182
|
+
return overlay;
|
|
183
|
+
}
|
|
184
|
+
|
|
159
185
|
/**
|
|
160
186
|
* Append the output schema the judge prompt refers to. Codex and Claude receive
|
|
161
187
|
* it through a native flag; this harness has none, so it travels in the prompt.
|
|
@@ -74,6 +74,8 @@ export const execJsonlHarness = {
|
|
|
74
74
|
// The protocol allows zero `message` events before the terminal one, so
|
|
75
75
|
// an arbitrary wrapper cannot honestly advertise incremental output either.
|
|
76
76
|
streamsOutput: false,
|
|
77
|
+
// Unmeasured: an arbitrary wrapper executable names no sandbox to measure.
|
|
78
|
+
signalsProcesses: null,
|
|
77
79
|
},
|
|
78
80
|
|
|
79
81
|
// The wrapper protocol exposes no permission mode.
|
package/src/harnesses/index.mjs
CHANGED
|
@@ -35,13 +35,14 @@ const CAPABILITY_NAMES = new Set([
|
|
|
35
35
|
"cost",
|
|
36
36
|
"toolPolicy",
|
|
37
37
|
"streamsOutput",
|
|
38
|
+
"signalsProcesses",
|
|
38
39
|
]);
|
|
39
40
|
|
|
40
|
-
/** @typedef {"structuredOutput"|"promptTransport"|"sandbox"|"permissions"|"continuation"|"tokenBudget"|"costBudget"|"usage"|"cost"|"toolPolicy"|"streamsOutput"} CapabilityName */
|
|
41
|
+
/** @typedef {"structuredOutput"|"promptTransport"|"sandbox"|"permissions"|"continuation"|"tokenBudget"|"costBudget"|"usage"|"cost"|"toolPolicy"|"streamsOutput"|"signalsProcesses"} CapabilityName */
|
|
41
42
|
|
|
42
|
-
/** @typedef {{structuredOutput: boolean, promptTransport: "stdin"|"argv", sandbox: boolean, permissions: boolean, continuation: boolean, tokenBudget: boolean, costBudget: boolean, usage: boolean, cost: boolean, toolPolicy: boolean, streamsOutput: boolean, maxArgvPromptBytes?: number}} HarnessCapabilities */
|
|
43
|
+
/** @typedef {{structuredOutput: boolean, promptTransport: "stdin"|"argv", sandbox: boolean, permissions: boolean, continuation: boolean, tokenBudget: boolean, costBudget: boolean, usage: boolean, cost: boolean, toolPolicy: boolean, streamsOutput: boolean, signalsProcesses: boolean|null, maxArgvPromptBytes?: number}} HarnessCapabilities */
|
|
43
44
|
|
|
44
|
-
/** @typedef {{structuredOutput?: boolean, promptTransport?: "stdin"|"argv", sandbox?: boolean, permissions?: boolean, continuation?: boolean, tokenBudget?: boolean, costBudget?: boolean, usage?: boolean, cost?: boolean, toolPolicy?: boolean, streamsOutput?: boolean}} CapabilityRequirements */
|
|
45
|
+
/** @typedef {{structuredOutput?: boolean, promptTransport?: "stdin"|"argv", sandbox?: boolean, permissions?: boolean, continuation?: boolean, tokenBudget?: boolean, costBudget?: boolean, usage?: boolean, cost?: boolean, toolPolicy?: boolean, streamsOutput?: boolean, signalsProcesses?: boolean|null}} CapabilityRequirements */
|
|
45
46
|
|
|
46
47
|
/** @typedef {{executable: string, args: string[], promptTransport: "stdin"|"argv", input: string|null, env?: Record<string, string|null>}} HarnessCommand */
|
|
47
48
|
|
|
@@ -324,6 +325,12 @@ export function validateCapabilityRequirements(requirements, label = "requiredCa
|
|
|
324
325
|
if (!isCapabilityName(name)) throw new TypeError(`${label}.${name} is unknown`);
|
|
325
326
|
if (name === "promptTransport") {
|
|
326
327
|
if (value !== "stdin" && value !== "argv") throw new TypeError(`${label}.promptTransport is invalid`);
|
|
328
|
+
} else if (name === "signalsProcesses") {
|
|
329
|
+
// Tri-state: `null` is a harness whose sandbox has not been measured, so
|
|
330
|
+
// it can never satisfy a true or false requirement.
|
|
331
|
+
if (value !== true && value !== false && value !== null) {
|
|
332
|
+
throw new TypeError(`${label}.signalsProcesses must be true, false, or null`);
|
|
333
|
+
}
|
|
327
334
|
} else if (typeof value !== "boolean") {
|
|
328
335
|
throw new TypeError(`${label}.${name} must be boolean`);
|
|
329
336
|
}
|
|
@@ -37,6 +37,8 @@ export const replayHarness = {
|
|
|
37
37
|
// replay/bin.mjs writes its one envelope line after the recorded delay,
|
|
38
38
|
// never incrementally.
|
|
39
39
|
streamsOutput: false,
|
|
40
|
+
// A recording executes nothing, so it never starts a process to signal.
|
|
41
|
+
signalsProcesses: false,
|
|
40
42
|
},
|
|
41
43
|
|
|
42
44
|
// A recording exposes no permission mode.
|
|
@@ -70,6 +70,9 @@ export const zcodeHarness = {
|
|
|
70
70
|
// log held its full 26 lines only once the process exited. Stall
|
|
71
71
|
// detection must not watch this harness's stdout/stderr mtime.
|
|
72
72
|
streamsOutput: false,
|
|
73
|
+
// Unmeasured: no run has proven whether zcode's sandbox can signal child
|
|
74
|
+
// processes or read the process table.
|
|
75
|
+
signalsProcesses: null,
|
|
73
76
|
},
|
|
74
77
|
|
|
75
78
|
// build/edit/plan do not execute commands; command() defaults to yolo.
|
package/src/host/preflight.mjs
CHANGED
|
@@ -20,6 +20,7 @@ import { delimiter, join, resolve } from "node:path";
|
|
|
20
20
|
import { CONTRACT_VERSION, PROTOCOL_SCHEMA_VERSION, getHarness, probeRuntime } from "../harnesses/index.mjs";
|
|
21
21
|
import { addRuntimeRequirement, failoverTargets, runtimeSnapshot } from "../engine/failover.mjs";
|
|
22
22
|
import { validateContract } from "../contract/index.mjs";
|
|
23
|
+
import { sharedVerificationCommands } from "../contract/final-verification.mjs";
|
|
23
24
|
import { DISCOVERY_RUNTIME_DEFINITIONS, discoverRuntimes } from "../engine/runtime-discovery.mjs";
|
|
24
25
|
import { errorMessage } from "../util.mjs";
|
|
25
26
|
import { boundedGitSync } from "../repo/worktree.mjs";
|
|
@@ -220,7 +221,9 @@ const VERIFICATION_DURATION_WARN_RATIO = 0.8;
|
|
|
220
221
|
|
|
221
222
|
/**
|
|
222
223
|
* Every distinct verification command the contract declares, with the
|
|
223
|
-
* strictest timeout any node gives it and the nodes that share it.
|
|
224
|
+
* strictest timeout any node gives it and the nodes that share it. The
|
|
225
|
+
* contract-wide `sharedVerification` set is included once, under the name
|
|
226
|
+
* `sharedVerification`, because every node runs it.
|
|
224
227
|
*
|
|
225
228
|
* Commands are keyed by argv and cwd, never merged across different argv, so
|
|
226
229
|
* one measurement stands in for every node that declares the same command —
|
|
@@ -234,6 +237,7 @@ export function declaredVerificationCommands(contract) {
|
|
|
234
237
|
const commands = new Map();
|
|
235
238
|
const declarations = [
|
|
236
239
|
...contract.nodes.flatMap((node) => (node.taskPacket?.verification ?? []).map((command) => ({ command, node: node.id }))),
|
|
240
|
+
...sharedVerificationCommands(contract).map((command) => ({ command, node: "sharedVerification" })),
|
|
237
241
|
...(contract.finalVerification ?? []).map((command) => ({ command, node: "finalVerification" })),
|
|
238
242
|
];
|
|
239
243
|
for (const { command, node } of declarations) {
|
package/src/notify/index.mjs
CHANGED
|
@@ -11,6 +11,33 @@
|
|
|
11
11
|
* a `no_transport` receipt is recorded instead — there is no implicit desktop
|
|
12
12
|
* fallback. The macOS notifier is reachable only by setting
|
|
13
13
|
* `FABERUN_NOTIFY_BIN=os-macos`, an explicit opt-in, never a default.
|
|
14
|
+
*
|
|
15
|
+
* Measured 2026-09-16: `test/cli/cli.test.mjs`'s two notifier fixtures were
|
|
16
|
+
* instrumented with `{at, phase}` timelines (spawned, stdin-end, exit) and run
|
|
17
|
+
* over 80 times (targeted loops, four-way parallel full-file bursts, and a
|
|
18
|
+
* 15-way parallel burst) alongside `node --test test/engine/` and
|
|
19
|
+
* `test/contract/` as background load; every completed timeline resolved in
|
|
20
|
+
* under 40ms end to end, and a direct spawn-to-first-line-of-JS measurement
|
|
21
|
+
* under the same load never exceeded 306ms across 40 concurrent spawns --
|
|
22
|
+
* ruling out class (a) (Node process launch under load), since 306ms is
|
|
23
|
+
* ~16x below the 5000ms budget that has been observed to fire. Two genuine
|
|
24
|
+
* `notification timed out after 5000ms` receipts turned up in leftover run
|
|
25
|
+
* directories from other concurrent sessions on this shared machine (their
|
|
26
|
+
* fixtures unmodified, so no timeline exists for them), confirming the flake
|
|
27
|
+
* is real but requires contention this harness could not reliably reproduce
|
|
28
|
+
* with an instrumented fixture. Class (c) (stdin never ends) is excluded by
|
|
29
|
+
* inspection: `spawnDeliver` below always calls `child.stdin.end(...)`
|
|
30
|
+
* synchronously right after spawning, unconditionally. That leaves class (b):
|
|
31
|
+
* the previous implementation resolved on the child's `close` event, which
|
|
32
|
+
* Node fires only once every stdio stream (including the piped, accumulating
|
|
33
|
+
* `stderr`) has finished closing -- a fd inherited or held open by a
|
|
34
|
+
* lingering grandchild, or slow to flush under load, delays `close` well
|
|
35
|
+
* past the point the notifier process itself has already exited. `spawnDeliver`
|
|
36
|
+
* now resolves on `exit` (fires as soon as the process itself terminates,
|
|
37
|
+
* independent of stdio stream closure) instead of `close`, and no longer
|
|
38
|
+
* gates delivery on the stderr stream ending. The timeout budget is left at
|
|
39
|
+
* its original 5000ms: no measurement here justified raising it, and the (b)
|
|
40
|
+
* fix removes the mechanism that budget was actually timing out on.
|
|
14
41
|
*/
|
|
15
42
|
|
|
16
43
|
import { spawn as defaultSpawn } from "node:child_process";
|
|
@@ -32,6 +59,15 @@ export const NOTIFY_LOG_FILE = "notify.jsonl";
|
|
|
32
59
|
*/
|
|
33
60
|
export const MAX_ATTEMPTS = 3;
|
|
34
61
|
|
|
62
|
+
/**
|
|
63
|
+
* The spawn-to-delivery budget for a non-macOS transport, in milliseconds.
|
|
64
|
+
* Unchanged from its original value: the 2026-09-16 measurement (see the
|
|
65
|
+
* module header) found no evidence this needed to be larger, only that the
|
|
66
|
+
* previous implementation could time out waiting on `close` while the child
|
|
67
|
+
* had already exited. That is fixed at the source in `spawnDeliver` below.
|
|
68
|
+
*/
|
|
69
|
+
export const notificationDeliveryTimeoutMs = 5_000;
|
|
70
|
+
|
|
35
71
|
const SUMMARY_CHARS = 200;
|
|
36
72
|
|
|
37
73
|
/**
|
|
@@ -248,12 +284,19 @@ function deliverNotification(event, options = {}) {
|
|
|
248
284
|
}
|
|
249
285
|
|
|
250
286
|
/**
|
|
287
|
+
* Resolves on the child's own `exit`, not `close`: `close` waits for every
|
|
288
|
+
* stdio stream to finish closing, and a piped `stderr` fd can be held open
|
|
289
|
+
* by a lingering grandchild or be slow to flush under load well after the
|
|
290
|
+
* notifier process itself has terminated. Gating delivery on that stream
|
|
291
|
+
* closing (as the previous implementation did) could stall a healthy,
|
|
292
|
+
* already-exited delivery until the timeout fired.
|
|
293
|
+
*
|
|
251
294
|
* @param {string} bin
|
|
252
295
|
* @param {JsonObject} event
|
|
253
296
|
* @param {{spawn?: typeof defaultSpawn, timeoutMs?: number}} options
|
|
254
297
|
* @returns {Promise<DeliveryResult>}
|
|
255
298
|
*/
|
|
256
|
-
function spawnDeliver(bin, event, { spawn = defaultSpawn, timeoutMs =
|
|
299
|
+
function spawnDeliver(bin, event, { spawn = defaultSpawn, timeoutMs = notificationDeliveryTimeoutMs } = {}) {
|
|
257
300
|
return new Promise((resolveDelivery) => {
|
|
258
301
|
let child;
|
|
259
302
|
try {
|
|
@@ -275,7 +318,7 @@ function spawnDeliver(bin, event, { spawn = defaultSpawn, timeoutMs = 5_000 } =
|
|
|
275
318
|
stderr = `${stderr}${chunk}`.slice(-1024);
|
|
276
319
|
});
|
|
277
320
|
child.once("error", (error) => finish({ ok: false, error: errorMessage(error) }));
|
|
278
|
-
child.once("
|
|
321
|
+
child.once("exit", (code) => finish(code === 0 ? { ok: true } : { ok: false, error: stderr || `notification exited ${code}` }));
|
|
279
322
|
const timer = setTimeout(() => {
|
|
280
323
|
try {
|
|
281
324
|
child.kill("SIGTERM");
|
|
@@ -46,7 +46,7 @@ export function validateSourceIdentity(value, label, expected = null) {
|
|
|
46
46
|
assertObject(value, label);
|
|
47
47
|
const allowed = new Set([
|
|
48
48
|
"kind", "id", "campaignId", "contractId", "nodeId", "cwd", "gitHead",
|
|
49
|
-
"dirtyTreeFingerprint", "packetHashes", "harnessVersions",
|
|
49
|
+
"dirtyTreeFingerprint", "packetHashes", "harnessVersions", "baseRef",
|
|
50
50
|
]);
|
|
51
51
|
rejectUnknown(value, allowed, label);
|
|
52
52
|
requireString(value.kind, `${label}.kind`);
|
|
@@ -54,7 +54,7 @@ export function validateSourceIdentity(value, label, expected = null) {
|
|
|
54
54
|
if (value[key] !== undefined) requireId(value[key], `${label}.${key}`);
|
|
55
55
|
}
|
|
56
56
|
if (value.cwd !== undefined) requireString(value.cwd, `${label}.cwd`);
|
|
57
|
-
for (const key of ["gitHead", "dirtyTreeFingerprint"]) {
|
|
57
|
+
for (const key of ["gitHead", "dirtyTreeFingerprint", "baseRef"]) {
|
|
58
58
|
if (value[key] !== undefined && value[key] !== null) requireString(value[key], `${label}.${key}`);
|
|
59
59
|
}
|
|
60
60
|
if (value.packetHashes !== undefined) validateHashMap(value.packetHashes, `${label}.packetHashes`);
|
|
@@ -75,7 +75,7 @@ export function validateSourceIdentity(value, label, expected = null) {
|
|
|
75
75
|
/**
|
|
76
76
|
* @param {{id: string, campaignId: string, cwd: string, nodes: {id: string, packetHash: string}[]}} contract
|
|
77
77
|
* @param {Record<string, string|null>} harnessVersions
|
|
78
|
-
* @param {{ignorePaths?: string[], ignoreRoots?: string[]}} options
|
|
78
|
+
* @param {{ignorePaths?: string[], ignoreRoots?: string[], baseRef?: string}} options
|
|
79
79
|
*/
|
|
80
80
|
export function captureSourceIdentity(contract, harnessVersions = {}, options = {}) {
|
|
81
81
|
const git = gitIdentity(contract.cwd, options);
|
|
@@ -88,6 +88,7 @@ export function captureSourceIdentity(contract, harnessVersions = {}, options =
|
|
|
88
88
|
dirtyTreeFingerprint: git.dirtyTreeFingerprint,
|
|
89
89
|
packetHashes: Object.fromEntries(contract.nodes.map((node) => [node.id, node.packetHash])),
|
|
90
90
|
harnessVersions,
|
|
91
|
+
baseRef: options.baseRef ?? null,
|
|
91
92
|
}, "run source identity", { kind: "run", contractId: contract.id, campaignId: contract.campaignId });
|
|
92
93
|
}
|
|
93
94
|
/**
|
package/src/report/final.mjs
CHANGED
|
@@ -90,7 +90,7 @@ export function renderFinalReport(runDir, contract, states) {
|
|
|
90
90
|
const widths = [3, 24, 9, 7, 7, 28, 10, 10, 10, 12, 64];
|
|
91
91
|
/** @param {unknown[]} cells */
|
|
92
92
|
const row = (cells) => cells.map((cell, index) => fit(String(cell ?? ""), widths[index])).join(" ");
|
|
93
|
-
const totals = { inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0 };
|
|
93
|
+
const totals = { inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, declaredReadBytes: 0 };
|
|
94
94
|
let totalCostUsd = null;
|
|
95
95
|
const lines = [
|
|
96
96
|
`# run ${basename(runDir)}`,
|
|
@@ -106,6 +106,7 @@ export function renderFinalReport(runDir, contract, states) {
|
|
|
106
106
|
totals.inputTokens += usage.inputTokens ?? 0;
|
|
107
107
|
totals.outputTokens += usage.outputTokens ?? 0;
|
|
108
108
|
totals.cacheReadInputTokens += usage.cacheReadInputTokens ?? 0;
|
|
109
|
+
if (typeof node.declaredReadBytes === "number") totals.declaredReadBytes += node.declaredReadBytes;
|
|
109
110
|
if (typeof node.costUsd === "number" && Number.isFinite(node.costUsd)) totalCostUsd = (totalCostUsd ?? 0) + node.costUsd;
|
|
110
111
|
const runtime = node.runtime ? `${node.runtime.harness}/${node.runtime.model}` : "-";
|
|
111
112
|
const planNode = contract.nodes.find((candidate) => candidate.id === node.id);
|
|
@@ -129,7 +130,7 @@ export function renderFinalReport(runDir, contract, states) {
|
|
|
129
130
|
]));
|
|
130
131
|
}
|
|
131
132
|
const roles = roleCosts(nodes);
|
|
132
|
-
lines.push("```", "", `totals · in ${compactTokens(totals.inputTokens)} · out ${compactTokens(totals.outputTokens)} · cache ${compactTokens(totals.cacheReadInputTokens)} · worker ${compactCost(roles.worker)} · judge ${compactCost(roles.judge)} · cost ${compactCost(totalCostUsd)}`);
|
|
133
|
+
lines.push("```", "", `totals · in ${compactTokens(totals.inputTokens)} · out ${compactTokens(totals.outputTokens)} · cache ${compactTokens(totals.cacheReadInputTokens)} · worker ${compactCost(roles.worker)} · judge ${compactCost(roles.judge)} · cost ${compactCost(totalCostUsd)} · read ${compactTokens(totals.declaredReadBytes)}`);
|
|
133
134
|
return `${lines.join("\n")}\n`;
|
|
134
135
|
}
|
|
135
136
|
/**
|