faberun 0.19.1 → 0.19.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/faberun/references/contract.md +17 -5
- package/src/campaign/index.mjs +56 -1
- package/src/cli/plan.mjs +164 -27
- package/src/cli/spec.mjs +17 -7
- package/src/cli.mjs +1 -0
- package/src/contract/definition-of-done.mjs +50 -0
- package/src/contract/final-verification.mjs +21 -0
- package/src/contract/index.mjs +20 -8
- package/src/contract/verification.mjs +63 -2
- package/src/engine/dispatch.mjs +2 -1
- package/src/engine/process.mjs +21 -1
- package/src/engine/scheduler.mjs +2 -2
- package/src/plan/pipeline.mjs +27 -4
- package/src/plan/proof-run.mjs +121 -0
- package/src/plan/repo-facts.mjs +5 -39
- package/src/plan/sizing.mjs +72 -4
- package/src/plan/spec.mjs +25 -1
- package/src/plan/template.mjs +24 -2
- package/src/report/final.mjs +44 -7
- package/src/report/render.mjs +98 -139
- package/src/report/role-usage.mjs +145 -0
package/src/engine/dispatch.mjs
CHANGED
|
@@ -25,6 +25,7 @@ import { attemptWorkspace, createAttemptWorktree, sealAttempt } from "../repo/wo
|
|
|
25
25
|
import { attemptWorktreePath } from "../run/paths.mjs";
|
|
26
26
|
import { basename, dirname, join } from "node:path";
|
|
27
27
|
import { errorCode, errorMessage } from "../util.mjs";
|
|
28
|
+
import { gateProofTimeoutMs } from "../contract/final-verification.mjs";
|
|
28
29
|
import { captureWorkspaceScope, captureWorkspaceSnapshot } from "../repo/workspace.mjs";
|
|
29
30
|
import { deterministicGate, judgeReaskReason, judgeRequired, judgeSkippedByScope } from "./judge-gate.mjs";
|
|
30
31
|
import { emptyScope, persistedScopeBoundary, workerScope } from "./scope.mjs";
|
|
@@ -459,7 +460,7 @@ export async function startJudge(contract, node, state, runDir, running, workerR
|
|
|
459
460
|
node,
|
|
460
461
|
workspace,
|
|
461
462
|
reask,
|
|
462
|
-
|
|
463
|
+
gateProofTimeoutMs(node, contract),
|
|
463
464
|
/** @type {import("../contract/index.mjs").VerificationState|null} */ (state.verification),
|
|
464
465
|
);
|
|
465
466
|
state.review = reviewMode(node.gate);
|
package/src/engine/process.mjs
CHANGED
|
@@ -36,7 +36,7 @@ import { killTarget } from "../host/platform.mjs";
|
|
|
36
36
|
/** @typedef {{prompt: string|null, stdout: string, stderr: string}} PathSet */
|
|
37
37
|
/** @typedef {{id: string, pid: number, processGroupId: number|null, processStartToken: string|null, harness: string, runtimeId: string|null, runtimeFingerprint?: string, revision?: number, phase: string, promptPath: string|null, stdoutPath: string, stderrPath: string, startedAt: string, deadlineAt: string|null, updatedAt: string, closedAt: string|null, exitCode: number|null, signal: string|null, status: "active"|"closed"|"terminated", executable: string, snapshotPath?: string, usage?: Usage, usageEstimated?: boolean, costUsd?: number|null, costProvenance?: "priced", runId?: string, campaignId?: string, nodeId?: string, attempt?: number, workspace?: string, worktreeBranch?: string|null, worktreeBaseSha?: string|null, planPhase?: string, role?: "worker"|"judge", model?: string, reasoning?: string|null, sandbox?: string|null, continuationId?: string|null, continuationMode?: "fresh"|"reuse"|"rotate", session?: import("../harnesses/session-metrics.mjs").SessionLedger|null}} Invocation */
|
|
38
38
|
/** @typedef {{pid: number|null, processGroupId?: number|null, processStartToken?: string|null}} InvocationProbe */
|
|
39
|
-
/** @typedef {{child: ChildProcess, contract: ValidatedContract, node: ValidatedNode, state: NodeSnapshot, runtime: HarnessRuntime & {id: string|null}, cwd: string, paths: PathSet, phase: string, invocation: Invocation, startedAt: string, startedTicks: bigint, progressTicks: bigint, lastOutputAt: number, closed: boolean, exitCode: number|null, signal: string|null, spawnError: Error|null, terminating: Promise<void>|null, gateConfigPath: string, gateReleasePath: string, scopeBaseline?: unknown, scopeChecked?: boolean, scopeViolation?: boolean, resultMaterialization?: boolean, recoveryBaseline?: unknown, observeTimer?: ReturnType<typeof setInterval>, monitorOffset?: number, monitorParser?: import("../harnesses/session-metrics.mjs").SessionMetricsParser, lastEventCount?: number, lastMonitorOffset?: number, observedOnce?: boolean, onClose?: (invocation: Invocation) => void, onInvocationUpdate?: (invocation: Invocation) => void, onProgress?: (state: NodeSnapshot) => void}} Job */
|
|
39
|
+
/** @typedef {{child: ChildProcess, contract: ValidatedContract, node: ValidatedNode, state: NodeSnapshot, runtime: HarnessRuntime & {id: string|null}, cwd: string, paths: PathSet, phase: string, invocation: Invocation, startedAt: string, startedTicks: bigint, progressTicks: bigint, lastOutputAt: number, closed: boolean, exitCode: number|null, signal: string|null, spawnError: Error|null, terminating: Promise<void>|null, gateConfigPath: string, gateReleasePath: string, scopeBaseline?: unknown, scopeChecked?: boolean, scopeViolation?: boolean, resultMaterialization?: boolean, recoveryBaseline?: unknown, observeTimer?: ReturnType<typeof setInterval>, monitorOffset?: number, monitorParser?: import("../harnesses/session-metrics.mjs").SessionMetricsParser, lastEventCount?: number, lastMonitorOffset?: number, observedOnce?: boolean, turnCapWarned?: boolean, onClose?: (invocation: Invocation) => void, onInvocationUpdate?: (invocation: Invocation) => void, onProgress?: (state: NodeSnapshot) => void}} Job */
|
|
40
40
|
/** @typedef {{graceMs?: number, killGraceMs?: number, escalate?: boolean, runDir?: string, kill?: (pid: number, signal: string|number) => unknown, child?: ChildProcess|null}} TerminateOptions */
|
|
41
41
|
|
|
42
42
|
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
@@ -278,6 +278,14 @@ export async function terminateInvocation(invocation, options = {}) {
|
|
|
278
278
|
*/
|
|
279
279
|
const SEAL_BEFORE_KILL_CODES = new Set(["wall_clock_timeout", "stall_timeout", "turn_limit"]);
|
|
280
280
|
|
|
281
|
+
/**
|
|
282
|
+
* How far into its request ceiling an attempt gets before it says so. Four
|
|
283
|
+
* fifths: late enough that an ordinary attempt never mentions it (measured
|
|
284
|
+
* 2026-09-20 over 200 completed claude worker turns, p90 was 83 of a 150
|
|
285
|
+
* default), early enough that the remaining fifth is still room to act in.
|
|
286
|
+
*/
|
|
287
|
+
const TURN_CAP_WARN_FRACTION = 0.8;
|
|
288
|
+
|
|
281
289
|
/**
|
|
282
290
|
* How long a `SIGSTOP`ped process group is given to actually stop before the
|
|
283
291
|
* seal begins. The stop is asynchronous; this bounded settle keeps the seal
|
|
@@ -468,6 +476,18 @@ export async function detectStalls(contract, running, onTimeout, onProgress, onB
|
|
|
468
476
|
// (`turns` counts provider requests for claude, dsh and agy; codex
|
|
469
477
|
// reports whole turns, so its cap is in effect a turn count.)
|
|
470
478
|
const turnCap = job.node?.maxTurns ?? contract.maxTurns;
|
|
479
|
+
// The ceiling used to arrive only as the kill. `maxTurns` appeared once
|
|
480
|
+
// in the whole documentation set and not at all in the contract
|
|
481
|
+
// reference, so the author of a long-running contract raised
|
|
482
|
+
// `timeoutSec` and `stallTimeoutSec` -- everything they knew existed --
|
|
483
|
+
// and left this at its default; two Opus attempts at maximum effort
|
|
484
|
+
// were then cut mid-turn with `turn_limit`, after the cost was already
|
|
485
|
+
// paid. Said once per attempt as the ceiling comes into view, it is a
|
|
486
|
+
// decision the operator can still make.
|
|
487
|
+
if (typeof turnCap === "number" && job.turnCapWarned !== true && monitored.turns >= Math.floor(turnCap * TURN_CAP_WARN_FRACTION)) {
|
|
488
|
+
job.turnCapWarned = true;
|
|
489
|
+
process.stdout.write(`[node] ${nodeId} ${job.phase} · ${monitored.turns} of the attempt's maxTurns of ${turnCap} provider requests · raise maxTurns to give it more\n`);
|
|
490
|
+
}
|
|
471
491
|
if (typeof turnCap === "number" && monitored.turns >= turnCap) {
|
|
472
492
|
const limit = {
|
|
473
493
|
code: "turn_limit",
|
package/src/engine/scheduler.mjs
CHANGED
|
@@ -42,7 +42,7 @@ import { operationNextState, providerReceipts, settleInvocation } from "../run/o
|
|
|
42
42
|
import { appendUsageRecord, invocationCost, invocationUsage, recordInvocationUsage } from "../run/usage.mjs";
|
|
43
43
|
import { captureNodeScopeBoundaries, checkWorkerScope, emptyScope } from "./scope.mjs";
|
|
44
44
|
import { validateContractForLaunch } from "../campaign/chain.mjs";
|
|
45
|
-
import { finalVerificationCommands, sharedVerificationCommands } from "../contract/final-verification.mjs";
|
|
45
|
+
import { finalVerificationCommands, gateProofTimeoutMs, sharedVerificationCommands } from "../contract/final-verification.mjs";
|
|
46
46
|
import { startJudge, startWorker } from "./dispatch.mjs";
|
|
47
47
|
import { assertEnvironmentReady, captureRunIdentity, createRunMetadata, serializableContract, statesFingerprint } from "./run-identity.mjs";
|
|
48
48
|
import { blockDependents, runtimeAssignments } from "./assignment.mjs";
|
|
@@ -118,7 +118,7 @@ export function nodeBudgetBasisMs(contract, node) {
|
|
|
118
118
|
// finalVerification set with each.
|
|
119
119
|
const attemptMs = packetMs + sharedMs;
|
|
120
120
|
const candidateMs = attemptMs + finalMs;
|
|
121
|
-
const gateTimeoutMs =
|
|
121
|
+
const gateTimeoutMs = gateProofTimeoutMs(node, contract);
|
|
122
122
|
const commandProofs = (node.definitionOfDone ?? []).filter((item) => item.proof?.kind === "command").length;
|
|
123
123
|
return defaultTimeoutMs + attemptMs + candidateMs + commandProofs * gateTimeoutMs + finalMs;
|
|
124
124
|
}
|
package/src/plan/pipeline.mjs
CHANGED
|
@@ -82,7 +82,7 @@ export const DEFAULT_NODE_BUDGET_MS = 600_000;
|
|
|
82
82
|
const APPROVE_BELOW_VALUES = new Set(["standard", "high", "none"]);
|
|
83
83
|
|
|
84
84
|
/**
|
|
85
|
-
* @param {{specPath: string, campaignId: string, phase: string, cwd?: string, reviewRounds?: number, approveBelow?: ApproveBelow, runtimeDefaults?: {worker?: string, judge?: string}, runtimes: Record<string, JsonObject>, verification?: VerificationSuites, launch: LaunchFn, wait: WaitFn, ask?: AskFn}} options
|
|
85
|
+
* @param {{specPath: string, campaignId: string, phase: string, cwd?: string, reviewRounds?: number, approveBelow?: ApproveBelow, runtimeDefaults?: {worker?: string, judge?: string}, runtimes: Record<string, JsonObject>, verification?: VerificationSuites, packageMode?: import("./sizing.mjs").PackageMode, launch: LaunchFn, wait: WaitFn, ask?: AskFn}} options
|
|
86
86
|
* @returns {Promise<FrozenPipelineResult|ContestedPipelineResult>}
|
|
87
87
|
*/
|
|
88
88
|
export async function runPlanningPipeline(options) {
|
|
@@ -91,6 +91,11 @@ export async function runPlanningPipeline(options) {
|
|
|
91
91
|
reviewRounds = 2, runtimeDefaults = {}, verification = {},
|
|
92
92
|
} = options;
|
|
93
93
|
const ask = options.ask ?? askPlanningRuntimes;
|
|
94
|
+
// Implementation work is sized by what it writes; exploratory work -- an
|
|
95
|
+
// audit, a review, a survey -- by what it reads, because it writes one
|
|
96
|
+
// findings file whatever surface it covers.
|
|
97
|
+
const packageMode = /** @type {import("./sizing.mjs").PackageMode} */ (options.packageMode ?? "implementation");
|
|
98
|
+
if (packageMode !== "implementation" && packageMode !== "exploratory") throw new TypeError(`packageMode must be implementation or exploratory: ${String(packageMode)}`);
|
|
94
99
|
const approveBelow = /** @type {ApproveBelow} */ (options.approveBelow ?? "standard");
|
|
95
100
|
if (!APPROVE_BELOW_VALUES.has(approveBelow)) throw new TypeError(`approveBelow must be one of ${[...APPROVE_BELOW_VALUES].join(", ")}`);
|
|
96
101
|
if (typeof launch !== "function") throw new TypeError("runPlanningPipeline requires a launch seam");
|
|
@@ -183,7 +188,7 @@ export async function runPlanningPipeline(options) {
|
|
|
183
188
|
return { contract: validated, output };
|
|
184
189
|
};
|
|
185
190
|
|
|
186
|
-
const draft = await runStage("draft", { specPath: relativeSpecPath, repoFactsPath: relativeRepoFactsPath, cataloguePath: relativeCataloguePath });
|
|
191
|
+
const draft = await runStage("draft", { specPath: relativeSpecPath, repoFactsPath: relativeRepoFactsPath, cataloguePath: relativeCataloguePath, packageMode });
|
|
187
192
|
/** @type {PlanOutput|null} */
|
|
188
193
|
let plan = null;
|
|
189
194
|
// Everything still open against the plan in hand, accumulated across rounds
|
|
@@ -253,7 +258,7 @@ export async function runPlanningPipeline(options) {
|
|
|
253
258
|
stage = "sizing";
|
|
254
259
|
const sizing = applySizingRules(
|
|
255
260
|
{ nodes: currentPlan.nodes.map(toSizingNode), justification: currentPlan.justification },
|
|
256
|
-
{ nodeBudgetMs: DEFAULT_NODE_BUDGET_MS, facts: repoFacts, minWriteFiles: MIN_WRITE_FILES, turnCeiling: DEFAULT_MAX_TURNS },
|
|
261
|
+
{ nodeBudgetMs: DEFAULT_NODE_BUDGET_MS, facts: repoFacts, minWriteFiles: MIN_WRITE_FILES, turnCeiling: DEFAULT_MAX_TURNS, packageMode, readVolume: (path) => fileLineCount(join(cwd, path)) },
|
|
257
262
|
);
|
|
258
263
|
stage = "routing";
|
|
259
264
|
const routing = resolveRuntimes(sizing.plan.nodes, {
|
|
@@ -371,7 +376,7 @@ export async function runPlanningPipeline(options) {
|
|
|
371
376
|
const findingsPath = join(scratchDir, `findings-round-${round}.json`);
|
|
372
377
|
writeJsonAtomic(findingsPath, findings);
|
|
373
378
|
const revise = await runStage("revise", {
|
|
374
|
-
specPath: relativeSpecPath, repoFactsPath: relativeRepoFactsPath, cataloguePath: relativeCataloguePath, findingsPath: relative(cwd, findingsPath),
|
|
379
|
+
specPath: relativeSpecPath, repoFactsPath: relativeRepoFactsPath, cataloguePath: relativeCataloguePath, findingsPath: relative(cwd, findingsPath), packageMode,
|
|
375
380
|
});
|
|
376
381
|
// Kept for the write-drop comparison: the plan the revise revised, against
|
|
377
382
|
// the plan it produced.
|
|
@@ -760,3 +765,21 @@ function toContractNode(node, phase, assignment) {
|
|
|
760
765
|
gate,
|
|
761
766
|
};
|
|
762
767
|
}
|
|
768
|
+
|
|
769
|
+
/**
|
|
770
|
+
* Lines in a file the plan declares as a read, or null when it cannot be
|
|
771
|
+
* counted (absent, a directory, unreadable). Exploratory sizing is measured
|
|
772
|
+
* against this: what a node must read is what it is paid for.
|
|
773
|
+
*
|
|
774
|
+
* @param {string} path
|
|
775
|
+
* @returns {number|null}
|
|
776
|
+
*/
|
|
777
|
+
function fileLineCount(path) {
|
|
778
|
+
try {
|
|
779
|
+
const text = readFileSync(path, "utf8");
|
|
780
|
+
if (text === "") return 0;
|
|
781
|
+
return text.split("\n").length - (text.endsWith("\n") ? 1 : 0);
|
|
782
|
+
} catch {
|
|
783
|
+
return null;
|
|
784
|
+
}
|
|
785
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Running a spec's declared commands against the repository as it stands:
|
|
3
|
+
* the `measure` a planning fact is collected from, and the `proof` a
|
|
4
|
+
* requirement claims. One spawn shape serves both, because they are the same
|
|
5
|
+
* act — read the repository by running what the author wrote — and only the
|
|
6
|
+
* caller's question differs.
|
|
7
|
+
*
|
|
8
|
+
* It lives apart from `repo-facts.mjs` for one measurable reason: `spec
|
|
9
|
+
* validate` invokes no model, and the suite proves it by walking
|
|
10
|
+
* `plan/spec.mjs`'s runtime import graph and refusing any reach into
|
|
11
|
+
* `engine/` or `harnesses/`. `repo-facts.mjs` reaches `host/preflight.mjs`
|
|
12
|
+
* for its verification timings, and that reaches every harness adapter. This
|
|
13
|
+
* module imports node builtins and two environment variable names, so the
|
|
14
|
+
* validator can run a proof without the graph growing a harness.
|
|
15
|
+
*/
|
|
16
|
+
import { Buffer } from "node:buffer";
|
|
17
|
+
import { spawnSync } from "node:child_process";
|
|
18
|
+
import { existsSync } from "node:fs";
|
|
19
|
+
import { isAbsolute, join } from "node:path";
|
|
20
|
+
import { NOTIFY_BIN_ENV } from "../notify/index.mjs";
|
|
21
|
+
import { NOTIFY_SESSION_ENV } from "../notify/session.mjs";
|
|
22
|
+
|
|
23
|
+
/** @typedef {import("./spec.mjs").SpecRequirement} SpecRequirement */
|
|
24
|
+
/** @typedef {{now?: () => number, run?: typeof import("node:child_process").spawnSync, pathExists?: (target: string) => boolean}} MeasureProbes */
|
|
25
|
+
/** @typedef {{requirementId: string|null, kind: "command"|"path", ref: string, pass: boolean, detail: string}} RequirementProofResult */
|
|
26
|
+
|
|
27
|
+
export const MEASURE_TIMEOUT_MS = 30_000;
|
|
28
|
+
export const MEASURE_OUTPUT_CAP_BYTES = 4096;
|
|
29
|
+
|
|
30
|
+
/** How much of a failing proof's own output a finding carries: enough to name the failure, never the whole log. */
|
|
31
|
+
const PROOF_DETAIL_CAP = 200;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The same named few `timeVerificationCommands` subtracts before spawning a
|
|
35
|
+
* measurement (SIDE_EFFECT_ENV_KEYS in src/host/preflight.mjs, which is
|
|
36
|
+
* module-private and could not be edited by the node that added this): a
|
|
37
|
+
* measure must not notify a human and must not be redirectable at a live,
|
|
38
|
+
* paid harness binary. A subtraction of a named few, not an allowlist — PATH,
|
|
39
|
+
* HOME and every ordinary variable still pass through unchanged.
|
|
40
|
+
*/
|
|
41
|
+
export const MEASURE_SIDE_EFFECT_ENV_KEYS = [
|
|
42
|
+
NOTIFY_BIN_ENV,
|
|
43
|
+
NOTIFY_SESSION_ENV,
|
|
44
|
+
"FABERUN_CODEX_BIN",
|
|
45
|
+
"FABERUN_CLAUDE_BIN",
|
|
46
|
+
"FABERUN_AGY_BIN",
|
|
47
|
+
"FABERUN_DSH_BIN",
|
|
48
|
+
"FABERUN_ZCODE_BIN",
|
|
49
|
+
"FABERUN_EXEC_JSONL_BIN",
|
|
50
|
+
];
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* The shell-capture core every declared command goes through: stripped
|
|
54
|
+
* side-effect environment, an injectable `run` for tests, a hard timeout, and
|
|
55
|
+
* output capped and reported as truncated rather than silently cut.
|
|
56
|
+
*
|
|
57
|
+
* The command goes through the shell (`spawnSync` with `shell: true`), so a
|
|
58
|
+
* spec author's own pipe — `| wc -l`, `| grep -v …` — works exactly as typed
|
|
59
|
+
* at a terminal. A non-zero exit is returned, never thrown: whether that is a
|
|
60
|
+
* fact or a failure is the caller's question, not this function's.
|
|
61
|
+
*
|
|
62
|
+
* @param {string} cwd
|
|
63
|
+
* @param {string} command
|
|
64
|
+
* @param {MeasureProbes} [probes]
|
|
65
|
+
* @returns {{output: string, exitCode: number|null, truncated: boolean}}
|
|
66
|
+
*/
|
|
67
|
+
export function runShellCapture(cwd, command, probes = {}) {
|
|
68
|
+
const run = probes.run ?? spawnSync;
|
|
69
|
+
const env = { ...process.env };
|
|
70
|
+
for (const key of MEASURE_SIDE_EFFECT_ENV_KEYS) delete env[key];
|
|
71
|
+
const result = run(command, { shell: true, cwd, timeout: MEASURE_TIMEOUT_MS, encoding: "utf8", env });
|
|
72
|
+
const combined = `${result.stdout ?? ""}${result.stderr ?? ""}`;
|
|
73
|
+
const bytes = Buffer.from(combined, "utf8");
|
|
74
|
+
const truncated = bytes.length > MEASURE_OUTPUT_CAP_BYTES;
|
|
75
|
+
return {
|
|
76
|
+
output: truncated ? bytes.subarray(0, MEASURE_OUTPUT_CAP_BYTES).toString("utf8") : combined,
|
|
77
|
+
exitCode: result.status,
|
|
78
|
+
truncated,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Run every requirement's declared `proof` against the repository as it
|
|
84
|
+
* stands right now. `spec validate`'s traceability rule only checks that
|
|
85
|
+
* `requirement.proof` is present, never that it passes, so a proof anchored
|
|
86
|
+
* to a form the artifact no longer has still reads as covered. Measured
|
|
87
|
+
* 2026-09-22: two requirements of one campaign's spec named a proof neither
|
|
88
|
+
* could satisfy and `--strict-traceability` said nothing about either,
|
|
89
|
+
* because it never ran them.
|
|
90
|
+
*
|
|
91
|
+
* A `judgment` proof cannot run without a model and is left unattempted —
|
|
92
|
+
* absent from the results rather than reported as passing — the same
|
|
93
|
+
* restraint `measureRequirements` already applies to its own unwired kinds.
|
|
94
|
+
*
|
|
95
|
+
* @param {string} cwd
|
|
96
|
+
* @param {SpecRequirement[]} requirements
|
|
97
|
+
* @param {MeasureProbes} [probes]
|
|
98
|
+
* @returns {RequirementProofResult[]}
|
|
99
|
+
*/
|
|
100
|
+
export function proveRequirements(cwd, requirements, probes = {}) {
|
|
101
|
+
/** @type {RequirementProofResult[]} */
|
|
102
|
+
const results = [];
|
|
103
|
+
for (const requirement of requirements) {
|
|
104
|
+
const proof = requirement.proof;
|
|
105
|
+
if (!proof?.ref) continue;
|
|
106
|
+
if (proof.kind === "command") {
|
|
107
|
+
const { output, exitCode } = runShellCapture(cwd, proof.ref, probes);
|
|
108
|
+
const detail = exitCode === 0
|
|
109
|
+
? "exit 0"
|
|
110
|
+
: `exit ${exitCode ?? "no exit code (killed or never started)"}${output.trim() ? `: ${output.trim().slice(0, PROOF_DETAIL_CAP)}` : ""}`;
|
|
111
|
+
results.push({ requirementId: requirement.id, kind: "command", ref: proof.ref, pass: exitCode === 0, detail });
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
if (proof.kind === "path") {
|
|
115
|
+
const target = isAbsolute(proof.ref) ? proof.ref : join(cwd, proof.ref);
|
|
116
|
+
const exists = probes.pathExists ? probes.pathExists(target) : existsSync(target);
|
|
117
|
+
results.push({ requirementId: requirement.id, kind: "path", ref: proof.ref, pass: exists, detail: exists ? "exists" : "no such path" });
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
return results;
|
|
121
|
+
}
|
package/src/plan/repo-facts.mjs
CHANGED
|
@@ -14,46 +14,23 @@
|
|
|
14
14
|
* injected measurer's own numbers) is what makes two calls at the same HEAD
|
|
15
15
|
* byte-identical.
|
|
16
16
|
*/
|
|
17
|
-
import { spawnSync } from "node:child_process";
|
|
18
17
|
import { existsSync, readFileSync } from "node:fs";
|
|
19
18
|
import { join } from "node:path";
|
|
20
19
|
import { timeVerificationCommands } from "../host/preflight.mjs";
|
|
21
|
-
import { NOTIFY_BIN_ENV } from "../notify/index.mjs";
|
|
22
|
-
import { NOTIFY_SESSION_ENV } from "../notify/session.mjs";
|
|
23
20
|
import { boundedGitSync, gitHead } from "../repo/worktree.mjs";
|
|
21
|
+
import { runShellCapture } from "./proof-run.mjs";
|
|
24
22
|
|
|
25
23
|
/** @typedef {import("./spec.mjs").SpecRequirement} SpecRequirement */
|
|
24
|
+
/** @typedef {import("./proof-run.mjs").MeasureProbes} MeasureProbes */
|
|
26
25
|
/** @typedef {{requirementId: string|null, command: string, output: string, exitCode: number|null, truncated: boolean}} RequirementMeasurement */
|
|
27
26
|
/** @typedef {{argv: string[], measuredMs: number, eligible: boolean}} VerificationCandidate */
|
|
28
27
|
/** @typedef {{path: string, covers: string|null}} TestFileEntry */
|
|
29
28
|
/** @typedef {{formatVersion: number, gitHead: string|null, paths: string[], truncated: boolean, scripts: Record<string, string>, verificationCandidates: VerificationCandidate[], testFiles: TestFileEntry[], requirementMeasurements: RequirementMeasurement[]}} RepoFacts */
|
|
30
|
-
/** @typedef {{now?: () => number, run?: typeof import("node:child_process").spawnSync}} MeasureProbes */
|
|
31
29
|
|
|
32
30
|
const FORMAT_VERSION = 1;
|
|
33
31
|
const DEFAULT_MAX_PATHS = 2000;
|
|
34
32
|
const ELIGIBLE_MS_CEILING = 600_000;
|
|
35
33
|
const CANDIDATE_TIMEOUT_SEC = ELIGIBLE_MS_CEILING / 1_000;
|
|
36
|
-
const MEASURE_TIMEOUT_MS = 30_000;
|
|
37
|
-
const MEASURE_OUTPUT_CAP_BYTES = 4096;
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
* The same named few `timeVerificationCommands` subtracts before spawning a
|
|
41
|
-
* measurement (SIDE_EFFECT_ENV_KEYS in src/host/preflight.mjs, which is
|
|
42
|
-
* module-private and could not be edited by the node that added this): a
|
|
43
|
-
* measure must not notify a human and must not be redirectable at a live,
|
|
44
|
-
* paid harness binary. A subtraction of a named few, not an allowlist — PATH,
|
|
45
|
-
* HOME and every ordinary variable still pass through unchanged.
|
|
46
|
-
*/
|
|
47
|
-
const MEASURE_SIDE_EFFECT_ENV_KEYS = [
|
|
48
|
-
NOTIFY_BIN_ENV,
|
|
49
|
-
NOTIFY_SESSION_ENV,
|
|
50
|
-
"FABERUN_CODEX_BIN",
|
|
51
|
-
"FABERUN_CLAUDE_BIN",
|
|
52
|
-
"FABERUN_AGY_BIN",
|
|
53
|
-
"FABERUN_DSH_BIN",
|
|
54
|
-
"FABERUN_ZCODE_BIN",
|
|
55
|
-
"FABERUN_EXEC_JSONL_BIN",
|
|
56
|
-
];
|
|
57
34
|
|
|
58
35
|
/**
|
|
59
36
|
* Every path git tracks at HEAD, sorted. The bounded spawn is the same
|
|
@@ -177,9 +154,6 @@ function measureCandidates(cwd, commands, probes) {
|
|
|
177
154
|
*/
|
|
178
155
|
export function measureRequirements(cwd, requirements, probes = {}) {
|
|
179
156
|
if (requirements.length === 0) return [];
|
|
180
|
-
const run = probes.run ?? spawnSync;
|
|
181
|
-
const env = { ...process.env };
|
|
182
|
-
for (const key of MEASURE_SIDE_EFFECT_ENV_KEYS) delete env[key];
|
|
183
157
|
/** @type {RequirementMeasurement[]} */
|
|
184
158
|
const measurements = [];
|
|
185
159
|
for (const requirement of requirements) {
|
|
@@ -188,21 +162,13 @@ export function measureRequirements(cwd, requirements, probes = {}) {
|
|
|
188
162
|
// rather than guessed at.
|
|
189
163
|
if (requirement.measure?.kind !== "command" || !requirement.measure.ref) continue;
|
|
190
164
|
const command = requirement.measure.ref;
|
|
191
|
-
const
|
|
192
|
-
|
|
193
|
-
const bytes = Buffer.from(combined, "utf8");
|
|
194
|
-
const truncated = bytes.length > MEASURE_OUTPUT_CAP_BYTES;
|
|
195
|
-
measurements.push({
|
|
196
|
-
requirementId: requirement.id,
|
|
197
|
-
command,
|
|
198
|
-
output: truncated ? bytes.subarray(0, MEASURE_OUTPUT_CAP_BYTES).toString("utf8") : combined,
|
|
199
|
-
exitCode: result.status,
|
|
200
|
-
truncated,
|
|
201
|
-
});
|
|
165
|
+
const { output, exitCode, truncated } = runShellCapture(cwd, command, probes);
|
|
166
|
+
measurements.push({ requirementId: requirement.id, command, output, exitCode, truncated });
|
|
202
167
|
}
|
|
203
168
|
return measurements;
|
|
204
169
|
}
|
|
205
170
|
|
|
171
|
+
|
|
206
172
|
/**
|
|
207
173
|
* @param {string} cwd
|
|
208
174
|
* @param {{measure?: MeasureProbes, maxPaths?: number, requirements?: SpecRequirement[]}} [options]
|
package/src/plan/sizing.mjs
CHANGED
|
@@ -20,7 +20,8 @@ import { dirname } from "node:path";
|
|
|
20
20
|
/** @typedef {{nodes: PlanNode[], justification?: string, [key: string]: unknown}} Plan */
|
|
21
21
|
/** @typedef {{path: string, covers: string|null}} SizingTestFileEntry */
|
|
22
22
|
/** @typedef {{testFiles?: SizingTestFileEntry[]}} SizingFacts */
|
|
23
|
-
/** @typedef {
|
|
23
|
+
/** @typedef {"implementation"|"exploratory"} PackageMode */
|
|
24
|
+
/** @typedef {{nodeBudgetMs: number, targetedFix?: boolean, facts?: SizingFacts, minWriteFiles?: number, maxMergedWriteFiles?: number, turnCeiling?: number, packageMode?: PackageMode, readVolume?: (path: string) => number|null}} SizingOptions */
|
|
24
25
|
/** @typedef {{rule: string, nodes: string[], detail: string}} SizingTransformation */
|
|
25
26
|
/** @typedef {{plan: Plan, transformations: SizingTransformation[], estimate: {nodes: number, overheadMinutes: number}}} SizingResult */
|
|
26
27
|
|
|
@@ -56,12 +57,22 @@ export function applySizingRules(plan, options) {
|
|
|
56
57
|
|
|
57
58
|
let nodes = structuredClone(plan.nodes);
|
|
58
59
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
60
|
+
// Every merge rule below reasons about the write set, which is the right
|
|
61
|
+
// question for implementation work and the wrong one for exploratory work:
|
|
62
|
+
// an audit or review node writes one findings file whatever surface it
|
|
63
|
+
// covers, so a floor of four write files either merges nodes that have
|
|
64
|
+
// nothing to do with each other or refuses the plan outright. That is why
|
|
65
|
+
// the audit of 2026-09-22 was written by hand as 50 KB of JSON instead of
|
|
66
|
+
// planned. In exploratory mode the write set says nothing, and what a node
|
|
67
|
+
// must read is what it is judged on.
|
|
68
|
+
const exploratory = options.packageMode === "exploratory";
|
|
69
|
+
nodes = exploratory ? nodes : mergeNoMechanicalProof(nodes, transformations);
|
|
70
|
+
nodes = exploratory ? nodes : mergeContainedWriteSet(nodes, transformations);
|
|
71
|
+
nodes = mergeUnderfilledSiblings(nodes, exploratory ? null : options.minWriteFiles ?? null, options.maxMergedWriteFiles ?? MAX_MERGED_WRITE_FILES, transformations);
|
|
62
72
|
nodes = splitOverBudgetVerification(nodes, nodeBudgetMs, facts, transformations);
|
|
63
73
|
nodes = markParallelisable(nodes, transformations);
|
|
64
74
|
nodes = flagOverTurnCeiling(nodes, options.turnCeiling ?? null, transformations);
|
|
75
|
+
if (exploratory) nodes = flagReadVolumeImbalance(nodes, options.readVolume ?? null, transformations);
|
|
65
76
|
|
|
66
77
|
if (nodes.length === 1 && !targetedFix) {
|
|
67
78
|
throw new Error(`sizing_single_node_plan: node ${nodes[0].id} is the plan's only node; pass options.targetedFix to allow a single-node plan`);
|
|
@@ -495,3 +506,60 @@ function longestChainDepth(nodes) {
|
|
|
495
506
|
};
|
|
496
507
|
return nodes.length === 0 ? 0 : Math.max(...nodes.map((node) => depthOf(node.id)));
|
|
497
508
|
}
|
|
509
|
+
|
|
510
|
+
/**
|
|
511
|
+
* How far a node's read volume may sit from its siblings' median before the
|
|
512
|
+
* imbalance is worth naming. Three times: measured 2026-09-22 on an audit
|
|
513
|
+
* whose nodes were sized by write set (they each wrote one findings file, so
|
|
514
|
+
* nothing merged and nothing split), one node read about 3,970 lines and cost
|
|
515
|
+
* $2.00 for 2 findings while another cost $0.57 for 4. The ratio between them
|
|
516
|
+
* was close to four, and nothing in the plan said so before either ran.
|
|
517
|
+
*/
|
|
518
|
+
const READ_VOLUME_IMBALANCE_RATIO = 3;
|
|
519
|
+
|
|
520
|
+
/**
|
|
521
|
+
* Exploratory work is sized by what a node must read, not by what it writes.
|
|
522
|
+
* An audit node writes one findings file whatever it covers, so every rule
|
|
523
|
+
* that reasons about the write set is silent here, and the surface a node
|
|
524
|
+
* carries is the thing that can be unbalanced.
|
|
525
|
+
*
|
|
526
|
+
* Advisory, never a transformation of the plan: which files a reviewer must
|
|
527
|
+
* read together is a judgment this cannot make, so it reports the imbalance
|
|
528
|
+
* and leaves the split to the author.
|
|
529
|
+
*
|
|
530
|
+
* @param {PlanNode[]} nodes
|
|
531
|
+
* @param {((path: string) => number|null)|null} readVolume lines in a declared read path, or null when it cannot be measured
|
|
532
|
+
* @param {SizingTransformation[]} transformations
|
|
533
|
+
* @returns {PlanNode[]}
|
|
534
|
+
*/
|
|
535
|
+
function flagReadVolumeImbalance(nodes, readVolume, transformations) {
|
|
536
|
+
if (!readVolume || nodes.length < 2) return nodes;
|
|
537
|
+
const volumes = nodes.map((node) => {
|
|
538
|
+
const paths = /** @type {string[]} */ (node.taskPacket.readFiles ?? []);
|
|
539
|
+
let total = 0;
|
|
540
|
+
let measured = false;
|
|
541
|
+
for (const path of paths) {
|
|
542
|
+
const lines = readVolume(path);
|
|
543
|
+
if (lines === null) continue;
|
|
544
|
+
measured = true;
|
|
545
|
+
total += lines;
|
|
546
|
+
}
|
|
547
|
+
return { id: node.id, total, measured };
|
|
548
|
+
}).filter((entry) => entry.measured);
|
|
549
|
+
if (volumes.length < 2) return nodes;
|
|
550
|
+
|
|
551
|
+
const sorted = [...volumes].sort((left, right) => left.total - right.total);
|
|
552
|
+
const middle = Math.floor(sorted.length / 2);
|
|
553
|
+
const median = sorted.length % 2 === 0 ? (sorted[middle - 1].total + sorted[middle].total) / 2 : sorted[middle].total;
|
|
554
|
+
if (median <= 0) return nodes;
|
|
555
|
+
|
|
556
|
+
for (const entry of volumes) {
|
|
557
|
+
if (entry.total <= median * READ_VOLUME_IMBALANCE_RATIO) continue;
|
|
558
|
+
transformations.push({
|
|
559
|
+
rule: "read-volume-imbalance",
|
|
560
|
+
nodes: [entry.id],
|
|
561
|
+
detail: `${entry.id} reads ${entry.total} lines against a median of ${median} across this plan; exploratory work is paid for by what it reads, so this node costs several times what its siblings do and is worth splitting`,
|
|
562
|
+
});
|
|
563
|
+
}
|
|
564
|
+
return nodes;
|
|
565
|
+
}
|
package/src/plan/spec.mjs
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* schedules, or reaches a provider — this module invokes no model.
|
|
7
7
|
*/
|
|
8
8
|
import { git } from "../repo/worktree.mjs";
|
|
9
|
+
import { proveRequirements } from "./proof-run.mjs";
|
|
9
10
|
|
|
10
11
|
/** @typedef {"command"|"path"|"judgment"} ProofKind */
|
|
11
12
|
/** @typedef {{kind: ProofKind, ref?: string}} SpecProof */
|
|
@@ -280,8 +281,18 @@ function targetMatchesOrigin(cwd, target) {
|
|
|
280
281
|
* Advisory by default — every violation is recorded and `ok` stays `true` —
|
|
281
282
|
* and blocking under `strict`, where any violation makes `ok` `false`.
|
|
282
283
|
*
|
|
284
|
+
* `runProofs` is the exception to "no model call, ever ... never executes
|
|
285
|
+
* a proof": it opts into actually running each requirement's `proof`
|
|
286
|
+
* (`proveRequirements`, the same shell-capture path `measure` already has),
|
|
287
|
+
* because a proof that is merely present is not a proof that passes.
|
|
288
|
+
* Measured 2026-09-22: two requirements anchored their proof to a form the
|
|
289
|
+
* artifact no longer had, and `--strict-traceability` alone never noticed,
|
|
290
|
+
* since it only checks that `proof` exists. A failed proof is blocking
|
|
291
|
+
* regardless of `strict` — nothing about "the proof does not run" is an
|
|
292
|
+
* advisory nicety like a missing Non-goals section.
|
|
293
|
+
*
|
|
283
294
|
* @param {string} text
|
|
284
|
-
* @param {{cwd?: string, strict?: boolean}} [options]
|
|
295
|
+
* @param {{cwd?: string, strict?: boolean, runProofs?: boolean, proofProbes?: import("./proof-run.mjs").MeasureProbes}} [options]
|
|
285
296
|
* @returns {SpecValidation}
|
|
286
297
|
*/
|
|
287
298
|
export function validateSpec(text, options = {}) {
|
|
@@ -316,6 +327,19 @@ export function validateSpec(text, options = {}) {
|
|
|
316
327
|
if (typeof parsed.frontMatter.target === "string" && !targetMatchesOrigin(cwd, parsed.frontMatter.target)) {
|
|
317
328
|
findings.push({ rule: "target-unresolved", severity: "advisory", message: `target "${parsed.frontMatter.target}" does not match the origin remote`, line: 1 });
|
|
318
329
|
}
|
|
330
|
+
if (options.runProofs === true) {
|
|
331
|
+
const byId = new Map(parsed.requirements.map((requirement) => [requirement.id, requirement]));
|
|
332
|
+
for (const result of proveRequirements(cwd, parsed.requirements, options.proofProbes)) {
|
|
333
|
+
if (result.pass) continue;
|
|
334
|
+
const requirement = result.requirementId === null ? null : byId.get(result.requirementId);
|
|
335
|
+
findings.push({
|
|
336
|
+
rule: "requirement-proof-failed",
|
|
337
|
+
severity: "blocking",
|
|
338
|
+
message: `requirement ${result.requirementId ?? requirement?.title ?? "?"}'s proof does not pass (${result.kind}: ${result.ref}): ${result.detail}`,
|
|
339
|
+
line: requirement?.line ?? 1,
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
}
|
|
319
343
|
const graded = findings.map((finding) => (strict ? { ...finding, severity: /** @type {const} */ ("blocking") } : finding));
|
|
320
344
|
return { class: "structured", ok: !graded.some((finding) => finding.severity === "blocking"), findings: graded };
|
|
321
345
|
}
|
package/src/plan/template.mjs
CHANGED
|
@@ -23,7 +23,7 @@ import { validateVerificationCommands } from "../contract/verification.mjs";
|
|
|
23
23
|
/** @typedef {import("../contract/index.mjs").JsonObject} JsonObject */
|
|
24
24
|
/** @typedef {"draft"|"review"|"revise"|"spec-author"|"spec-review"} PlanningKind */
|
|
25
25
|
/** @typedef {"low"|"standard"|"high"} RiskTier */
|
|
26
|
-
/** @typedef {{campaignId: string, phase: string, n: number, goal?: string, cwd?: string, runtimes: Record<string, JsonObject>, runtimeDefaults: {worker?: string, judge?: string}, specPath?: string, repoFactsPath?: string, cataloguePath?: string, planPath?: string, findingsPath?: string, notesPath?: string}} PlanningContractInputs */
|
|
26
|
+
/** @typedef {{campaignId: string, phase: string, n: number, goal?: string, cwd?: string, runtimes: Record<string, JsonObject>, runtimeDefaults: {worker?: string, judge?: string}, specPath?: string, repoFactsPath?: string, cataloguePath?: string, packageMode?: import("./sizing.mjs").PackageMode, planPath?: string, findingsPath?: string, notesPath?: string}} PlanningContractInputs */
|
|
27
27
|
/** @typedef {{id: string, objective: string, taskKind: string, riskTier: RiskTier, dependsOn: string[], readFiles: string[], writeFiles: string[], scopeAcknowledged: string[], definitionOfDone: import("../contract/definition-of-done.mjs").DefinitionOfDoneItem[], verification: import("../contract/verification.mjs").VerificationCommand[], expectedTurns?: number}} PlanOutputNode */
|
|
28
28
|
/** @typedef {{nodes: PlanOutputNode[], phases?: PlanPhase[], findings?: PlanFindingOutput[], justification?: string}} PlanOutput */
|
|
29
29
|
/** @typedef {{id: string, requirementIds: string[], deliverable: string}} PlanPhase */
|
|
@@ -118,8 +118,30 @@ const PLAN_OUTPUT_SHAPE = '{nodes: [{id, objective, taskKind, riskTier, dependsO
|
|
|
118
118
|
* 150 requests.
|
|
119
119
|
*/
|
|
120
120
|
const SIZING_INSTRUCTION = "Size nodes to 4 to 6 write files where the work allows, and give every node an expectedTurns: the provider requests one worker needs to finish it end to end (measured median 49 for 4 to 6 files). A smaller node pays the same orientation and about 15 minutes of verification, judge and integration for less delivered work; a node you expect past 150 requests must be split, because a run cuts an attempt there.";
|
|
121
|
+
/**
|
|
122
|
+
* The exploratory counterpart to the sizing guidance above: sizing by what a
|
|
123
|
+
* node reads, because that is what exploratory work is paid for. An audit
|
|
124
|
+
* node writes one findings file whatever surface it covers, so the write-set
|
|
125
|
+
* sentence would size every node in such a package identically and wrongly --
|
|
126
|
+
* which is why the audit of 2026-09-22 was written by hand as 50 KB of JSON
|
|
127
|
+
* instead of planned.
|
|
128
|
+
*/
|
|
129
|
+
const EXPLORATORY_SIZING_INSTRUCTION = "Size nodes by what each must read and by risk, never by what it writes: one write file is the normal shape for a finding, a review or an audit. Give every node an expectedTurns (the provider requests one worker needs end to end), keep the read volume of the nodes within the same order of each other so one does not cost several times its siblings, and split a node you expect past 150 requests, because a run cuts an attempt there.";
|
|
121
130
|
const FINDINGS_SHAPE = "[{id, severity, nodeId, text}]";
|
|
122
131
|
|
|
132
|
+
/**
|
|
133
|
+
* The instruction list is a frozen table because it is the same for every
|
|
134
|
+
* campaign; only the sizing sentence depends on what kind of package this is.
|
|
135
|
+
*
|
|
136
|
+
* @param {PlanningKind} kind
|
|
137
|
+
* @param {PlanningContractInputs} inputs
|
|
138
|
+
* @returns {string[]}
|
|
139
|
+
*/
|
|
140
|
+
function instructionsFor(kind, inputs) {
|
|
141
|
+
if (inputs.packageMode !== "exploratory") return INSTRUCTIONS[kind];
|
|
142
|
+
return INSTRUCTIONS[kind].map((line) => (line === SIZING_INSTRUCTION ? EXPLORATORY_SIZING_INSTRUCTION : line));
|
|
143
|
+
}
|
|
144
|
+
|
|
123
145
|
// The rule every planned packet is held to at freeze time, worded from
|
|
124
146
|
// AGENTS.md's Faberun protocol and src/repo/scope-closure.mjs ("reading it
|
|
125
147
|
// cannot fix it"): a first draft that ignores it produces a plan that fails
|
|
@@ -225,7 +247,7 @@ export function buildPlanningContract(kind, inputs) {
|
|
|
225
247
|
const taskPacket = {
|
|
226
248
|
mode: "discovery",
|
|
227
249
|
objective: OBJECTIVES[kind],
|
|
228
|
-
instructions:
|
|
250
|
+
instructions: instructionsFor(kind, inputs),
|
|
229
251
|
readFiles,
|
|
230
252
|
writeFiles: [],
|
|
231
253
|
symbols: [],
|