faberun 0.30.0 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/campaign/campaign-brief.mjs +17 -6
- package/src/campaign/chain.mjs +124 -8
- package/src/campaign/ledger.mjs +19 -7
- package/src/campaign/metrics-command.mjs +68 -9
- package/src/campaign/metrics.mjs +75 -45
- package/src/campaign/record.mjs +8 -1
- package/src/campaign/reserve.mjs +340 -0
- package/src/campaign/spend.mjs +229 -0
- package/src/campaign/unpark.mjs +54 -1
- package/src/campaign/watch.mjs +314 -37
- package/src/cli/brand.mjs +2 -1
- package/src/cli.mjs +52 -2
- package/src/contract/judge-envelope.mjs +17 -2
- package/src/contract/scope-findings.mjs +3 -1
- package/src/contract/task-packet.mjs +40 -5
- package/src/contract/verification.mjs +67 -1
- package/src/engine/aux-admission.mjs +124 -0
- package/src/engine/capacity.mjs +78 -0
- package/src/engine/dispatch.mjs +116 -23
- package/src/engine/failure-cause.mjs +59 -0
- package/src/engine/lifecycle.mjs +3 -2
- package/src/engine/phase-session.mjs +56 -10
- package/src/engine/prompts.mjs +57 -2
- package/src/engine/reserve-admission.mjs +99 -0
- package/src/engine/resume.mjs +3 -1
- package/src/engine/review.mjs +21 -0
- package/src/engine/run-command.mjs +166 -11
- package/src/engine/scheduler.mjs +92 -88
- package/src/engine/settle.mjs +15 -8
- package/src/engine/settlement-overlap.mjs +143 -0
- package/src/engine/verify.mjs +131 -7
- package/src/harnesses/session-metrics.mjs +112 -20
- package/src/notify/index.mjs +86 -10
- package/src/plan/freeze.mjs +7 -2
- package/src/plan/human-step.mjs +36 -0
- package/src/plan/pipeline.mjs +53 -22
- package/src/plan/preflight.mjs +81 -5
- package/src/plan/progress.mjs +101 -1
- package/src/plan/repo-facts.mjs +403 -49
- package/src/plan/resolve.mjs +7 -0
- package/src/plan/revise-context.mjs +108 -0
- package/src/plan/rounds.mjs +180 -45
- package/src/plan/sizing.mjs +33 -1
- package/src/plan/template.mjs +134 -22
- package/src/repo/integrate.mjs +47 -5
- package/src/report/campaign-brief-estimate.mjs +47 -1
- package/src/report/campaign-brief.mjs +31 -2
- package/src/report/locale.mjs +20 -0
- package/src/report/message.mjs +13 -1
- package/src/report/metrics-report.mjs +38 -0
- package/src/report/progress.mjs +194 -10
- package/src/report/render.mjs +1 -1
- package/src/report/report-html.mjs +235 -0
- package/src/report/report-video.mjs +323 -0
- package/src/run/usage-windows.mjs +99 -4
- package/src/run/usage.mjs +69 -7
- package/src/web/api.mjs +179 -24
- package/src/web/app.mjs +27 -2
- package/src/web/server.mjs +32 -0
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
* Which provider session a node's next invocation runs in: fresh, a
|
|
3
3
|
* continuation of a compatible earlier session in the same plan phase, or a
|
|
4
4
|
* rotation -- a fresh session that carries the prior nodes' structured
|
|
5
|
-
* summaries
|
|
5
|
+
* summaries for a worker, and the role's own prompt, whole, for a judge.
|
|
6
|
+
* Separate from dispatch.mjs, which spawns whatever this decides,
|
|
6
7
|
* because the decision reads only persisted node snapshots and the routing
|
|
7
8
|
* table, and because dispatch.mjs crossed the 800-line ceiling carrying it.
|
|
8
9
|
*/
|
|
@@ -12,6 +13,7 @@ import { boundedUtf8, stableJson } from "../util.mjs";
|
|
|
12
13
|
import { providerCommand } from "../harnesses/index.mjs";
|
|
13
14
|
import { readJson } from "../run/store.mjs";
|
|
14
15
|
import { routeRuntimeForState } from "./failover.mjs";
|
|
16
|
+
import { JUDGE_PROMPT_BYTES, JUDGE_PROMPT_REASON } from "../contract/judge-envelope.mjs";
|
|
15
17
|
import { validateNodeSnapshot } from "../contract/snapshot.mjs";
|
|
16
18
|
|
|
17
19
|
/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
|
|
@@ -57,6 +59,13 @@ export function forceFreshSession(state, explicit = {}) {
|
|
|
57
59
|
* the call site is not enough, because this function rediscovers the prior
|
|
58
60
|
* continuation from the persisted ledger.
|
|
59
61
|
*
|
|
62
|
+
* A rotation never substitutes a synthesized text for the prompt the role was
|
|
63
|
+
* dispatched with. A worker gets the phase's prior summaries in front of its
|
|
64
|
+
* own prompt; a judge gets its own prompt, whole, because the definition of
|
|
65
|
+
* done, the evidence and the re-ask instruction it carries are the review, and
|
|
66
|
+
* the author's packet and reasoning summary are what the review is independent
|
|
67
|
+
* of.
|
|
68
|
+
*
|
|
60
69
|
* @param {ValidatedContract} contract
|
|
61
70
|
* @param {ValidatedNode} node
|
|
62
71
|
* @param {NodeSnapshot} state
|
|
@@ -99,13 +108,11 @@ export function phaseInvocationPlan(contract, node, state, runDir, role, prompt,
|
|
|
99
108
|
// A harness that cannot continue at all, or a session picked up from a
|
|
100
109
|
// different phase-sibling node whose identity does not match this one, has
|
|
101
110
|
// no native continuity: the fresh attempt carries the prior nodes'
|
|
102
|
-
// structured summaries forward instead of starting blind.
|
|
111
|
+
// structured summaries forward instead of starting blind. The judge is the
|
|
112
|
+
// exception -- it rotates onto its own prompt, never the author's.
|
|
103
113
|
if (session && (!canContinue || session.nodeId !== node.id)) {
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
continuationId: null,
|
|
107
|
-
mode: "rotate",
|
|
108
|
-
};
|
|
114
|
+
const rotated = role === "judge" ? preservedJudgePrompt(prompt) : phaseHandoffPrompt(contract, node, state, runDir, role, prompt);
|
|
115
|
+
return { prompt: rotated, continuationId: null, mode: "rotate" };
|
|
109
116
|
}
|
|
110
117
|
// A capable harness continuing its own node whose identity merely drifted
|
|
111
118
|
// (the run directory moved, or a runtime edge) still gets the caller's own
|
|
@@ -188,8 +195,22 @@ export function fingerprintRuntime(runtime) {
|
|
|
188
195
|
return createHash("sha256").update(stableJson({ runtime, executable })).digest("hex");
|
|
189
196
|
}
|
|
190
197
|
|
|
191
|
-
/**
|
|
192
|
-
|
|
198
|
+
/**
|
|
199
|
+
* The worker's rotation prompt: the phase's prior structured summaries in front
|
|
200
|
+
* of the prompt the caller built for this attempt — the node's own packet, or
|
|
201
|
+
* the retry prompt carrying the gate's findings. Reading `node.prompt` here
|
|
202
|
+
* instead discarded whatever the caller had added to the role's effective
|
|
203
|
+
* prompt, which is the defect this rotation exists to avoid.
|
|
204
|
+
*
|
|
205
|
+
* @param {ValidatedContract} contract
|
|
206
|
+
* @param {ValidatedNode} node
|
|
207
|
+
* @param {NodeSnapshot} state
|
|
208
|
+
* @param {string} runDir
|
|
209
|
+
* @param {"worker"|"judge"} role
|
|
210
|
+
* @param {string} prompt
|
|
211
|
+
* @returns {string}
|
|
212
|
+
*/
|
|
213
|
+
function phaseHandoffPrompt(contract, node, state, runDir, role, prompt) {
|
|
193
214
|
const summaries = phaseSessionCandidates(contract, node, state, runDir, role)
|
|
194
215
|
.map(({ nodeId }) => {
|
|
195
216
|
const candidate = contract.nodes.find((item) => item.id === nodeId);
|
|
@@ -211,7 +232,32 @@ function phaseHandoffPrompt(contract, node, state, runDir, role) {
|
|
|
211
232
|
"Prior structured node summaries:",
|
|
212
233
|
summaries.length ? summaries.map((summary) => `- ${summary}`).join("\n") : "- (none)",
|
|
213
234
|
"Current closed task packet:",
|
|
214
|
-
boundedUtf8(
|
|
235
|
+
boundedUtf8(prompt, 48 * 1024),
|
|
215
236
|
].join("\n\n");
|
|
216
237
|
return boundedUtf8(handoff, 60 * 1024);
|
|
217
238
|
}
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* The judge's effective prompt, carried whole into a fresh session, or refused.
|
|
242
|
+
*
|
|
243
|
+
* measured 2026-09-29 (review R1): the rotation used to rebuild the text from
|
|
244
|
+
* `node.prompt`, so a judge picking up a phase sibling's session was handed the
|
|
245
|
+
* author's closed packet under a "continue as the judge agent" preamble and
|
|
246
|
+
* lost the criteria, the evidence and the re-ask instruction — a judge that
|
|
247
|
+
* never sees the checklist can only be discarded by it. The judge takes none of
|
|
248
|
+
* the author's context: it is not worker continuity, it is an independent
|
|
249
|
+
* review of work the prompt already describes.
|
|
250
|
+
*
|
|
251
|
+
* The prompt is returned unmodified or not at all. Bounding it to fit would cut
|
|
252
|
+
* exactly the mandatory content, so one over `JUDGE_PROMPT_BYTES` fails before
|
|
253
|
+
* any provider is spawned, with the code every oversized judge prompt carries.
|
|
254
|
+
*
|
|
255
|
+
* @param {string} prompt
|
|
256
|
+
* @returns {string}
|
|
257
|
+
*/
|
|
258
|
+
function preservedJudgePrompt(prompt) {
|
|
259
|
+
if (Buffer.byteLength(prompt, "utf8") <= JUDGE_PROMPT_BYTES) return prompt;
|
|
260
|
+
const error = /** @type {Error & {code: string}} */ (new Error(JUDGE_PROMPT_REASON));
|
|
261
|
+
error.code = "judge_prompt_too_large";
|
|
262
|
+
throw error;
|
|
263
|
+
}
|
package/src/engine/prompts.mjs
CHANGED
|
@@ -202,6 +202,58 @@ export function previousAttemptOperatorAnswer(previousAttempt) {
|
|
|
202
202
|
return answer || null;
|
|
203
203
|
}
|
|
204
204
|
|
|
205
|
+
/**
|
|
206
|
+
* The heading the variable previous-attempt state travels under, byte-equal
|
|
207
|
+
* to the line `renderPreviousAttemptSection` (engine/retry.mjs) writes: two
|
|
208
|
+
* hashes, one space, `Previous attempt`, end of line. Every tolerance this
|
|
209
|
+
* match ever carried cost a node: a tolerant suffix read the task's own
|
|
210
|
+
* `## Previous attempt notes:` heading as retry state (this node's attempt
|
|
211
|
+
* 2), and a tolerant `#{1,6}` hash run read a task instruction headed
|
|
212
|
+
* `# Previous attempt` the same way (attempt 3's revision_cap) — with a
|
|
213
|
+
* state section supplied, the lifted block was replaced and the stable
|
|
214
|
+
* instructions after that heading were dropped from the worker and judge
|
|
215
|
+
* prompts. Only the writer's exact spelling may open the variable block.
|
|
216
|
+
*/
|
|
217
|
+
const PREVIOUS_ATTEMPT_HEADING = /^## Previous attempt$/gm;
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* The line every rendered record opens with — `Attempt N failed; this is
|
|
221
|
+
* attempt M.`, the `Attempt N failed` opening `previousAttemptOperatorAnswer`'s
|
|
222
|
+
* stop regex reads as a section start, and the first body line under the
|
|
223
|
+
* heading in every record the retry writer lays down (this node's own retry
|
|
224
|
+
* records included). Requiring it behind the heading keeps the split on real
|
|
225
|
+
* trailing record blocks: an instruction block that borrows even the exact
|
|
226
|
+
* heading opens its body with other words — measured on attempt 3's
|
|
227
|
+
* revision_cap, where `Attempt 2 failed in staging` under a borrowed heading
|
|
228
|
+
* was classified as a record and split out of the stable prefix.
|
|
229
|
+
*/
|
|
230
|
+
const PREVIOUS_ATTEMPT_RECORD_LINE = /^Attempt \d+ failed; this is attempt \d+\./u;
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Split a prompt that already carries the variable `Previous attempt` section
|
|
234
|
+
* into the stable instructions and that section, so assembly can re-append it
|
|
235
|
+
* after the stable sandbox notice and result-file protocol — cacheable prefix
|
|
236
|
+
* first, variable state last. The section is the block from its heading to
|
|
237
|
+
* the end of the prompt: every writer lays it down as the trailing block of
|
|
238
|
+
* whatever it was built on. `section` is null when the base carries none.
|
|
239
|
+
*
|
|
240
|
+
* @param {string} basePrompt
|
|
241
|
+
* @returns {{stable: string, section: string|null}}
|
|
242
|
+
*/
|
|
243
|
+
export function splitPreviousAttemptSection(basePrompt) {
|
|
244
|
+
for (const match of basePrompt.matchAll(PREVIOUS_ATTEMPT_HEADING)) {
|
|
245
|
+
const rest = basePrompt.slice(match.index + match[0].length);
|
|
246
|
+
const bodyStart = rest.split("\n").find((line) => line.trim() !== "");
|
|
247
|
+
if (bodyStart !== undefined && PREVIOUS_ATTEMPT_RECORD_LINE.test(bodyStart)) {
|
|
248
|
+
return {
|
|
249
|
+
stable: basePrompt.slice(0, match.index).replace(/\n+$/u, ""),
|
|
250
|
+
section: basePrompt.slice(match.index).replace(/\n+$/u, ""),
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
return { stable: basePrompt, section: null };
|
|
255
|
+
}
|
|
256
|
+
|
|
205
257
|
/**
|
|
206
258
|
* @param {JudgeNode} node
|
|
207
259
|
* @param {unknown} workerResult
|
|
@@ -256,7 +308,6 @@ export function judgePrompt(node, workerResult, context = {}) {
|
|
|
256
308
|
`Controller diff paths:\n${diff.length ? diff.map((path) => `- ${path}`).join("\n") : "- (none)"}\n\n` +
|
|
257
309
|
`Controller verification:\n${JSON.stringify(verificationResult)}\n\n` +
|
|
258
310
|
(scopeSection ? `${scopeSection}\n\n` : "") +
|
|
259
|
-
(context.previousAttempt ? `${context.previousAttempt}\n\n` : "") +
|
|
260
311
|
"Return only the JSON object required by the output schema, with every field present: a verdict with nothing to report still carries `findings: []`, never an omitted key. The verdict is a record, not a report: keep `summary` within " + JUDGE_LIMITS.summaryBytes + " bytes, use at most " + JUDGE_LIMITS.findings + " findings, and keep each finding's `description` within " + JUDGE_LIMITS.descriptionBytes + " bytes and its `evidence` within " + JUDGE_LIMITS.evidenceBytes + " bytes. A verdict that overshoots this envelope is rejected unread, however sound the arbitration. Evidence must be concrete. " +
|
|
261
312
|
"Use verdict pass only when findings is empty and maxSeverity is none. " +
|
|
262
313
|
"Use verdict fail whenever findings is non-empty, including advisory findings below failOn. " +
|
|
@@ -264,7 +315,11 @@ export function judgePrompt(node, workerResult, context = {}) {
|
|
|
264
315
|
? "Arbitrate only the judgment items; deterministic items are already proven by the controller and must not be re-arbitrated. " +
|
|
265
316
|
"Use pass only when every judgment item is satisfied. Every finding must cite the id of the judgment item it addresses; a fail verdict whose findings cite no id is a protocol failure."
|
|
266
317
|
: "Use pass only when every Definition of Done item is satisfied. " +
|
|
267
|
-
"Assess every Definition of Done item by its id and cite the id you are addressing in each finding.")
|
|
318
|
+
"Assess every Definition of Done item by its id and cite the id you are addressing in each finding.") +
|
|
319
|
+
// The variable state trails every stable instruction, including the
|
|
320
|
+
// output-schema rules, so the cacheable prefix ends at the section's
|
|
321
|
+
// heading and dispatch's assembly keeps it last when it re-appends.
|
|
322
|
+
(context.previousAttempt ? `\n\n${context.previousAttempt}` : "");
|
|
268
323
|
}
|
|
269
324
|
|
|
270
325
|
/**
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Admission of one new paid dispatch through the campaign's optional reserve
|
|
3
|
+
* (ADR 0011): read the campaign record, and either admit the dispatch with a
|
|
4
|
+
* hold, admit it as unconfigured, or refuse it before any provider starts.
|
|
5
|
+
* Split from `dispatch.mjs` because the decision is one gate the worker and
|
|
6
|
+
* the judge paths must apply identically, and because its fail-closed rule
|
|
7
|
+
* needs the record read in exactly one place: a record that is present but
|
|
8
|
+
* cannot be read or validated refuses the dispatch, it is not waved through
|
|
9
|
+
* on the validation the launch performed.
|
|
10
|
+
*/
|
|
11
|
+
import { existsSync } from "node:fs";
|
|
12
|
+
import { join } from "node:path";
|
|
13
|
+
import { errorMessage } from "../util.mjs";
|
|
14
|
+
import { CAMPAIGN_FILE } from "../campaign/layout.mjs";
|
|
15
|
+
import { readCampaign } from "../campaign/record.mjs";
|
|
16
|
+
import { releaseCampaignReservation, reserveCampaignCost } from "../campaign/reserve.mjs";
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The error code a node is parked blocked with when the campaign's reserve
|
|
20
|
+
* refuses its new dispatch. Deliberately outside every auto-retry set: the
|
|
21
|
+
* balance does not recover by re-dispatching, only by an operator topping it
|
|
22
|
+
* up (or the in-flight holds settling) and resuming.
|
|
23
|
+
*/
|
|
24
|
+
const RESERVE_INSUFFICIENT_CODE = "reserve_insufficient";
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* The error code a node is parked blocked with when the campaign record is
|
|
28
|
+
* present but cannot be read or validated. Distinct from
|
|
29
|
+
* `reserve_insufficient` because the operator responses differ -- a balance
|
|
30
|
+
* needs a top-up, a broken record needs a repair -- and like it, deliberately
|
|
31
|
+
* outside every auto-retry set: re-dispatching does not make a record
|
|
32
|
+
* readable.
|
|
33
|
+
*/
|
|
34
|
+
const CAMPAIGN_RECORD_UNREADABLE_CODE = "campaign_record_unreadable";
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Admit one new paid dispatch through the campaign's optional reserve
|
|
38
|
+
* (ADR 0011), before the provider starts. The only measured predictor of
|
|
39
|
+
* what the call will cost is what this node's most recent invocation actually
|
|
40
|
+
* charged; a first dispatch has no measurement, so it is reserved as unknown
|
|
41
|
+
* exposure and holds nothing -- the reserve never invents a number to gate
|
|
42
|
+
* with. A refusal takes no hold and carries the error the caller parks the
|
|
43
|
+
* node blocked with. Calls already running are never touched: the reserve
|
|
44
|
+
* gates only the admission of a new one.
|
|
45
|
+
*
|
|
46
|
+
* A missing campaign path or record file is the unconfigured path the reserve
|
|
47
|
+
* never touched, and a readable record without `reserveUsd` is the same. A
|
|
48
|
+
* record that is present but cannot be read or validated is the opposite of
|
|
49
|
+
* unconfigured: it may have changed since the launch validated it, so the
|
|
50
|
+
* dispatch is refused before the provider starts instead of being assumed
|
|
51
|
+
* safe.
|
|
52
|
+
*
|
|
53
|
+
* @param {string} campaignPath
|
|
54
|
+
* @param {import("../contract/index.mjs").NodeSnapshot} state
|
|
55
|
+
* @param {string} runId
|
|
56
|
+
* @param {string} nodeId
|
|
57
|
+
* @param {"worker"|"judge"} role
|
|
58
|
+
* @returns {{refused: false, reservation: import("../campaign/reserve.mjs").Reservation|null}|{refused: true, error: {code: string, message: string}}}
|
|
59
|
+
*/
|
|
60
|
+
export function reserveAdmission(campaignPath, state, runId, nodeId, role) {
|
|
61
|
+
if (!campaignPath || !existsSync(join(campaignPath, CAMPAIGN_FILE))) return { refused: false, reservation: null };
|
|
62
|
+
try {
|
|
63
|
+
const campaign = /** @type {{reserveUsd?: number}} */ (/** @type {unknown} */ (readCampaign(campaignPath)));
|
|
64
|
+
if (campaign.reserveUsd === undefined) return { refused: false, reservation: null };
|
|
65
|
+
} catch (error) {
|
|
66
|
+
return { refused: true, error: {
|
|
67
|
+
code: CAMPAIGN_RECORD_UNREADABLE_CODE,
|
|
68
|
+
message: `campaign record at ${campaignPath} cannot be read or validated, refusing the new ${role} dispatch: ${errorMessage(error)}`,
|
|
69
|
+
} };
|
|
70
|
+
}
|
|
71
|
+
const last = state.invocations?.at(-1);
|
|
72
|
+
const estimateUsd = typeof last?.costUsd === "number" && Number.isFinite(last.costUsd) ? last.costUsd : null;
|
|
73
|
+
const decision = reserveCampaignCost(campaignPath, { runId, node: nodeId, costUsd: estimateUsd });
|
|
74
|
+
if (!decision.admitted) return { refused: true, error: {
|
|
75
|
+
code: RESERVE_INSUFFICIENT_CODE,
|
|
76
|
+
message: `reserve refused the new ${role} dispatch: estimated ${estimateUsd} USD cannot be held against ${decision.availableUsd ?? 0} USD available`,
|
|
77
|
+
} };
|
|
78
|
+
return { refused: false, reservation: decision.reservation };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Give a reservation back when the gated call never started (the provider
|
|
83
|
+
* could not be spawned). Never throws: a hold that cannot be released stays
|
|
84
|
+
* visible as held in `readCampaignReserve`, and the dispatch failure it rode
|
|
85
|
+
* on still settles the node.
|
|
86
|
+
*
|
|
87
|
+
* @param {string} campaignPath
|
|
88
|
+
* @param {import("../campaign/reserve.mjs").Reservation|null} reservation
|
|
89
|
+
* @returns {void}
|
|
90
|
+
*/
|
|
91
|
+
export function releaseReservation(campaignPath, reservation) {
|
|
92
|
+
if (!campaignPath || !reservation) return;
|
|
93
|
+
try {
|
|
94
|
+
releaseCampaignReservation(campaignPath, reservation.id);
|
|
95
|
+
} catch {
|
|
96
|
+
// The hold remains recorded; it is visible in the reserve status and no
|
|
97
|
+
// other reservation can consume it.
|
|
98
|
+
}
|
|
99
|
+
}
|
package/src/engine/resume.mjs
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
import { JUDGE_SCHEMA, SETTLED, TERMINAL } from "./prompts.mjs";
|
|
16
16
|
import { Buffer } from "node:buffer";
|
|
17
17
|
import { acquire as acquireLock } from "../run/lock.mjs";
|
|
18
|
+
import { classifyFailureCause } from "./failure-cause.mjs";
|
|
18
19
|
import { applyInvalidWorkerResult, assertRunMutable, handleProviderExhaustion } from "./lifecycle.mjs";
|
|
19
20
|
import { applyJudgeResult } from "./review.mjs";
|
|
20
21
|
import { assertSourceUnchanged, captureRunIdentity, recordedBaseRef, statesFingerprint } from "./run-identity.mjs";
|
|
@@ -618,6 +619,7 @@ export async function reauthorRefusedNode(options) {
|
|
|
618
619
|
const raw = JSON.parse(readFileSync(contractPath, "utf8"));
|
|
619
620
|
const refused = /** @type {{missingContext?: string[]}|null|undefined} */ (state.result);
|
|
620
621
|
const missingContext = Array.isArray(refused?.missingContext) ? refused.missingContext : [];
|
|
622
|
+
const failureCause = classifyFailureCause(node, state);
|
|
621
623
|
const budget = await runBoundedRounds(rounds, async (round, history) => {
|
|
622
624
|
const additions = (await discover({ round, node, state, missingContext, findings: history.flatMap((item) => item.findings) })) ?? {};
|
|
623
625
|
const packet = widenTaskPacket(node.taskPacket, additions);
|
|
@@ -641,7 +643,7 @@ export async function reauthorRefusedNode(options) {
|
|
|
641
643
|
const outcome = budget.accepted ? (reauthorApproved(node, { approve, approveBelow }) ? "applied" : "approval_required") : "rounds_exhausted";
|
|
642
644
|
const record = reauthorProposalRecord({
|
|
643
645
|
nodeId: node.id, outcome, riskTier: reauthorRiskTier(node), budget: rounds, roundsUsed: budget.roundsUsed,
|
|
644
|
-
additions: last?.additions, packet: last?.packet, findings: last?.findings, history: budget.history,
|
|
646
|
+
additions: last?.additions, packet: last?.packet, findings: last?.findings, history: budget.history, failureCause,
|
|
645
647
|
});
|
|
646
648
|
appendJsonl(join(runDir, "reauthor.jsonl"), record);
|
|
647
649
|
return {
|
package/src/engine/review.mjs
CHANGED
|
@@ -23,6 +23,7 @@ import {
|
|
|
23
23
|
import { errorMessage, excerpt } from "../util.mjs";
|
|
24
24
|
import { appendTransitionEvent, transition, writeNode } from "./state.mjs";
|
|
25
25
|
import { startJudge } from "./dispatch.mjs";
|
|
26
|
+
import { auxSpawnAdmission, recordAuxRefusal } from "./aux-admission.mjs";
|
|
26
27
|
import { applyRejection, raiseNodeAttention, settleDone } from "./settle.mjs";
|
|
27
28
|
|
|
28
29
|
/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
|
|
@@ -86,6 +87,16 @@ export async function applyJudgeProtocolFailure(contract, node, state, runDir, r
|
|
|
86
87
|
transition(runDir, state, "pending", { phase: "judge", gate: null, result: state.result, error: null, blockedBy: [] }, lock);
|
|
87
88
|
return;
|
|
88
89
|
}
|
|
90
|
+
const admission = auxSpawnAdmission(contract, node, state, running, states, "judge");
|
|
91
|
+
if (!admission.admitted) {
|
|
92
|
+
// The runtime the re-ask would land on holds its ceiling: defer the
|
|
93
|
+
// bounded re-ask the same way a controller-loss recovery defers it, bound
|
|
94
|
+
// and result durable, until the scheduler can admit it.
|
|
95
|
+
const from = state.status;
|
|
96
|
+
transition(runDir, state, "pending", { phase: "judge", gate: null, result: state.result, error: null, blockedBy: [] }, lock);
|
|
97
|
+
recordAuxRefusal(runDir, state, from, lock, "judge", admission);
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
89
100
|
await applyJudgeRound(await startJudge(contract, node, state, runDir, running, state.result, lock, states, campaignPath),
|
|
90
101
|
contract, node, state, runDir, running, lock, states, campaignPath, state.result);
|
|
91
102
|
}
|
|
@@ -179,6 +190,16 @@ export async function applyJudgeResult(contract, node, state, result, runDir, lo
|
|
|
179
190
|
transition(runDir, state, "pending", { phase: "judge", gate: null, result: state.result, error: null, blockedBy: [] }, lock);
|
|
180
191
|
return;
|
|
181
192
|
}
|
|
193
|
+
const admission = auxSpawnAdmission(contract, node, state, running, states, "judge");
|
|
194
|
+
if (!admission.admitted) {
|
|
195
|
+
// The runtime the re-ask would land on holds its ceiling: defer the
|
|
196
|
+
// bounded re-ask with its bound and result durable until the scheduler
|
|
197
|
+
// can admit it, exactly as a controller-loss recovery defers it.
|
|
198
|
+
const from = state.status;
|
|
199
|
+
transition(runDir, state, "pending", { phase: "judge", gate: null, result: state.result, error: null, blockedBy: [] }, lock);
|
|
200
|
+
recordAuxRefusal(runDir, state, from, lock, "judge", admission);
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
182
203
|
await applyJudgeRound(await startJudge(contract, node, state, runDir, running, state.result, lock, states, campaignPath),
|
|
183
204
|
contract, node, state, runDir, running, lock, states, campaignPath, state.result);
|
|
184
205
|
return;
|
|
@@ -9,18 +9,38 @@
|
|
|
9
9
|
*
|
|
10
10
|
* The schema for what may be run lives in `contract/verification.mjs`; this is
|
|
11
11
|
* only the doing.
|
|
12
|
+
*
|
|
13
|
+
* A pass interrupted mid-list restarts every command on resume (review
|
|
14
|
+
* 2026-09-29: an interrupted verification re-runs the whole list, and the
|
|
15
|
+
* engine's own suite costs 1035-1058s per full pass — `VERIFICATION_LIMITS.maxTimeoutSec`).
|
|
16
|
+
* Each completed command therefore also writes a checkpoint beside its log,
|
|
17
|
+
* `checkpoint-<ordinal>.json`: the identity hash of everything the proof
|
|
18
|
+
* depends on — the committed tree (the HEAD tree id) and the dirty tree
|
|
19
|
+
* fingerprint, the validated command, the exact child environment, the
|
|
20
|
+
* dependency-inputs hash — plus the bounded result. Both tree hashes are
|
|
21
|
+
* needed: the fingerprint hashes only the difference from HEAD, so clean
|
|
22
|
+
* trees at different commits share one, and a stale passing result survived a
|
|
23
|
+
* commit until the HEAD tree joined the identity (review 2026-10-02, d5.1). A
|
|
24
|
+
* resumed pass serves a checkpoint only when that identity is unchanged and
|
|
25
|
+
* the tree actually carried a fingerprint; any difference forces
|
|
26
|
+
* re-execution. A
|
|
27
|
+
* mutation entry is the explicit invalidation policy: it is never checkpointed
|
|
28
|
+
* and never served, because its proof is a function of the tree it breaks and
|
|
29
|
+
* restores, so a reused record could certify a tree that no longer exists.
|
|
12
30
|
*/
|
|
13
31
|
import { Buffer } from "node:buffer";
|
|
14
|
-
import { VERIFICATION_LIMITS, compactVerification, resolveVerificationCwd, validateVerificationCommands } from "../contract/verification.mjs";
|
|
15
|
-
import { mkdirSync, writeFileSync } from "node:fs";
|
|
32
|
+
import { VERIFICATION_LIMITS, commandCheckpointIdentity, compactVerification, parseCommandCheckpoint, resolveVerificationCwd, validateVerificationCommands } from "../contract/verification.mjs";
|
|
33
|
+
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
16
34
|
import { processStartToken } from "../run/lock.mjs";
|
|
17
|
-
import { randomUUID } from "node:crypto";
|
|
35
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
18
36
|
import { runMutation } from "./mutation.mjs";
|
|
19
37
|
import { SANDBOX_BLOCKED_WRITE } from "../contract/worker-result.mjs";
|
|
20
|
-
import { isAbsolute, resolve } from "node:path";
|
|
38
|
+
import { isAbsolute, join, resolve } from "node:path";
|
|
21
39
|
import { isContained } from "../util.mjs";
|
|
22
40
|
import { spawn } from "node:child_process";
|
|
23
41
|
import { killTarget, spawnInvocation } from "../host/platform.mjs";
|
|
42
|
+
import { gitIdentity } from "../repo/source-identity.mjs";
|
|
43
|
+
import { boundedGitSync } from "../repo/worktree.mjs";
|
|
24
44
|
/** @typedef {import("../contract/verification.mjs").VerificationOptions} VerificationOptions */
|
|
25
45
|
|
|
26
46
|
/**
|
|
@@ -76,14 +96,15 @@ const VERIFICATION_ENV_BASE_NAMES = Object.freeze([
|
|
|
76
96
|
|
|
77
97
|
/**
|
|
78
98
|
* @param {VerificationCommand} command
|
|
79
|
-
* @returns {Record<string, string
|
|
99
|
+
* @returns {Record<string, string>}
|
|
80
100
|
*/
|
|
81
101
|
function verificationEnv(command) {
|
|
82
102
|
const names = new Set([...(command.env ?? []), ...VERIFICATION_ENV_BASE_NAMES]);
|
|
83
|
-
/** @type {Record<string, string
|
|
103
|
+
/** @type {Record<string, string>} */
|
|
84
104
|
const env = {};
|
|
85
105
|
for (const name of names) {
|
|
86
|
-
|
|
106
|
+
const value = process.env[name];
|
|
107
|
+
if (value !== undefined) env[name] = value;
|
|
87
108
|
}
|
|
88
109
|
return env;
|
|
89
110
|
}
|
|
@@ -126,6 +147,123 @@ export function classifySandboxBlockedWrite({ text, workspace, mode }) {
|
|
|
126
147
|
}
|
|
127
148
|
return null;
|
|
128
149
|
}
|
|
150
|
+
/**
|
|
151
|
+
* Everything one pass needs to decide checkpoint reuse, cached per pass: the
|
|
152
|
+
* tree identity per command cwd (one `gitIdentity` and one `rev-parse` each,
|
|
153
|
+
* paid only when a checkpoint directory is in play) and the dependency-inputs
|
|
154
|
+
* hash (the packet's writeFiles do not change during a pass; a mutation entry
|
|
155
|
+
* restores them in `finally`).
|
|
156
|
+
*
|
|
157
|
+
* @typedef {{trees: Map<string, WorkspaceTreeIdentity>, dependencyHash: string|undefined}} CheckpointCaches
|
|
158
|
+
*/
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* The two tree hashes a checkpoint identity sits on: the dirty fingerprint
|
|
162
|
+
* (what the working tree adds to HEAD) and the HEAD tree id (what HEAD
|
|
163
|
+
* commits). Either changing forces re-execution.
|
|
164
|
+
*
|
|
165
|
+
* @typedef {{dirtyTreeFingerprint: string|null, headTree: string|null}} WorkspaceTreeIdentity
|
|
166
|
+
*/
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* The committed tree the workspace sits on: the HEAD tree id, which the dirty
|
|
170
|
+
* fingerprint cannot see — it hashes only the difference from HEAD, so two
|
|
171
|
+
* clean trees at different commits used to share one checkpoint identity and
|
|
172
|
+
* a stale pass survived a commit (review 2026-10-02, d5.1). Null when there
|
|
173
|
+
* is no commit to name (no repository, unborn HEAD): the identity still
|
|
174
|
+
* compares, and null to null is the same unborn state across passes.
|
|
175
|
+
*
|
|
176
|
+
* @param {string} workspace
|
|
177
|
+
* @returns {string|null}
|
|
178
|
+
*/
|
|
179
|
+
function headTreeId(workspace) {
|
|
180
|
+
try {
|
|
181
|
+
const result = boundedGitSync(["-C", workspace, "rev-parse", "HEAD^{tree}"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
|
|
182
|
+
if (result.error || result.status !== 0) return null;
|
|
183
|
+
return String(result.stdout).trim() || null;
|
|
184
|
+
} catch {
|
|
185
|
+
// Git absent or unreadable: no committed tree to name, matching the
|
|
186
|
+
// dirty fingerprint's own null for a workspace without a repository.
|
|
187
|
+
return null;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* A hash over the dependency inputs a proof is about: the packet's writeFiles,
|
|
193
|
+
* the mutation sample's input. ReadFiles and every other tracked or untracked
|
|
194
|
+
* source ride inside the tree fingerprint instead; installed dependencies that
|
|
195
|
+
* git ignores (a mid-attempt `npm ci`) are covered only when the packet
|
|
196
|
+
* declares them here — a known bound, not a solved one.
|
|
197
|
+
*
|
|
198
|
+
* @param {string} baseCwd
|
|
199
|
+
* @param {string[]} writeFiles
|
|
200
|
+
* @returns {string}
|
|
201
|
+
*/
|
|
202
|
+
function dependencyInputsHash(baseCwd, writeFiles) {
|
|
203
|
+
const hash = createHash("sha256");
|
|
204
|
+
for (const path of [...new Set(writeFiles)].sort()) {
|
|
205
|
+
hash.update(`${path}\0`);
|
|
206
|
+
try {
|
|
207
|
+
hash.update(readFileSync(resolve(baseCwd, path)));
|
|
208
|
+
} catch {
|
|
209
|
+
// ENOENT or a non-file: the mutation sampler skips the same paths, so a
|
|
210
|
+
// missing dependency input hashes as missing rather than failing the pass.
|
|
211
|
+
hash.update("missing");
|
|
212
|
+
}
|
|
213
|
+
hash.update("\0");
|
|
214
|
+
}
|
|
215
|
+
return hash.digest("hex");
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* The pre-run identity of one command's proof, plus whether the tree gave a
|
|
220
|
+
* fingerprint to compare at all: a workspace that is not a git repository has
|
|
221
|
+
* no tree identity, so its checkpoints never serve.
|
|
222
|
+
*
|
|
223
|
+
* @param {VerificationCommand} command
|
|
224
|
+
* @param {string} baseCwd
|
|
225
|
+
* @param {RunVerificationOptions} options
|
|
226
|
+
* @param {CheckpointCaches} caches
|
|
227
|
+
* @returns {{identity: string, reusable: boolean}}
|
|
228
|
+
*/
|
|
229
|
+
function checkpointIdentity(command, baseCwd, options, caches) {
|
|
230
|
+
const workspace = resolveVerificationCwd(baseCwd, command.cwd ?? ".");
|
|
231
|
+
let tree = caches.trees.get(workspace);
|
|
232
|
+
if (tree === undefined) {
|
|
233
|
+
tree = { dirtyTreeFingerprint: gitIdentity(workspace).dirtyTreeFingerprint, headTree: headTreeId(workspace) };
|
|
234
|
+
caches.trees.set(workspace, tree);
|
|
235
|
+
}
|
|
236
|
+
if (caches.dependencyHash === undefined) {
|
|
237
|
+
caches.dependencyHash = dependencyInputsHash(baseCwd, options.writeFiles ?? []);
|
|
238
|
+
}
|
|
239
|
+
return {
|
|
240
|
+
identity: commandCheckpointIdentity({ treeFingerprint: tree.dirtyTreeFingerprint, headTree: tree.headTree, command, env: verificationEnv(command), dependencyHash: caches.dependencyHash }),
|
|
241
|
+
reusable: tree.dirtyTreeFingerprint !== null,
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* The recorded result for this ordinal, served only when the checkpoint
|
|
247
|
+
* validates, the tree carried a fingerprint, and the identity is unchanged.
|
|
248
|
+
* Every read or parse failure lands on re-execution: a corrupted checkpoint is
|
|
249
|
+
* never a verdict on the command and never fatal to the pass.
|
|
250
|
+
*
|
|
251
|
+
* @param {string} logDir
|
|
252
|
+
* @param {number} ordinal
|
|
253
|
+
* @param {{identity: string, reusable: boolean}} identity
|
|
254
|
+
* @returns {VerificationCommandResult|null}
|
|
255
|
+
*/
|
|
256
|
+
function servedCheckpoint(logDir, ordinal, identity) {
|
|
257
|
+
try {
|
|
258
|
+
const parsed = parseCommandCheckpoint(JSON.parse(readFileSync(join(logDir, `checkpoint-${ordinal}.json`), "utf8")));
|
|
259
|
+
if (!identity.reusable || parsed.identity !== identity.identity) return null;
|
|
260
|
+
return parsed.result;
|
|
261
|
+
} catch {
|
|
262
|
+
// ENOENT is the no-checkpoint case; every other failure (corrupt JSON, a
|
|
263
|
+
// record past its bounds) fails closed to a re-run for the same reason.
|
|
264
|
+
return null;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
129
267
|
/**
|
|
130
268
|
* Run every declared command `repeat` times inside the workspace.
|
|
131
269
|
*
|
|
@@ -138,14 +276,31 @@ export async function runVerification(commands, baseCwd, options = {}) {
|
|
|
138
276
|
const validated = validateVerificationCommands(commands);
|
|
139
277
|
/** @type {VerificationCommandResult[]} */
|
|
140
278
|
const results = [];
|
|
279
|
+
/** @type {CheckpointCaches|null} */
|
|
280
|
+
const checkpoints = options.logDir ? { trees: new Map(), dependencyHash: undefined } : null;
|
|
141
281
|
for (const [commandIndex, command] of validated.entries()) {
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
282
|
+
// Identity before the run: a checkpoint records the tree, environment and
|
|
283
|
+
// dependency inputs the proof was made against, which are the pre-run ones.
|
|
284
|
+
const identity = checkpoints && !command.mutation
|
|
285
|
+
? checkpointIdentity(command, baseCwd, options, checkpoints)
|
|
286
|
+
: null;
|
|
287
|
+
const reused = identity
|
|
288
|
+
? servedCheckpoint(/** @type {string} */ (options.logDir), results.length + 1, identity)
|
|
289
|
+
: null;
|
|
290
|
+
const result = reused ?? await (command.mutation
|
|
291
|
+
? runMutationCommand(command, baseCwd, commandIndex, options)
|
|
292
|
+
: runRepeatedCommand(command, baseCwd, commandIndex, options));
|
|
145
293
|
results.push(result);
|
|
146
294
|
if (options.logDir) {
|
|
147
295
|
mkdirSync(options.logDir, { recursive: true });
|
|
148
|
-
|
|
296
|
+
const compacted = compactVerification({ passed: result.passed, commands: [result] });
|
|
297
|
+
writeFileSync(`${options.logDir}/verification-${results.length}.json`, `${JSON.stringify(compacted)}\n`, { mode: 0o600 });
|
|
298
|
+
// A mutation entry records no checkpoint (the invalidation policy in the
|
|
299
|
+
// header); an argv past the persisted bounds compacts to no command, and
|
|
300
|
+
// its unreadable record simply fails closed to a re-run on resume.
|
|
301
|
+
if (identity) {
|
|
302
|
+
writeFileSync(`${options.logDir}/checkpoint-${results.length}.json`, `${JSON.stringify({ identity: identity.identity, result: compacted })}\n`, { mode: 0o600 });
|
|
303
|
+
}
|
|
149
304
|
}
|
|
150
305
|
}
|
|
151
306
|
return { passed: results.every((result) => result.passed), commands: results };
|