faberun 0.30.0 → 0.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/package.json +1 -1
  2. package/src/campaign/campaign-brief.mjs +17 -6
  3. package/src/campaign/chain.mjs +124 -8
  4. package/src/campaign/ledger.mjs +19 -7
  5. package/src/campaign/metrics-command.mjs +68 -9
  6. package/src/campaign/metrics.mjs +75 -45
  7. package/src/campaign/record.mjs +8 -1
  8. package/src/campaign/reserve.mjs +340 -0
  9. package/src/campaign/spend.mjs +229 -0
  10. package/src/campaign/unpark.mjs +54 -1
  11. package/src/campaign/watch.mjs +314 -37
  12. package/src/cli/brand.mjs +2 -1
  13. package/src/cli.mjs +52 -2
  14. package/src/contract/judge-envelope.mjs +17 -2
  15. package/src/contract/scope-findings.mjs +3 -1
  16. package/src/contract/task-packet.mjs +40 -5
  17. package/src/contract/verification.mjs +67 -1
  18. package/src/engine/aux-admission.mjs +124 -0
  19. package/src/engine/capacity.mjs +78 -0
  20. package/src/engine/dispatch.mjs +116 -23
  21. package/src/engine/failure-cause.mjs +59 -0
  22. package/src/engine/lifecycle.mjs +3 -2
  23. package/src/engine/phase-session.mjs +56 -10
  24. package/src/engine/prompts.mjs +57 -2
  25. package/src/engine/reserve-admission.mjs +99 -0
  26. package/src/engine/resume.mjs +3 -1
  27. package/src/engine/review.mjs +21 -0
  28. package/src/engine/run-command.mjs +166 -11
  29. package/src/engine/scheduler.mjs +92 -88
  30. package/src/engine/settle.mjs +15 -8
  31. package/src/engine/settlement-overlap.mjs +143 -0
  32. package/src/engine/verify.mjs +131 -7
  33. package/src/harnesses/session-metrics.mjs +112 -20
  34. package/src/notify/index.mjs +86 -10
  35. package/src/plan/freeze.mjs +7 -2
  36. package/src/plan/human-step.mjs +36 -0
  37. package/src/plan/pipeline.mjs +53 -22
  38. package/src/plan/preflight.mjs +81 -5
  39. package/src/plan/progress.mjs +101 -1
  40. package/src/plan/repo-facts.mjs +403 -49
  41. package/src/plan/resolve.mjs +7 -0
  42. package/src/plan/revise-context.mjs +108 -0
  43. package/src/plan/rounds.mjs +180 -45
  44. package/src/plan/sizing.mjs +33 -1
  45. package/src/plan/template.mjs +134 -22
  46. package/src/repo/integrate.mjs +47 -5
  47. package/src/report/campaign-brief-estimate.mjs +47 -1
  48. package/src/report/campaign-brief.mjs +31 -2
  49. package/src/report/locale.mjs +20 -0
  50. package/src/report/message.mjs +13 -1
  51. package/src/report/metrics-report.mjs +38 -0
  52. package/src/report/progress.mjs +194 -10
  53. package/src/report/render.mjs +1 -1
  54. package/src/report/report-html.mjs +235 -0
  55. package/src/report/report-video.mjs +323 -0
  56. package/src/run/usage-windows.mjs +99 -4
  57. package/src/run/usage.mjs +69 -7
  58. package/src/web/api.mjs +179 -24
  59. package/src/web/app.mjs +27 -2
  60. package/src/web/server.mjs +32 -0
@@ -2,7 +2,8 @@
2
2
  * Which provider session a node's next invocation runs in: fresh, a
3
3
  * continuation of a compatible earlier session in the same plan phase, or a
4
4
  * rotation -- a fresh session that carries the prior nodes' structured
5
- * summaries. Separate from dispatch.mjs, which spawns whatever this decides,
5
+ * summaries for a worker, and the role's own prompt, whole, for a judge.
6
+ * Separate from dispatch.mjs, which spawns whatever this decides,
6
7
  * because the decision reads only persisted node snapshots and the routing
7
8
  * table, and because dispatch.mjs crossed the 800-line ceiling carrying it.
8
9
  */
@@ -12,6 +13,7 @@ import { boundedUtf8, stableJson } from "../util.mjs";
12
13
  import { providerCommand } from "../harnesses/index.mjs";
13
14
  import { readJson } from "../run/store.mjs";
14
15
  import { routeRuntimeForState } from "./failover.mjs";
16
+ import { JUDGE_PROMPT_BYTES, JUDGE_PROMPT_REASON } from "../contract/judge-envelope.mjs";
15
17
  import { validateNodeSnapshot } from "../contract/snapshot.mjs";
16
18
 
17
19
  /** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
@@ -57,6 +59,13 @@ export function forceFreshSession(state, explicit = {}) {
57
59
  * the call site is not enough, because this function rediscovers the prior
58
60
  * continuation from the persisted ledger.
59
61
  *
62
+ * A rotation never substitutes a synthesized text for the prompt the role was
63
+ * dispatched with. A worker gets the phase's prior summaries in front of its
64
+ * own prompt; a judge gets its own prompt, whole, because the definition of
65
+ * done, the evidence and the re-ask instruction it carries are the review, and
66
+ * the author's packet and reasoning summary are what the review is independent
67
+ * of.
68
+ *
60
69
  * @param {ValidatedContract} contract
61
70
  * @param {ValidatedNode} node
62
71
  * @param {NodeSnapshot} state
@@ -99,13 +108,11 @@ export function phaseInvocationPlan(contract, node, state, runDir, role, prompt,
99
108
  // A harness that cannot continue at all, or a session picked up from a
100
109
  // different phase-sibling node whose identity does not match this one, has
101
110
  // no native continuity: the fresh attempt carries the prior nodes'
102
- // structured summaries forward instead of starting blind.
111
+ // structured summaries forward instead of starting blind. The judge is the
112
+ // exception -- it rotates onto its own prompt, never the author's.
103
113
  if (session && (!canContinue || session.nodeId !== node.id)) {
104
- return {
105
- prompt: phaseHandoffPrompt(contract, node, state, runDir, role),
106
- continuationId: null,
107
- mode: "rotate",
108
- };
114
+ const rotated = role === "judge" ? preservedJudgePrompt(prompt) : phaseHandoffPrompt(contract, node, state, runDir, role, prompt);
115
+ return { prompt: rotated, continuationId: null, mode: "rotate" };
109
116
  }
110
117
  // A capable harness continuing its own node whose identity merely drifted
111
118
  // (the run directory moved, or a runtime edge) still gets the caller's own
@@ -188,8 +195,22 @@ export function fingerprintRuntime(runtime) {
188
195
  return createHash("sha256").update(stableJson({ runtime, executable })).digest("hex");
189
196
  }
190
197
 
191
- /** @param {ValidatedContract} contract @param {ValidatedNode} node @param {NodeSnapshot} state @param {string} runDir @param {"worker"|"judge"} role @returns {string} */
192
- function phaseHandoffPrompt(contract, node, state, runDir, role) {
198
+ /**
199
+ * The worker's rotation prompt: the phase's prior structured summaries in front
200
+ * of the prompt the caller built for this attempt — the node's own packet, or
201
+ * the retry prompt carrying the gate's findings. Reading `node.prompt` here
202
+ * instead discarded whatever the caller had added to the role's effective
203
+ * prompt, which is the defect this rotation exists to avoid.
204
+ *
205
+ * @param {ValidatedContract} contract
206
+ * @param {ValidatedNode} node
207
+ * @param {NodeSnapshot} state
208
+ * @param {string} runDir
209
+ * @param {"worker"|"judge"} role
210
+ * @param {string} prompt
211
+ * @returns {string}
212
+ */
213
+ function phaseHandoffPrompt(contract, node, state, runDir, role, prompt) {
193
214
  const summaries = phaseSessionCandidates(contract, node, state, runDir, role)
194
215
  .map(({ nodeId }) => {
195
216
  const candidate = contract.nodes.find((item) => item.id === nodeId);
@@ -211,7 +232,32 @@ function phaseHandoffPrompt(contract, node, state, runDir, role) {
211
232
  "Prior structured node summaries:",
212
233
  summaries.length ? summaries.map((summary) => `- ${summary}`).join("\n") : "- (none)",
213
234
  "Current closed task packet:",
214
- boundedUtf8(node.prompt, 48 * 1024),
235
+ boundedUtf8(prompt, 48 * 1024),
215
236
  ].join("\n\n");
216
237
  return boundedUtf8(handoff, 60 * 1024);
217
238
  }
239
+
240
+ /**
241
+ * The judge's effective prompt, carried whole into a fresh session, or refused.
242
+ *
243
+ * measured 2026-09-29 (review R1): the rotation used to rebuild the text from
244
+ * `node.prompt`, so a judge picking up a phase sibling's session was handed the
245
+ * author's closed packet under a "continue as the judge agent" preamble and
246
+ * lost the criteria, the evidence and the re-ask instruction — a judge that
247
+ * never sees the checklist can only be discarded by it. The judge takes none of
248
+ * the author's context: it is not worker continuity, it is an independent
249
+ * review of work the prompt already describes.
250
+ *
251
+ * The prompt is returned unmodified or not at all. Bounding it to fit would cut
252
+ * exactly the mandatory content, so one over `JUDGE_PROMPT_BYTES` fails before
253
+ * any provider is spawned, with the code every oversized judge prompt carries.
254
+ *
255
+ * @param {string} prompt
256
+ * @returns {string}
257
+ */
258
+ function preservedJudgePrompt(prompt) {
259
+ if (Buffer.byteLength(prompt, "utf8") <= JUDGE_PROMPT_BYTES) return prompt;
260
+ const error = /** @type {Error & {code: string}} */ (new Error(JUDGE_PROMPT_REASON));
261
+ error.code = "judge_prompt_too_large";
262
+ throw error;
263
+ }
@@ -202,6 +202,58 @@ export function previousAttemptOperatorAnswer(previousAttempt) {
202
202
  return answer || null;
203
203
  }
204
204
 
205
+ /**
206
+ * The heading the variable previous-attempt state travels under, byte-equal
207
+ * to the line `renderPreviousAttemptSection` (engine/retry.mjs) writes: two
208
+ * hashes, one space, `Previous attempt`, end of line. Every tolerance this
209
+ * match ever carried cost a node: a tolerant suffix read the task's own
210
+ * `## Previous attempt notes:` heading as retry state (this node's attempt
211
+ * 2), and a tolerant `#{1,6}` hash run read a task instruction headed
212
+ * `# Previous attempt` the same way (attempt 3's revision_cap) — with a
213
+ * state section supplied, the lifted block was replaced and the stable
214
+ * instructions after that heading were dropped from the worker and judge
215
+ * prompts. Only the writer's exact spelling may open the variable block.
216
+ */
217
+ const PREVIOUS_ATTEMPT_HEADING = /^## Previous attempt$/gm;
218
+
219
+ /**
220
+ * The line every rendered record opens with — `Attempt N failed; this is
221
+ * attempt M.`, the `Attempt N failed` opening `previousAttemptOperatorAnswer`'s
222
+ * stop regex reads as a section start, and the first body line under the
223
+ * heading in every record the retry writer lays down (this node's own retry
224
+ * records included). Requiring it behind the heading keeps the split on real
225
+ * trailing record blocks: an instruction block that borrows even the exact
226
+ * heading opens its body with other words — measured on attempt 3's
227
+ * revision_cap, where `Attempt 2 failed in staging` under a borrowed heading
228
+ * was classified as a record and split out of the stable prefix.
229
+ */
230
+ const PREVIOUS_ATTEMPT_RECORD_LINE = /^Attempt \d+ failed; this is attempt \d+\./u;
231
+
232
+ /**
233
+ * Split a prompt that already carries the variable `Previous attempt` section
234
+ * into the stable instructions and that section, so assembly can re-append it
235
+ * after the stable sandbox notice and result-file protocol — cacheable prefix
236
+ * first, variable state last. The section is the block from its heading to
237
+ * the end of the prompt: every writer lays it down as the trailing block of
238
+ * whatever it was built on. `section` is null when the base carries none.
239
+ *
240
+ * @param {string} basePrompt
241
+ * @returns {{stable: string, section: string|null}}
242
+ */
243
+ export function splitPreviousAttemptSection(basePrompt) {
244
+ for (const match of basePrompt.matchAll(PREVIOUS_ATTEMPT_HEADING)) {
245
+ const rest = basePrompt.slice(match.index + match[0].length);
246
+ const bodyStart = rest.split("\n").find((line) => line.trim() !== "");
247
+ if (bodyStart !== undefined && PREVIOUS_ATTEMPT_RECORD_LINE.test(bodyStart)) {
248
+ return {
249
+ stable: basePrompt.slice(0, match.index).replace(/\n+$/u, ""),
250
+ section: basePrompt.slice(match.index).replace(/\n+$/u, ""),
251
+ };
252
+ }
253
+ }
254
+ return { stable: basePrompt, section: null };
255
+ }
256
+
205
257
  /**
206
258
  * @param {JudgeNode} node
207
259
  * @param {unknown} workerResult
@@ -256,7 +308,6 @@ export function judgePrompt(node, workerResult, context = {}) {
256
308
  `Controller diff paths:\n${diff.length ? diff.map((path) => `- ${path}`).join("\n") : "- (none)"}\n\n` +
257
309
  `Controller verification:\n${JSON.stringify(verificationResult)}\n\n` +
258
310
  (scopeSection ? `${scopeSection}\n\n` : "") +
259
- (context.previousAttempt ? `${context.previousAttempt}\n\n` : "") +
260
311
  "Return only the JSON object required by the output schema, with every field present: a verdict with nothing to report still carries `findings: []`, never an omitted key. The verdict is a record, not a report: keep `summary` within " + JUDGE_LIMITS.summaryBytes + " bytes, use at most " + JUDGE_LIMITS.findings + " findings, and keep each finding's `description` within " + JUDGE_LIMITS.descriptionBytes + " bytes and its `evidence` within " + JUDGE_LIMITS.evidenceBytes + " bytes. A verdict that overshoots this envelope is rejected unread, however sound the arbitration. Evidence must be concrete. " +
261
312
  "Use verdict pass only when findings is empty and maxSeverity is none. " +
262
313
  "Use verdict fail whenever findings is non-empty, including advisory findings below failOn. " +
@@ -264,7 +315,11 @@ export function judgePrompt(node, workerResult, context = {}) {
264
315
  ? "Arbitrate only the judgment items; deterministic items are already proven by the controller and must not be re-arbitrated. " +
265
316
  "Use pass only when every judgment item is satisfied. Every finding must cite the id of the judgment item it addresses; a fail verdict whose findings cite no id is a protocol failure."
266
317
  : "Use pass only when every Definition of Done item is satisfied. " +
267
- "Assess every Definition of Done item by its id and cite the id you are addressing in each finding.");
318
+ "Assess every Definition of Done item by its id and cite the id you are addressing in each finding.") +
319
+ // The variable state trails every stable instruction, including the
320
+ // output-schema rules, so the cacheable prefix ends at the section's
321
+ // heading and dispatch's assembly keeps it last when it re-appends.
322
+ (context.previousAttempt ? `\n\n${context.previousAttempt}` : "");
268
323
  }
269
324
 
270
325
  /**
@@ -0,0 +1,99 @@
1
+ /**
2
+ * Admission of one new paid dispatch through the campaign's optional reserve
3
+ * (ADR 0011): read the campaign record, and either admit the dispatch with a
4
+ * hold, admit it as unconfigured, or refuse it before any provider starts.
5
+ * Split from `dispatch.mjs` because the decision is one gate the worker and
6
+ * the judge paths must apply identically, and because its fail-closed rule
7
+ * needs the record read in exactly one place: a record that is present but
8
+ * cannot be read or validated refuses the dispatch, it is not waved through
9
+ * on the validation the launch performed.
10
+ */
11
+ import { existsSync } from "node:fs";
12
+ import { join } from "node:path";
13
+ import { errorMessage } from "../util.mjs";
14
+ import { CAMPAIGN_FILE } from "../campaign/layout.mjs";
15
+ import { readCampaign } from "../campaign/record.mjs";
16
+ import { releaseCampaignReservation, reserveCampaignCost } from "../campaign/reserve.mjs";
17
+
18
+ /**
19
+ * The error code a node is parked blocked with when the campaign's reserve
20
+ * refuses its new dispatch. Deliberately outside every auto-retry set: the
21
+ * balance does not recover by re-dispatching, only by an operator topping it
22
+ * up (or the in-flight holds settling) and resuming.
23
+ */
24
+ const RESERVE_INSUFFICIENT_CODE = "reserve_insufficient";
25
+
26
+ /**
27
+ * The error code a node is parked blocked with when the campaign record is
28
+ * present but cannot be read or validated. Distinct from
29
+ * `reserve_insufficient` because the operator responses differ -- a balance
30
+ * needs a top-up, a broken record needs a repair -- and like it, deliberately
31
+ * outside every auto-retry set: re-dispatching does not make a record
32
+ * readable.
33
+ */
34
+ const CAMPAIGN_RECORD_UNREADABLE_CODE = "campaign_record_unreadable";
35
+
36
+ /**
37
+ * Admit one new paid dispatch through the campaign's optional reserve
38
+ * (ADR 0011), before the provider starts. The only measured predictor of
39
+ * what the call will cost is what this node's most recent invocation actually
40
+ * charged; a first dispatch has no measurement, so it is reserved as unknown
41
+ * exposure and holds nothing -- the reserve never invents a number to gate
42
+ * with. A refusal takes no hold and carries the error the caller parks the
43
+ * node blocked with. Calls already running are never touched: the reserve
44
+ * gates only the admission of a new one.
45
+ *
46
+ * A missing campaign path or record file is the unconfigured path the reserve
47
+ * never touched, and a readable record without `reserveUsd` is the same. A
48
+ * record that is present but cannot be read or validated is the opposite of
49
+ * unconfigured: it may have changed since the launch validated it, so the
50
+ * dispatch is refused before the provider starts instead of being assumed
51
+ * safe.
52
+ *
53
+ * @param {string} campaignPath
54
+ * @param {import("../contract/index.mjs").NodeSnapshot} state
55
+ * @param {string} runId
56
+ * @param {string} nodeId
57
+ * @param {"worker"|"judge"} role
58
+ * @returns {{refused: false, reservation: import("../campaign/reserve.mjs").Reservation|null}|{refused: true, error: {code: string, message: string}}}
59
+ */
60
+ export function reserveAdmission(campaignPath, state, runId, nodeId, role) {
61
+ if (!campaignPath || !existsSync(join(campaignPath, CAMPAIGN_FILE))) return { refused: false, reservation: null };
62
+ try {
63
+ const campaign = /** @type {{reserveUsd?: number}} */ (/** @type {unknown} */ (readCampaign(campaignPath)));
64
+ if (campaign.reserveUsd === undefined) return { refused: false, reservation: null };
65
+ } catch (error) {
66
+ return { refused: true, error: {
67
+ code: CAMPAIGN_RECORD_UNREADABLE_CODE,
68
+ message: `campaign record at ${campaignPath} cannot be read or validated, refusing the new ${role} dispatch: ${errorMessage(error)}`,
69
+ } };
70
+ }
71
+ const last = state.invocations?.at(-1);
72
+ const estimateUsd = typeof last?.costUsd === "number" && Number.isFinite(last.costUsd) ? last.costUsd : null;
73
+ const decision = reserveCampaignCost(campaignPath, { runId, node: nodeId, costUsd: estimateUsd });
74
+ if (!decision.admitted) return { refused: true, error: {
75
+ code: RESERVE_INSUFFICIENT_CODE,
76
+ message: `reserve refused the new ${role} dispatch: estimated ${estimateUsd} USD cannot be held against ${decision.availableUsd ?? 0} USD available`,
77
+ } };
78
+ return { refused: false, reservation: decision.reservation };
79
+ }
80
+
81
+ /**
82
+ * Give a reservation back when the gated call never started (the provider
83
+ * could not be spawned). Never throws: a hold that cannot be released stays
84
+ * visible as held in `readCampaignReserve`, and the dispatch failure it rode
85
+ * on still settles the node.
86
+ *
87
+ * @param {string} campaignPath
88
+ * @param {import("../campaign/reserve.mjs").Reservation|null} reservation
89
+ * @returns {void}
90
+ */
91
+ export function releaseReservation(campaignPath, reservation) {
92
+ if (!campaignPath || !reservation) return;
93
+ try {
94
+ releaseCampaignReservation(campaignPath, reservation.id);
95
+ } catch {
96
+ // The hold remains recorded; it is visible in the reserve status and no
97
+ // other reservation can consume it.
98
+ }
99
+ }
@@ -15,6 +15,7 @@
15
15
  import { JUDGE_SCHEMA, SETTLED, TERMINAL } from "./prompts.mjs";
16
16
  import { Buffer } from "node:buffer";
17
17
  import { acquire as acquireLock } from "../run/lock.mjs";
18
+ import { classifyFailureCause } from "./failure-cause.mjs";
18
19
  import { applyInvalidWorkerResult, assertRunMutable, handleProviderExhaustion } from "./lifecycle.mjs";
19
20
  import { applyJudgeResult } from "./review.mjs";
20
21
  import { assertSourceUnchanged, captureRunIdentity, recordedBaseRef, statesFingerprint } from "./run-identity.mjs";
@@ -618,6 +619,7 @@ export async function reauthorRefusedNode(options) {
618
619
  const raw = JSON.parse(readFileSync(contractPath, "utf8"));
619
620
  const refused = /** @type {{missingContext?: string[]}|null|undefined} */ (state.result);
620
621
  const missingContext = Array.isArray(refused?.missingContext) ? refused.missingContext : [];
622
+ const failureCause = classifyFailureCause(node, state);
621
623
  const budget = await runBoundedRounds(rounds, async (round, history) => {
622
624
  const additions = (await discover({ round, node, state, missingContext, findings: history.flatMap((item) => item.findings) })) ?? {};
623
625
  const packet = widenTaskPacket(node.taskPacket, additions);
@@ -641,7 +643,7 @@ export async function reauthorRefusedNode(options) {
641
643
  const outcome = budget.accepted ? (reauthorApproved(node, { approve, approveBelow }) ? "applied" : "approval_required") : "rounds_exhausted";
642
644
  const record = reauthorProposalRecord({
643
645
  nodeId: node.id, outcome, riskTier: reauthorRiskTier(node), budget: rounds, roundsUsed: budget.roundsUsed,
644
- additions: last?.additions, packet: last?.packet, findings: last?.findings, history: budget.history,
646
+ additions: last?.additions, packet: last?.packet, findings: last?.findings, history: budget.history, failureCause,
645
647
  });
646
648
  appendJsonl(join(runDir, "reauthor.jsonl"), record);
647
649
  return {
@@ -23,6 +23,7 @@ import {
23
23
  import { errorMessage, excerpt } from "../util.mjs";
24
24
  import { appendTransitionEvent, transition, writeNode } from "./state.mjs";
25
25
  import { startJudge } from "./dispatch.mjs";
26
+ import { auxSpawnAdmission, recordAuxRefusal } from "./aux-admission.mjs";
26
27
  import { applyRejection, raiseNodeAttention, settleDone } from "./settle.mjs";
27
28
 
28
29
  /** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
@@ -86,6 +87,16 @@ export async function applyJudgeProtocolFailure(contract, node, state, runDir, r
86
87
  transition(runDir, state, "pending", { phase: "judge", gate: null, result: state.result, error: null, blockedBy: [] }, lock);
87
88
  return;
88
89
  }
90
+ const admission = auxSpawnAdmission(contract, node, state, running, states, "judge");
91
+ if (!admission.admitted) {
92
+ // The runtime the re-ask would land on holds its ceiling: defer the
93
+ // bounded re-ask the same way a controller-loss recovery defers it, bound
94
+ // and result durable, until the scheduler can admit it.
95
+ const from = state.status;
96
+ transition(runDir, state, "pending", { phase: "judge", gate: null, result: state.result, error: null, blockedBy: [] }, lock);
97
+ recordAuxRefusal(runDir, state, from, lock, "judge", admission);
98
+ return;
99
+ }
89
100
  await applyJudgeRound(await startJudge(contract, node, state, runDir, running, state.result, lock, states, campaignPath),
90
101
  contract, node, state, runDir, running, lock, states, campaignPath, state.result);
91
102
  }
@@ -179,6 +190,16 @@ export async function applyJudgeResult(contract, node, state, result, runDir, lo
179
190
  transition(runDir, state, "pending", { phase: "judge", gate: null, result: state.result, error: null, blockedBy: [] }, lock);
180
191
  return;
181
192
  }
193
+ const admission = auxSpawnAdmission(contract, node, state, running, states, "judge");
194
+ if (!admission.admitted) {
195
+ // The runtime the re-ask would land on holds its ceiling: defer the
196
+ // bounded re-ask with its bound and result durable until the scheduler
197
+ // can admit it, exactly as a controller-loss recovery defers it.
198
+ const from = state.status;
199
+ transition(runDir, state, "pending", { phase: "judge", gate: null, result: state.result, error: null, blockedBy: [] }, lock);
200
+ recordAuxRefusal(runDir, state, from, lock, "judge", admission);
201
+ return;
202
+ }
182
203
  await applyJudgeRound(await startJudge(contract, node, state, runDir, running, state.result, lock, states, campaignPath),
183
204
  contract, node, state, runDir, running, lock, states, campaignPath, state.result);
184
205
  return;
@@ -9,18 +9,38 @@
9
9
  *
10
10
  * The schema for what may be run lives in `contract/verification.mjs`; this is
11
11
  * only the doing.
12
+ *
13
+ * A pass interrupted mid-list restarts every command on resume (review
14
+ * 2026-09-29: an interrupted verification re-runs the whole list, and the
15
+ * engine's own suite costs 1035-1058s per full pass — `VERIFICATION_LIMITS.maxTimeoutSec`).
16
+ * Each completed command therefore also writes a checkpoint beside its log,
17
+ * `checkpoint-<ordinal>.json`: the identity hash of everything the proof
18
+ * depends on — the committed tree (the HEAD tree id) and the dirty tree
19
+ * fingerprint, the validated command, the exact child environment, the
20
+ * dependency-inputs hash — plus the bounded result. Both tree hashes are
21
+ * needed: the fingerprint hashes only the difference from HEAD, so clean
22
+ * trees at different commits share one, and a stale passing result survived a
23
+ * commit until the HEAD tree joined the identity (review 2026-10-02, d5.1). A
24
+ * resumed pass serves a checkpoint only when that identity is unchanged and
25
+ * the tree actually carried a fingerprint; any difference forces
26
+ * re-execution. A
27
+ * mutation entry is the explicit invalidation policy: it is never checkpointed
28
+ * and never served, because its proof is a function of the tree it breaks and
29
+ * restores, so a reused record could certify a tree that no longer exists.
12
30
  */
13
31
  import { Buffer } from "node:buffer";
14
- import { VERIFICATION_LIMITS, compactVerification, resolveVerificationCwd, validateVerificationCommands } from "../contract/verification.mjs";
15
- import { mkdirSync, writeFileSync } from "node:fs";
32
+ import { VERIFICATION_LIMITS, commandCheckpointIdentity, compactVerification, parseCommandCheckpoint, resolveVerificationCwd, validateVerificationCommands } from "../contract/verification.mjs";
33
+ import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
16
34
  import { processStartToken } from "../run/lock.mjs";
17
- import { randomUUID } from "node:crypto";
35
+ import { createHash, randomUUID } from "node:crypto";
18
36
  import { runMutation } from "./mutation.mjs";
19
37
  import { SANDBOX_BLOCKED_WRITE } from "../contract/worker-result.mjs";
20
- import { isAbsolute, resolve } from "node:path";
38
+ import { isAbsolute, join, resolve } from "node:path";
21
39
  import { isContained } from "../util.mjs";
22
40
  import { spawn } from "node:child_process";
23
41
  import { killTarget, spawnInvocation } from "../host/platform.mjs";
42
+ import { gitIdentity } from "../repo/source-identity.mjs";
43
+ import { boundedGitSync } from "../repo/worktree.mjs";
24
44
  /** @typedef {import("../contract/verification.mjs").VerificationOptions} VerificationOptions */
25
45
 
26
46
  /**
@@ -76,14 +96,15 @@ const VERIFICATION_ENV_BASE_NAMES = Object.freeze([
76
96
 
77
97
  /**
78
98
  * @param {VerificationCommand} command
79
- * @returns {Record<string, string|undefined>}
99
+ * @returns {Record<string, string>}
80
100
  */
81
101
  function verificationEnv(command) {
82
102
  const names = new Set([...(command.env ?? []), ...VERIFICATION_ENV_BASE_NAMES]);
83
- /** @type {Record<string, string|undefined>} */
103
+ /** @type {Record<string, string>} */
84
104
  const env = {};
85
105
  for (const name of names) {
86
- if (process.env[name] !== undefined) env[name] = process.env[name];
106
+ const value = process.env[name];
107
+ if (value !== undefined) env[name] = value;
87
108
  }
88
109
  return env;
89
110
  }
@@ -126,6 +147,123 @@ export function classifySandboxBlockedWrite({ text, workspace, mode }) {
126
147
  }
127
148
  return null;
128
149
  }
150
+ /**
151
+ * Everything one pass needs to decide checkpoint reuse, cached per pass: the
152
+ * tree identity per command cwd (one `gitIdentity` and one `rev-parse` each,
153
+ * paid only when a checkpoint directory is in play) and the dependency-inputs
154
+ * hash (the packet's writeFiles do not change during a pass; a mutation entry
155
+ * restores them in `finally`).
156
+ *
157
+ * @typedef {{trees: Map<string, WorkspaceTreeIdentity>, dependencyHash: string|undefined}} CheckpointCaches
158
+ */
159
+
160
+ /**
161
+ * The two tree hashes a checkpoint identity sits on: the dirty fingerprint
162
+ * (what the working tree adds to HEAD) and the HEAD tree id (what HEAD
163
+ * commits). Either changing forces re-execution.
164
+ *
165
+ * @typedef {{dirtyTreeFingerprint: string|null, headTree: string|null}} WorkspaceTreeIdentity
166
+ */
167
+
168
+ /**
169
+ * The committed tree the workspace sits on: the HEAD tree id, which the dirty
170
+ * fingerprint cannot see — it hashes only the difference from HEAD, so two
171
+ * clean trees at different commits used to share one checkpoint identity and
172
+ * a stale pass survived a commit (review 2026-10-02, d5.1). Null when there
173
+ * is no commit to name (no repository, unborn HEAD): the identity still
174
+ * compares, and null to null is the same unborn state across passes.
175
+ *
176
+ * @param {string} workspace
177
+ * @returns {string|null}
178
+ */
179
+ function headTreeId(workspace) {
180
+ try {
181
+ const result = boundedGitSync(["-C", workspace, "rev-parse", "HEAD^{tree}"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
182
+ if (result.error || result.status !== 0) return null;
183
+ return String(result.stdout).trim() || null;
184
+ } catch {
185
+ // Git absent or unreadable: no committed tree to name, matching the
186
+ // dirty fingerprint's own null for a workspace without a repository.
187
+ return null;
188
+ }
189
+ }
190
+
191
+ /**
192
+ * A hash over the dependency inputs a proof is about: the packet's writeFiles,
193
+ * the mutation sample's input. ReadFiles and every other tracked or untracked
194
+ * source ride inside the tree fingerprint instead; installed dependencies that
195
+ * git ignores (a mid-attempt `npm ci`) are covered only when the packet
196
+ * declares them here — a known bound, not a solved one.
197
+ *
198
+ * @param {string} baseCwd
199
+ * @param {string[]} writeFiles
200
+ * @returns {string}
201
+ */
202
+ function dependencyInputsHash(baseCwd, writeFiles) {
203
+ const hash = createHash("sha256");
204
+ for (const path of [...new Set(writeFiles)].sort()) {
205
+ hash.update(`${path}\0`);
206
+ try {
207
+ hash.update(readFileSync(resolve(baseCwd, path)));
208
+ } catch {
209
+ // ENOENT or a non-file: the mutation sampler skips the same paths, so a
210
+ // missing dependency input hashes as missing rather than failing the pass.
211
+ hash.update("missing");
212
+ }
213
+ hash.update("\0");
214
+ }
215
+ return hash.digest("hex");
216
+ }
217
+
218
+ /**
219
+ * The pre-run identity of one command's proof, plus whether the tree gave a
220
+ * fingerprint to compare at all: a workspace that is not a git repository has
221
+ * no tree identity, so its checkpoints never serve.
222
+ *
223
+ * @param {VerificationCommand} command
224
+ * @param {string} baseCwd
225
+ * @param {RunVerificationOptions} options
226
+ * @param {CheckpointCaches} caches
227
+ * @returns {{identity: string, reusable: boolean}}
228
+ */
229
+ function checkpointIdentity(command, baseCwd, options, caches) {
230
+ const workspace = resolveVerificationCwd(baseCwd, command.cwd ?? ".");
231
+ let tree = caches.trees.get(workspace);
232
+ if (tree === undefined) {
233
+ tree = { dirtyTreeFingerprint: gitIdentity(workspace).dirtyTreeFingerprint, headTree: headTreeId(workspace) };
234
+ caches.trees.set(workspace, tree);
235
+ }
236
+ if (caches.dependencyHash === undefined) {
237
+ caches.dependencyHash = dependencyInputsHash(baseCwd, options.writeFiles ?? []);
238
+ }
239
+ return {
240
+ identity: commandCheckpointIdentity({ treeFingerprint: tree.dirtyTreeFingerprint, headTree: tree.headTree, command, env: verificationEnv(command), dependencyHash: caches.dependencyHash }),
241
+ reusable: tree.dirtyTreeFingerprint !== null,
242
+ };
243
+ }
244
+
245
+ /**
246
+ * The recorded result for this ordinal, served only when the checkpoint
247
+ * validates, the tree carried a fingerprint, and the identity is unchanged.
248
+ * Every read or parse failure lands on re-execution: a corrupted checkpoint is
249
+ * never a verdict on the command and never fatal to the pass.
250
+ *
251
+ * @param {string} logDir
252
+ * @param {number} ordinal
253
+ * @param {{identity: string, reusable: boolean}} identity
254
+ * @returns {VerificationCommandResult|null}
255
+ */
256
+ function servedCheckpoint(logDir, ordinal, identity) {
257
+ try {
258
+ const parsed = parseCommandCheckpoint(JSON.parse(readFileSync(join(logDir, `checkpoint-${ordinal}.json`), "utf8")));
259
+ if (!identity.reusable || parsed.identity !== identity.identity) return null;
260
+ return parsed.result;
261
+ } catch {
262
+ // ENOENT is the no-checkpoint case; every other failure (corrupt JSON, a
263
+ // record past its bounds) fails closed to a re-run for the same reason.
264
+ return null;
265
+ }
266
+ }
129
267
  /**
130
268
  * Run every declared command `repeat` times inside the workspace.
131
269
  *
@@ -138,14 +276,31 @@ export async function runVerification(commands, baseCwd, options = {}) {
138
276
  const validated = validateVerificationCommands(commands);
139
277
  /** @type {VerificationCommandResult[]} */
140
278
  const results = [];
279
+ /** @type {CheckpointCaches|null} */
280
+ const checkpoints = options.logDir ? { trees: new Map(), dependencyHash: undefined } : null;
141
281
  for (const [commandIndex, command] of validated.entries()) {
142
- const result = command.mutation
143
- ? await runMutationCommand(command, baseCwd, commandIndex, options)
144
- : await runRepeatedCommand(command, baseCwd, commandIndex, options);
282
+ // Identity before the run: a checkpoint records the tree, environment and
283
+ // dependency inputs the proof was made against, which are the pre-run ones.
284
+ const identity = checkpoints && !command.mutation
285
+ ? checkpointIdentity(command, baseCwd, options, checkpoints)
286
+ : null;
287
+ const reused = identity
288
+ ? servedCheckpoint(/** @type {string} */ (options.logDir), results.length + 1, identity)
289
+ : null;
290
+ const result = reused ?? await (command.mutation
291
+ ? runMutationCommand(command, baseCwd, commandIndex, options)
292
+ : runRepeatedCommand(command, baseCwd, commandIndex, options));
145
293
  results.push(result);
146
294
  if (options.logDir) {
147
295
  mkdirSync(options.logDir, { recursive: true });
148
- writeFileSync(`${options.logDir}/verification-${results.length}.json`, `${JSON.stringify(compactVerification({ passed: result.passed, commands: [result] }))}\n`, { mode: 0o600 });
296
+ const compacted = compactVerification({ passed: result.passed, commands: [result] });
297
+ writeFileSync(`${options.logDir}/verification-${results.length}.json`, `${JSON.stringify(compacted)}\n`, { mode: 0o600 });
298
+ // A mutation entry records no checkpoint (the invalidation policy in the
299
+ // header); an argv past the persisted bounds compacts to no command, and
300
+ // its unreadable record simply fails closed to a re-run on resume.
301
+ if (identity) {
302
+ writeFileSync(`${options.logDir}/checkpoint-${results.length}.json`, `${JSON.stringify({ identity: identity.identity, result: compacted })}\n`, { mode: 0o600 });
303
+ }
149
304
  }
150
305
  }
151
306
  return { passed: results.every((result) => result.passed), commands: results };