faberun 0.30.0 → 0.31.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/package.json +1 -1
  2. package/src/campaign/campaign-brief.mjs +17 -6
  3. package/src/campaign/chain.mjs +124 -8
  4. package/src/campaign/ledger.mjs +19 -7
  5. package/src/campaign/metrics-command.mjs +68 -9
  6. package/src/campaign/metrics.mjs +75 -45
  7. package/src/campaign/record.mjs +8 -1
  8. package/src/campaign/reserve.mjs +340 -0
  9. package/src/campaign/spend.mjs +229 -0
  10. package/src/campaign/unpark.mjs +54 -1
  11. package/src/campaign/watch.mjs +314 -37
  12. package/src/contract/judge-envelope.mjs +17 -2
  13. package/src/contract/scope-findings.mjs +3 -1
  14. package/src/contract/task-packet.mjs +40 -5
  15. package/src/contract/verification.mjs +67 -1
  16. package/src/engine/aux-admission.mjs +124 -0
  17. package/src/engine/capacity.mjs +78 -0
  18. package/src/engine/dispatch.mjs +116 -23
  19. package/src/engine/failure-cause.mjs +59 -0
  20. package/src/engine/lifecycle.mjs +3 -2
  21. package/src/engine/phase-session.mjs +56 -10
  22. package/src/engine/prompts.mjs +57 -2
  23. package/src/engine/reserve-admission.mjs +99 -0
  24. package/src/engine/resume.mjs +3 -1
  25. package/src/engine/review.mjs +21 -0
  26. package/src/engine/run-command.mjs +166 -11
  27. package/src/engine/scheduler.mjs +71 -88
  28. package/src/engine/settle.mjs +15 -8
  29. package/src/engine/settlement-overlap.mjs +143 -0
  30. package/src/engine/verify.mjs +131 -7
  31. package/src/harnesses/session-metrics.mjs +112 -20
  32. package/src/notify/index.mjs +86 -10
  33. package/src/plan/freeze.mjs +7 -2
  34. package/src/plan/human-step.mjs +36 -0
  35. package/src/plan/pipeline.mjs +53 -22
  36. package/src/plan/preflight.mjs +81 -5
  37. package/src/plan/progress.mjs +101 -1
  38. package/src/plan/repo-facts.mjs +403 -49
  39. package/src/plan/resolve.mjs +7 -0
  40. package/src/plan/revise-context.mjs +108 -0
  41. package/src/plan/rounds.mjs +180 -45
  42. package/src/plan/sizing.mjs +33 -1
  43. package/src/plan/template.mjs +134 -22
  44. package/src/repo/integrate.mjs +47 -5
  45. package/src/report/campaign-brief-estimate.mjs +47 -1
  46. package/src/report/campaign-brief.mjs +27 -0
  47. package/src/report/locale.mjs +20 -0
  48. package/src/report/metrics-report.mjs +38 -0
  49. package/src/report/progress.mjs +194 -10
  50. package/src/run/usage-windows.mjs +99 -4
  51. package/src/run/usage.mjs +69 -7
  52. package/src/web/api.mjs +179 -24
  53. package/src/web/app.mjs +27 -2
@@ -8,7 +8,7 @@ import {
8
8
  } from "./review.mjs";
9
9
  import { CONTRACT_VERSION, PROTOCOL_SCHEMA_VERSION } from "../harnesses/index.mjs";
10
10
  import { routeRuntimeForState, routingBackoffActive } from "./failover.mjs";
11
- import { quotaHeldRuntimes, runningPerRuntime, runtimeHasCapacity } from "./capacity.mjs";
11
+ import { admissionPass } from "./capacity.mjs";
12
12
 
13
13
  import {
14
14
  bootstrapAttemptPath,
@@ -45,6 +45,7 @@ import { validateContractForLaunch } from "../campaign/chain.mjs";
45
45
  import { finalVerificationCommands, gateProofTimeoutMs, sharedVerificationCommands } from "../contract/final-verification.mjs";
46
46
  import { startJudge, startWorker } from "./dispatch.mjs";
47
47
  import { raiseNodeAttention } from "./settle.mjs";
48
+ import { createSettlementOverlap } from "./settlement-overlap.mjs";
48
49
  import { HUMAN_STEP_ERROR_CODE, humanStepAttentionMessage } from "../contract/human-step.mjs";
49
50
  import { assertEnvironmentReady, captureRunIdentity, createRunMetadata, serializableContract, statesFingerprint } from "./run-identity.mjs";
50
51
  import { blockDependents, runtimeAssignments } from "./assignment.mjs";
@@ -429,17 +430,24 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
429
430
  // tick's critical path so an eligible sibling still dispatches into a free
430
431
  // slot while this node's own invocation has already exited. A node's own
431
432
  // steps stay ordered because only one settlement per node is ever in flight
432
- // (see the dispatch loop below); a *different* node's settlement is queued
433
- // behind whichever one is already running (`settlementQueue`), not run
434
- // alongside it -- `finalizeClosedJobs` shares state a concurrent second call
435
- // would corrupt: the phase-continuation selection that picks at most one
436
- // live node to carry a session forward, and `repo/integrate.mjs`'s one
437
- // candidate ref and worktree per run. Only *dispatching a sibling* skips
438
- // ahead of a node's settlement; two nodes' settlements never interleave.
439
- /** @type {Map<string, Promise<void>>} */
440
- const pendingSettlements = new Map();
441
- /** @type {Promise<void>} */
442
- let settlementQueue = Promise.resolve();
433
+ // (see the dispatch loop below). Different nodes' settlements overlap in
434
+ // exactly one place, their verification proofs (`parkDuringProof` in
435
+ // `engine/settlement-overlap.mjs`), and nowhere else: the sections between
436
+ // proofs run one at a time, because `finalizeClosedJobs` shares state a
437
+ // concurrent second call would corrupt -- the dispatch decisions that count
438
+ // `running` against `maxParallel`, the phase-continuation selection that
439
+ // picks at most one live node to carry a session forward, and
440
+ // `repo/integrate.mjs`'s one candidate ref and worktree per run. Only
441
+ // *dispatching a sibling* otherwise skips ahead of a node's settlement.
442
+ const settlements = createSettlementOverlap();
443
+ const pendingSettlements = settlements.pending;
444
+ // F5 admission's settle window: nodes whose job closed but whose settlement
445
+ // has not finished still hold the global and runtime slot their dispatch
446
+ // reserved. Released when the settlement itself settles, not at close --
447
+ // released at close, a sibling took the slot during the node's verification
448
+ // gate and a judge ran on top of it (2026-09-29, achado 2, peak 2 under
449
+ // `maxParallel: 1`; judged by `admissionPass` in capacity.mjs).
450
+ const settling = settlements.settling;
443
451
  let canceled = false;
444
452
  const cancel = () => { canceled = true; };
445
453
  process.once("SIGINT", cancel);
@@ -459,57 +467,12 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
459
467
  .map((state) => ({ nodeId: state.id, budgetBasis: nodeBudgetBasisMs(contract, /** @type {ValidatedNode} */ (heartbeatNodes.get(state.id))) }));
460
468
  // A programmer error surfacing inside a background settlement must still
461
469
  // crash the whole run, exactly as an unguarded `await finalizeClosedJobs`
462
- // used to: it is recorded here and thrown from the top of the loop on the
463
- // very next tick, rather than immediately, so it cannot itself become the
464
- // block a sibling's dispatch is waiting behind. A lost lock is not this --
470
+ // used to: the overlap records it (`settlements.failure()`) and it is
471
+ // thrown from the top of the loop on the very next tick, rather than
472
+ // immediately, so it cannot itself become the block a sibling's dispatch is
473
+ // waiting behind. A lost lock is not this --
465
474
  // the loop's own `lock.assert()` calls surface that same condition on their
466
475
  // own schedule, so it is left for them.
467
- /** @type {unknown} */
468
- let backgroundSettlementFailure = null;
469
- // Mark every job that closed this tick as settling, and free its slot,
470
- // without waiting for any of them: that alone is what lets an eligible
471
- // sibling dispatch into the freed slot while this node's minutes-long
472
- // controller verification or judge round is still running. The actual
473
- // settlement work is chained onto `settlementQueue`, one node at a time in
474
- // the order its job closed, so it still runs exactly as serialized against
475
- // every *other* node's settlement as it did when this loop awaited
476
- // `finalizeClosedJobs` directly -- `finalizeClosedJobs` runs against a
477
- // one-entry map per node, so this is one call per node rather than the one
478
- // batched call it used to be, but the chain still runs them one at a time.
479
- // A settlement may itself dispatch the node's next phase (a judge, a
480
- // revision) through the same `startJudge`/`startWorker` calls dispatch below
481
- // uses, so it is handed the real `running` to dispatch into: a job it starts
482
- // is counted against `maxParallel` from the instant the process exists, and
483
- // `applyRejection` reads that same map to decide whether the run has a slot
484
- // for the revision at all. It used to dispatch into the throwaway one-entry
485
- // map instead, copied back only once the settlement returned, which is how
486
- // run state-location-and-routing-economics-13 came to hold two workers under
487
- // `maxParallel: 1` on 2026-09-21. What it may *settle* is still only its own
488
- // node: the one-entry map below is the job, not the run. The node's own
489
- // steps stay ordered by never starting a second settlement for a node whose
490
- // first has not yet cleared `pendingSettlements`.
491
- const settleClosedJobsInBackground = () => {
492
- for (const [nodeId, job] of [...running]) {
493
- if (pendingSettlements.has(nodeId) || !job.closed || invocationAlive(job.invocation)) continue;
494
- running.delete(nodeId);
495
- const settlement = settlementQueue
496
- .then(() => finalizeClosedJobs(contract, runDir, states, new Map([[nodeId, job]]), lock, campaign.path, running))
497
- .finally(() => pendingSettlements.delete(nodeId));
498
- // The queue itself must never reject -- a rejected settlement (a lost
499
- // lock, a programmer error) would otherwise wedge every node queued
500
- // behind it. The rejection still reaches whoever awaits the real
501
- // `settlement` promise (`pendingSettlements`, below).
502
- settlementQueue = settlement.catch(() => {});
503
- // Handled here so an in-flight settlement never becomes an unhandled
504
- // rejection when nobody happens to await `pendingSettlements` before the
505
- // process exits; the original promise, still held below, carries the
506
- // rejection to whichever checkpoint (cancel, shutdown) awaits it.
507
- settlement.catch((error) => {
508
- if (!(error instanceof LockLostError) && backgroundSettlementFailure === null) backgroundSettlementFailure = error;
509
- });
510
- pendingSettlements.set(nodeId, settlement);
511
- }
512
- };
513
476
  // Captured once, before the loop, rather than re-derived every tick: it is
514
477
  // "parked when this controller invocation started" (autoRetryParkedNodes's
515
478
  // own contract), and a node a background settlement parks between two ticks
@@ -531,7 +494,7 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
531
494
  // before the loop is allowed to end.
532
495
  if (anyUnsettled()) for (;;) {
533
496
  lock.assert();
534
- if (backgroundSettlementFailure !== null) throw backgroundSettlementFailure;
497
+ if (settlements.failure() !== null) throw settlements.failure();
535
498
  if (existsSync(join(runDir, "cancel.request.json"))) canceled = true;
536
499
  if (canceled) {
537
500
  // A settlement already in flight owns the one decision a cancellation
@@ -569,9 +532,34 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
569
532
  // running, not only once it closes. `finalizeClosedJobs` used to open
570
533
  // with this same check; calling it once here, rather than once per
571
534
  // node settled this tick, is what keeps it at one pass per tick now
572
- // that settlement runs per node in `settleClosedJobsInBackground`.
535
+ // that settlement runs per node below.
573
536
  await emitNodeAdvisories(contract, runDir, states);
574
- settleClosedJobsInBackground();
537
+ // Mark every job that closed this tick as settling -- its admission slot
538
+ // moves from `running` into `settling` (see the settle window above) --
539
+ // without waiting for any of them. The settlement work is handed to the
540
+ // overlap one node at a time in the order its job closed: its sections
541
+ // are serialized against every *other* node's exactly as they were when
542
+ // this loop awaited `finalizeClosedJobs` directly, and only its
543
+ // verification proof is parked out of the slot while it runs (see
544
+ // `engine/settlement-overlap.mjs`). A settlement
545
+ // may itself dispatch the node's next phase (a judge, a revision)
546
+ // through the same `startJudge`/`startWorker` calls dispatch below uses,
547
+ // so it is handed the real `running` to dispatch into: a job it starts
548
+ // is counted against `maxParallel` from the instant the process exists,
549
+ // and `applyRejection` reads that same map to decide whether the run has
550
+ // a slot for the revision at all. It used to dispatch into the throwaway
551
+ // one-entry map instead, copied back only once the settlement returned,
552
+ // which is how run state-location-and-routing-economics-13 came to hold
553
+ // two workers under `maxParallel: 1` on 2026-09-21. What it may *settle*
554
+ // is still only its own node: the one-entry map below is the job, not
555
+ // the run. The node's own steps stay ordered by never starting a second
556
+ // settlement for a node whose first has not yet cleared
557
+ // `pendingSettlements`.
558
+ for (const [nodeId, job] of [...running]) {
559
+ if (pendingSettlements.has(nodeId) || !job.closed || invocationAlive(job.invocation)) continue;
560
+ running.delete(nodeId);
561
+ settlements.enqueue(nodeId, job, () => finalizeClosedJobs(contract, runDir, states, new Map([[nodeId, job]]), lock, campaign.path, running));
562
+ }
575
563
  await detectStalls(contract, running, async (job, status, error) => {
576
564
  const envelope = recordInvocationUsage(job);
577
565
  job.state.usage = invocationUsage(job.state);
@@ -642,44 +630,39 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
642
630
  }, lock);
643
631
  await raiseNodeAttention(campaign.path, runDir, /** @type {NodeSnapshot} */ (states.get(node.id)), HUMAN_STEP_ERROR_CODE);
644
632
  }
645
- const slots = contract.maxParallel - running.size;
646
- if (slots > 0) {
633
+ // F5 admission is judged against every attempt holding a slot: the live
634
+ // jobs plus the closed jobs still inside their settle window. The
635
+ // contract's `maxParallel` bounds the total; a spawn the settlement
636
+ // starts itself (the node's judge) runs inside the node's own held slot.
637
+ const admission = admissionPass(contract, running, settling, states);
638
+ if (admission.slots > 0) {
647
639
  const dispatchable = ready.filter((node) => !node.humanStep);
648
- // Per-runtime capacity is judged per dispatch, not per tick: the
649
- // counts include what this tick has already started, and a runtime a
650
- // sibling is waiting out a quota reset on accepts nothing new.
651
- const counts = runningPerRuntime(running.values());
652
- const held = quotaHeldRuntimes(states.values(), Date.now());
653
640
  let dispatched = 0;
654
641
  for (const node of dispatchable) {
655
- if (dispatched >= slots) break;
642
+ if (dispatched >= admission.slots) break;
656
643
  const state = states.get(node.id);
657
644
  if (!state || routingBackoffActive(state, state.phase)) continue;
658
645
  const routed = routeRuntimeForState(contract, node, state, state.phase === "judge" ? "judge" : "worker");
659
- if (!runtimeHasCapacity(routed.id, contract, counts, held)) continue;
660
- counts.set(routed.id, (counts.get(routed.id) ?? 0) + 1);
646
+ if (!admission.take(routed.id)) continue;
661
647
  dispatched += 1;
662
648
  // A node recovered pending a re-ask judge (its own worker attempt
663
649
  // already accepted, `state.result` durable) reaches `settleDone` /
664
650
  // `integrateAttempt` exactly like a closed job's own settlement
665
- // does, on the same per-run candidate ref and worktree
666
- // `settlementQueue` exists to serialize -- so it is dispatched the
667
- // same way: chained onto the queue rather than awaited here, using
651
+ // does, on the same per-run candidate ref and worktree the
652
+ // overlap's serialized sections exist to protect -- so it is
653
+ // dispatched the same way: handed to the overlap rather than
654
+ // awaited here, using
668
655
  // `running` itself as its dispatch map, so the judge it starts is
669
656
  // counted the instant it exists. The node's own order is untouched
670
657
  // (still one entry at a time, gated by `pendingSettlements`); only
671
658
  // the tick stops waiting behind it.
672
659
  if (state.phase === "judge" && state.result) {
673
660
  const workerResult = state.result;
674
- const settlement = settlementQueue
675
- .then(() => startJudge(contract, node, state, runDir, running, workerResult, lock, states, campaign.path))
676
- .then((round) => applyJudgeRound(round, contract, node, state, runDir, running, lock, states, campaign.path, workerResult))
677
- .finally(() => pendingSettlements.delete(node.id));
678
- settlementQueue = settlement.catch(() => {});
679
- settlement.catch((error) => {
680
- if (!(error instanceof LockLostError) && backgroundSettlementFailure === null) backgroundSettlementFailure = error;
681
- });
682
- pendingSettlements.set(node.id, settlement);
661
+ // Counted twice while the judge job is live -- in `settling` and
662
+ // in `running` -- which can only hold a sibling back.
663
+ settlements.enqueue(node.id, { runtime: routed }, () =>
664
+ startJudge(contract, node, state, runDir, running, workerResult, lock, states, campaign.path)
665
+ .then((round) => applyJudgeRound(round, contract, node, state, runDir, running, lock, states, campaign.path, workerResult)));
683
666
  continue;
684
667
  }
685
668
  state.attempt += 1;
@@ -734,12 +717,12 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
734
717
  // instant it settles, win or lose -- so a rejection recorded here can
735
718
  // already be gone from the map above by the time this line runs, with
736
719
  // nothing left to await it. The loop's own top-of-tick check
737
- // (`if (backgroundSettlementFailure !== null) throw ...`) cannot save
720
+ // (`if (settlements.failure() !== null) throw ...`) cannot save
738
721
  // that case either: the loop has already exited. Checked again here,
739
722
  // once, so a background settlement failure can never read as a clean
740
723
  // finish just because it settled on the same tick the run's last node
741
724
  // did.
742
- if (backgroundSettlementFailure !== null) throw backgroundSettlementFailure;
725
+ if (settlements.failure() !== null) throw settlements.failure();
743
726
  } catch (error) {
744
727
  if (!(error instanceof LockLostError)) throw error;
745
728
  // A settlement still merges whatever it started into `running` in its own
@@ -16,6 +16,7 @@ import { candidateOnlyFailures, resetPhaseRouting, verificationFailureVerdict }
16
16
  import { retryPrompt } from "./prompts.mjs";
17
17
  import { renderPreviousAttemptSection } from "./retry.mjs";
18
18
  import { startWorker } from "./dispatch.mjs";
19
+ import { auxSpawnAdmission, recordAuxRefusal } from "./aux-admission.mjs";
19
20
  import { ensureTerminalEvent, transition, writeNode } from "./state.mjs";
20
21
  import { verificationFailureWithScope } from "../contract/scope-findings.mjs";
21
22
  import { alreadyNotified, notifyQueueFor } from "./notify-queue.mjs";
@@ -70,14 +71,18 @@ export function applyRejection(contract, node, state, runDir, running, lock, sta
70
71
  // path) carry it.
71
72
  state.previousAttempt = renderPreviousAttemptSection(state) ?? state.previousAttempt;
72
73
  // A revision is a new attempt, and a new attempt is a dispatch: it starts
73
- // here only against a slot the run actually has free. `running` is the
74
- // scheduler's own map, with this node's own closed job already out of it,
75
- // so `running.size` is exactly what the dispatch loop's own
76
- // `maxParallel - running.size` will read. A settlement that dispatched
77
- // regardless is how run state-location-and-routing-economics-13 ran two
78
- // workers under `maxParallel: 1` on 2026-09-21, and two of that day's
79
- // three OOM kills happened with more running than the contract declared.
80
- if (running && running.size < contract.maxParallel) {
74
+ // here only against a slot the run actually has free -- the global
75
+ // `maxParallel` and the routed runtime's own ceiling, the same judgment a
76
+ // dispatch pass applies (`auxSpawnAdmission`). `running` is the
77
+ // scheduler's own map, with this node's own closed job already out of it.
78
+ // A settlement that dispatched regardless is how run
79
+ // state-location-and-routing-economics-13 ran two workers under
80
+ // `maxParallel: 1` on 2026-09-21, and two of that day's three OOM kills
81
+ // happened with more running than the contract declared.
82
+ const admission = running
83
+ ? auxSpawnAdmission(contract, node, state, running, states, "worker")
84
+ : null;
85
+ if (admission?.admitted && running) {
81
86
  // Dispatching here owns the increment, because `startWorker` expects the
82
87
  // attempt number it is about to run under.
83
88
  state.attempt += 1;
@@ -92,7 +97,9 @@ export function applyRejection(contract, node, state, runDir, running, lock, sta
92
97
  // killed controller — a node that ran twice reported attempt 3, with no
93
98
  // `…2.*` logs and a `worktree.previousAttempt` naming an attempt that
94
99
  // never existed.
100
+ const from = state.status;
95
101
  transition(runDir, state, "pending", { phase: "worker", error: null }, lock);
102
+ if (admission) recordAuxRefusal(runDir, state, from, lock, "worker", admission);
96
103
  return;
97
104
  }
98
105
  transition(runDir, state, node.gate.enabled ? "exhausted" : "failed", {
@@ -0,0 +1,143 @@
1
+ /**
2
+ * The settlement overlap: what makes two closed nodes' verification proofs run
3
+ * at the same time through one real run, while everything a concurrent second
4
+ * settlement would corrupt stays serialized.
5
+ *
6
+ * The verify layer's proof gate (`engine/verify.mjs`) already bounds concurrent
7
+ * passes by the contract's `maxParallel` and serializes passes that share a
8
+ * tree; what held the engine back was the caller. `driveRun` chained whole
9
+ * settlements onto one promise queue, so one node's controller verification --
10
+ * minutes of child processes -- held every other node's settlement behind it
11
+ * and two independent worktree proofs never overlapped (phase-5 d4.1). This
12
+ * module replaces that queue: a settlement is a sequence of serialized
13
+ * *sections* separated by *proofs*. A section holds the run's one settlement
14
+ * slot; `parkDuringProof`, called around the controller verification pass,
15
+ * hands the slot back while the proof's child processes run and re-acquires it
16
+ * before the settlement's next decision. Dispatch decisions, the
17
+ * phase-continuation choice in `finalizeClosedJobs`, and ref updates and the
18
+ * whole integration transaction therefore never interleave between two nodes,
19
+ * while the proofs between them do. The integration candidate's own
20
+ * verification is deliberately never parked: it runs inside
21
+ * `repo/integrate.mjs`'s candidate lock, and parking it would deadlock -- its
22
+ * settlement would wait to re-acquire the slot behind a sibling whose section
23
+ * is itself waiting on the candidate lock this settlement holds -- and
24
+ * integration is exactly what phase 5 keeps serialized.
25
+ *
26
+ * Per-node ordering needs no lock here: the scheduler never starts a second
27
+ * settlement for a node whose first is still in `pending`, and the admission
28
+ * settle window (`settling`) holds each node's slot from its job's close until
29
+ * its settlement finishes, parked proof included.
30
+ */
31
+ import { AsyncLocalStorage } from "node:async_hooks";
32
+ import { LockLostError } from "../run/lock.mjs";
33
+
34
+ /** @typedef {import("./lifecycle.mjs").Job} Job */
35
+
36
+ /** @typedef {{park: (factory: () => Promise<unknown>) => Promise<unknown>}} SettlementContext */
37
+
38
+ /** The overlap of the settlement currently running on this controller, if any. @type {AsyncLocalStorage<SettlementContext>} */
39
+ const settlementContext = new AsyncLocalStorage();
40
+
41
+ /**
42
+ * Run a verification proof outside the caller's serialized section: while the
43
+ * proof's child processes run, the settlement's slot is handed back and a
44
+ * sibling's settlement advances. Outside a settlement -- a recovery path, a
45
+ * test driving `executeControllerVerification` directly -- the proof runs
46
+ * exactly inline, unchanged.
47
+ *
48
+ * @template T
49
+ * @param {() => Promise<T>|T} factory
50
+ * @returns {Promise<T>}
51
+ */
52
+ export async function parkDuringProof(factory) {
53
+ const overlap = settlementContext.getStore();
54
+ if (!overlap) return /** @type {T} */ (await factory());
55
+ return /** @type {Promise<T>} */ (overlap.park(/** @type {() => Promise<unknown>} */ (factory)));
56
+ }
57
+
58
+ /**
59
+ * @returns {{pending: Map<string, Promise<void>>, settling: Map<string, Pick<Job, "runtime">>, enqueue: (nodeId: string, job: Pick<Job, "runtime">, work: () => Promise<void>) => void, failure: () => unknown}}
60
+ */
61
+ export function createSettlementOverlap() {
62
+ /** @type {Map<string, Promise<void>>} */
63
+ const pending = new Map();
64
+ /** @type {Map<string, Pick<Job, "runtime">>} */
65
+ const settling = new Map();
66
+ /** @type {unknown} */
67
+ let failure = null;
68
+ let sectionBusy = false;
69
+ /** @type {(() => void)[]} */
70
+ const sectionWaiters = [];
71
+ /** @returns {Promise<void>} */
72
+ const acquireSection = () => new Promise((satisfy) => {
73
+ if (!sectionBusy) {
74
+ sectionBusy = true;
75
+ satisfy();
76
+ return;
77
+ }
78
+ sectionWaiters.push(satisfy);
79
+ });
80
+ // Hand-off, not vacancy: the next waiter is granted the slot directly, so a
81
+ // caller that acquires between the release and the grant joins the queue
82
+ // behind it instead of slipping past.
83
+ /** @returns {void} */
84
+ const releaseSection = () => {
85
+ const next = sectionWaiters.shift();
86
+ if (next) next();
87
+ else sectionBusy = false;
88
+ };
89
+ /**
90
+ * Hand the slot back while a proof runs and take it again before the next
91
+ * decision. Releasing is safe exactly because the proof holds no shared
92
+ * settlement state: its writes go to its own node's snapshot, its child
93
+ * processes run in their own worktree, and its overlap with sibling proofs
94
+ * is bounded by the verify layer's proof gate.
95
+ *
96
+ * @template T
97
+ * @param {() => Promise<T>} factory
98
+ * @returns {Promise<T>}
99
+ */
100
+ const park = async (factory) => {
101
+ releaseSection();
102
+ try {
103
+ return await factory();
104
+ } finally {
105
+ await acquireSection();
106
+ }
107
+ };
108
+ return {
109
+ pending,
110
+ settling,
111
+ /**
112
+ * Chain one node's settlement onto the overlap. The reservation semantics
113
+ * are the scheduler's old `enqueueSettlement` verbatim: the settle-window
114
+ * slot is reserved before the work starts, released in the same `finally`
115
+ * that retires `pending`, and the first rejection that is not a lost lock
116
+ * is recorded for the top of the next tick to throw.
117
+ *
118
+ * @param {string} nodeId
119
+ * @param {Pick<Job, "runtime">} job
120
+ * @param {() => Promise<void>} work
121
+ */
122
+ enqueue(nodeId, job, work) {
123
+ settling.set(nodeId, job);
124
+ const settlement = settlementContext.run({ park }, async () => {
125
+ await acquireSection();
126
+ try {
127
+ return await work();
128
+ } finally {
129
+ releaseSection();
130
+ }
131
+ }).finally(() => {
132
+ pending.delete(nodeId);
133
+ settling.delete(nodeId);
134
+ });
135
+ settlement.catch((error) => {
136
+ if (!(error instanceof LockLostError) && failure === null) failure = error;
137
+ });
138
+ pending.set(nodeId, settlement);
139
+ },
140
+ /** @returns {unknown} */
141
+ failure: () => failure,
142
+ };
143
+ }
@@ -13,12 +13,14 @@ import { boundedUtf8, errorMessage } from "../util.mjs";
13
13
  import { compactVerification } from "../contract/verification.mjs";
14
14
  import { MAX_SCOPE_FINDING_PATHS } from "../contract/scope-findings.mjs";
15
15
  import { finalVerificationCommands, phaseTerminalNode, sharedVerificationCommands } from "../contract/final-verification.mjs";
16
- import { join } from "node:path";
16
+ import { realpathSync } from "node:fs";
17
+ import { join, resolve } from "node:path";
17
18
  import { listNodeSnapshots, readNodeSnapshot } from "../run/node-store.mjs";
18
19
  import { SETTLED } from "./prompts.mjs";
19
20
  import { terminateInvocation } from "./process.mjs";
20
21
  import { writeNode } from "./state.mjs";
21
22
  import { runVerification } from "./run-command.mjs";
23
+ import { parkDuringProof } from "./settlement-overlap.mjs";
22
24
 
23
25
  /** @typedef {import("../repo/integrate.mjs").CandidateEvidence} CandidateEvidence */
24
26
  /** @typedef {import("../cli.mjs").LockHandle} LockHandle */
@@ -31,6 +33,119 @@ import { runVerification } from "./run-command.mjs";
31
33
  /** @typedef {import("../contract/index.mjs").VerificationState} VerificationState */
32
34
  /** @typedef {{index: number, total: number, argv: string}} VerificationProgress */
33
35
 
36
+ /**
37
+ * The proof gate: the verification layer's own admission for concurrent
38
+ * passes. R7 (phase 5) allows proofs in independent worktrees to advance
39
+ * together, bounded by the contract's `maxParallel` — the ceiling the planner
40
+ * already sized from this machine, and nothing higher (phase-5 restriction:
41
+ * the host is measured before concurrency is raised). Passes that share a
42
+ * tree can interfere, so they never overlap no matter what the budget allows:
43
+ * the base checkout (`contract.cwd`, where a node without an attempt worktree
44
+ * runs) and the run's one candidate worktree are each serialized under their
45
+ * own key. The settlement overlap (`engine/settlement-overlap.mjs`) is the
46
+ * caller that puts the budget to work: it parks a settlement for the length
47
+ * of each pass, so two nodes' independent worktree proofs overlap through a
48
+ * real run while this gate holds the bound, shared-tree passes, and the
49
+ * candidate worktree's pass serialized.
50
+ *
51
+ * @typedef {{limit: number, active: number, held: Set<string>, waiters: {key: string, satisfy: () => void}[]}} ProofGate
52
+ */
53
+
54
+ /** @type {WeakMap<ValidatedContract, ProofGate>} */
55
+ const proofGates = new WeakMap();
56
+
57
+ /**
58
+ * @param {ValidatedContract} contract
59
+ * @returns {ProofGate}
60
+ */
61
+ function proofGateFor(contract) {
62
+ let gate = proofGates.get(contract);
63
+ if (!gate) {
64
+ gate = { limit: Math.max(1, contract.maxParallel ?? 1), active: 0, held: new Set(), waiters: [] };
65
+ proofGates.set(contract, gate);
66
+ }
67
+ return gate;
68
+ }
69
+
70
+ /**
71
+ * The tree a pass runs in. `realpath` because a worktree path reached through
72
+ * a symlinked parent (/var vs /private/var on darwin) must read as the same
73
+ * key on every call.
74
+ *
75
+ * @param {string} workspace
76
+ * @returns {string}
77
+ */
78
+ function proofWorkspaceKey(workspace) {
79
+ try {
80
+ return realpathSync(workspace);
81
+ } catch {
82
+ // An absent workspace has no realpath; nothing can run in it anyway, so
83
+ // the lexical resolution is as good a key as any.
84
+ return resolve(workspace);
85
+ }
86
+ }
87
+
88
+ /** @param {ProofGate} gate @returns {void} */
89
+ function drainProofWaiters(gate) {
90
+ for (let index = 0; index < gate.waiters.length && gate.active < gate.limit;) {
91
+ const waiter = /** @type {{key: string, satisfy: () => void}} */ (gate.waiters[index]);
92
+ if (gate.held.has(waiter.key)) {
93
+ index += 1;
94
+ continue;
95
+ }
96
+ gate.waiters.splice(index, 1);
97
+ waiter.satisfy();
98
+ }
99
+ }
100
+
101
+ /**
102
+ * @param {ProofGate} gate
103
+ * @param {string} key
104
+ * @returns {Promise<() => void>}
105
+ */
106
+ function acquireProofSlot(gate, key) {
107
+ return new Promise((resolveSlot) => {
108
+ /** @returns {void} */
109
+ const satisfy = () => {
110
+ gate.active += 1;
111
+ gate.held.add(key);
112
+ let released = false;
113
+ resolveSlot(() => {
114
+ if (released) return;
115
+ released = true;
116
+ gate.active -= 1;
117
+ gate.held.delete(key);
118
+ drainProofWaiters(gate);
119
+ });
120
+ };
121
+ if (gate.active < gate.limit && !gate.held.has(key)) {
122
+ satisfy();
123
+ return;
124
+ }
125
+ gate.waiters.push({ key, satisfy });
126
+ });
127
+ }
128
+
129
+ /**
130
+ * Run one verification pass holding its proof slot: the pass overlaps other
131
+ * passes only in worktrees independent of its own, and only while the run's
132
+ * budget has room.
133
+ *
134
+ * @template T
135
+ * @param {ValidatedContract} contract
136
+ * @param {string} workspace
137
+ * @param {() => Promise<T>} run
138
+ * @returns {Promise<T>}
139
+ */
140
+ async function withProofSlot(contract, workspace, run) {
141
+ const release = await acquireProofSlot(proofGateFor(contract), proofWorkspaceKey(workspace));
142
+ try {
143
+ return await run();
144
+ } finally {
145
+ release();
146
+ }
147
+ }
148
+
34
149
  /**
35
150
  * The sibling phase-terminal node ids already settled, read straight off
36
151
  * disk: every node snapshot is persisted from the run's first tick (see
@@ -147,7 +262,16 @@ export async function executeControllerVerification(contract, runDir, node, stat
147
262
  /** @param {VerificationAttempt} attempt @returns {VerificationProgress} */
148
263
  const progressFor = (attempt) => verificationProgress(attempt.commandIndex + 1, commands.length, /** @type {VerificationCommand|undefined} */ (commands[attempt.commandIndex])?.argv);
149
264
  try {
150
- const result = await runVerification(commands, workspace, {
265
+ // Parked for the length of the pass: the settlement's section slot goes
266
+ // back to the overlap while these child processes run, so a sibling's
267
+ // settlement -- and its own proof -- advances. Everything after this
268
+ // await (judge dispatch, gate verdicts, integration) runs in the
269
+ // re-acquired section; the proof gate below is what bounds the overlap.
270
+ // The candidate pass in `verifyCandidateWorkspace` is deliberately not
271
+ // parked: it runs inside the integration transaction's candidate lock,
272
+ // and parking it would wait to re-acquire a section a sibling holds
273
+ // while that sibling waits on this same candidate lock.
274
+ const result = await parkDuringProof(() => withProofSlot(contract, workspace, () => runVerification(commands, workspace, {
151
275
  logDir: join(runDir, "logs", `${node.id}.${state.attempt}.verification`),
152
276
  writeFiles: node.taskPacket.writeFiles ?? [],
153
277
  onAttemptStart: (attempt) => persistVerificationAttempt(runDir, state, lock, attempt, progressFor(attempt)),
@@ -156,7 +280,7 @@ export async function executeControllerVerification(contract, runDir, node, stat
156
280
  ...attempt,
157
281
  result: boundedVerificationAttemptResult(attempt.result),
158
282
  }),
159
- });
283
+ })));
160
284
  state.verification = {
161
285
  ...compactVerification(result),
162
286
  completed: true,
@@ -318,19 +442,19 @@ export async function verifyCandidateWorkspace(contract, node, state, runDir, wo
318
442
  const commands = [...node.taskPacket.verification, ...sharedVerificationCommands(contract), ...finalVerificationCommands(contract, node, settledSiblingIds(runDir, contract, node))];
319
443
  markCandidateVerification(runDir, state, lock, true);
320
444
  try {
321
- const result = await runVerification(commands, workspace, {
445
+ const result = await withProofSlot(contract, workspace, () => runVerification(commands, workspace, {
322
446
  logDir: join(runDir, "logs", `${node.id}.${state.attempt}.candidate-verification`),
323
447
  writeFiles: node.taskPacket.writeFiles ?? [],
324
- });
448
+ }));
325
449
  /** @type {import("../contract/verification.mjs").VerificationResult & {retried?: number[]}} */
326
450
  let settled = result;
327
451
  if (!result.passed) {
328
452
  settled = await retryDivergentCandidateCommands(result, state.verification, async (indexes) => {
329
453
  const subset = indexes.map((index) => commands[index]);
330
- const rerun = await runVerification(subset, workspace, {
454
+ const rerun = await withProofSlot(contract, workspace, () => runVerification(subset, workspace, {
331
455
  logDir: join(runDir, "logs", `${node.id}.${state.attempt}.candidate-retry`),
332
456
  writeFiles: node.taskPacket.writeFiles ?? [],
333
- });
457
+ }));
334
458
  return rerun.commands;
335
459
  });
336
460
  }