faberun 0.30.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/campaign/campaign-brief.mjs +17 -6
- package/src/campaign/chain.mjs +124 -8
- package/src/campaign/ledger.mjs +19 -7
- package/src/campaign/metrics-command.mjs +68 -9
- package/src/campaign/metrics.mjs +75 -45
- package/src/campaign/record.mjs +8 -1
- package/src/campaign/reserve.mjs +340 -0
- package/src/campaign/spend.mjs +229 -0
- package/src/campaign/unpark.mjs +54 -1
- package/src/campaign/watch.mjs +314 -37
- package/src/contract/judge-envelope.mjs +17 -2
- package/src/contract/scope-findings.mjs +3 -1
- package/src/contract/task-packet.mjs +40 -5
- package/src/contract/verification.mjs +67 -1
- package/src/engine/aux-admission.mjs +124 -0
- package/src/engine/capacity.mjs +78 -0
- package/src/engine/dispatch.mjs +116 -23
- package/src/engine/failure-cause.mjs +59 -0
- package/src/engine/lifecycle.mjs +3 -2
- package/src/engine/phase-session.mjs +56 -10
- package/src/engine/prompts.mjs +57 -2
- package/src/engine/reserve-admission.mjs +99 -0
- package/src/engine/resume.mjs +3 -1
- package/src/engine/review.mjs +21 -0
- package/src/engine/run-command.mjs +166 -11
- package/src/engine/scheduler.mjs +71 -88
- package/src/engine/settle.mjs +15 -8
- package/src/engine/settlement-overlap.mjs +143 -0
- package/src/engine/verify.mjs +131 -7
- package/src/harnesses/session-metrics.mjs +112 -20
- package/src/notify/index.mjs +86 -10
- package/src/plan/freeze.mjs +7 -2
- package/src/plan/human-step.mjs +36 -0
- package/src/plan/pipeline.mjs +53 -22
- package/src/plan/preflight.mjs +81 -5
- package/src/plan/progress.mjs +101 -1
- package/src/plan/repo-facts.mjs +403 -49
- package/src/plan/resolve.mjs +7 -0
- package/src/plan/revise-context.mjs +108 -0
- package/src/plan/rounds.mjs +180 -45
- package/src/plan/sizing.mjs +33 -1
- package/src/plan/template.mjs +134 -22
- package/src/repo/integrate.mjs +47 -5
- package/src/report/campaign-brief-estimate.mjs +47 -1
- package/src/report/campaign-brief.mjs +27 -0
- package/src/report/locale.mjs +20 -0
- package/src/report/metrics-report.mjs +38 -0
- package/src/report/progress.mjs +194 -10
- package/src/run/usage-windows.mjs +99 -4
- package/src/run/usage.mjs +69 -7
- package/src/web/api.mjs +179 -24
- package/src/web/app.mjs +27 -2
package/src/engine/scheduler.mjs
CHANGED
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
} from "./review.mjs";
|
|
9
9
|
import { CONTRACT_VERSION, PROTOCOL_SCHEMA_VERSION } from "../harnesses/index.mjs";
|
|
10
10
|
import { routeRuntimeForState, routingBackoffActive } from "./failover.mjs";
|
|
11
|
-
import {
|
|
11
|
+
import { admissionPass } from "./capacity.mjs";
|
|
12
12
|
|
|
13
13
|
import {
|
|
14
14
|
bootstrapAttemptPath,
|
|
@@ -45,6 +45,7 @@ import { validateContractForLaunch } from "../campaign/chain.mjs";
|
|
|
45
45
|
import { finalVerificationCommands, gateProofTimeoutMs, sharedVerificationCommands } from "../contract/final-verification.mjs";
|
|
46
46
|
import { startJudge, startWorker } from "./dispatch.mjs";
|
|
47
47
|
import { raiseNodeAttention } from "./settle.mjs";
|
|
48
|
+
import { createSettlementOverlap } from "./settlement-overlap.mjs";
|
|
48
49
|
import { HUMAN_STEP_ERROR_CODE, humanStepAttentionMessage } from "../contract/human-step.mjs";
|
|
49
50
|
import { assertEnvironmentReady, captureRunIdentity, createRunMetadata, serializableContract, statesFingerprint } from "./run-identity.mjs";
|
|
50
51
|
import { blockDependents, runtimeAssignments } from "./assignment.mjs";
|
|
@@ -429,17 +430,24 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
|
|
|
429
430
|
// tick's critical path so an eligible sibling still dispatches into a free
|
|
430
431
|
// slot while this node's own invocation has already exited. A node's own
|
|
431
432
|
// steps stay ordered because only one settlement per node is ever in flight
|
|
432
|
-
// (see the dispatch loop below)
|
|
433
|
-
//
|
|
434
|
-
//
|
|
435
|
-
//
|
|
436
|
-
//
|
|
437
|
-
//
|
|
438
|
-
//
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
433
|
+
// (see the dispatch loop below). Different nodes' settlements overlap in
|
|
434
|
+
// exactly one place, their verification proofs (`parkDuringProof` in
|
|
435
|
+
// `engine/settlement-overlap.mjs`), and nowhere else: the sections between
|
|
436
|
+
// proofs run one at a time, because `finalizeClosedJobs` shares state a
|
|
437
|
+
// concurrent second call would corrupt -- the dispatch decisions that count
|
|
438
|
+
// `running` against `maxParallel`, the phase-continuation selection that
|
|
439
|
+
// picks at most one live node to carry a session forward, and
|
|
440
|
+
// `repo/integrate.mjs`'s one candidate ref and worktree per run. Only
|
|
441
|
+
// *dispatching a sibling* otherwise skips ahead of a node's settlement.
|
|
442
|
+
const settlements = createSettlementOverlap();
|
|
443
|
+
const pendingSettlements = settlements.pending;
|
|
444
|
+
// F5 admission's settle window: nodes whose job closed but whose settlement
|
|
445
|
+
// has not finished still hold the global and runtime slot their dispatch
|
|
446
|
+
// reserved. Released when the settlement itself settles, not at close --
|
|
447
|
+
// released at close, a sibling took the slot during the node's verification
|
|
448
|
+
// gate and a judge ran on top of it (2026-09-29, achado 2, peak 2 under
|
|
449
|
+
// `maxParallel: 1`; judged by `admissionPass` in capacity.mjs).
|
|
450
|
+
const settling = settlements.settling;
|
|
443
451
|
let canceled = false;
|
|
444
452
|
const cancel = () => { canceled = true; };
|
|
445
453
|
process.once("SIGINT", cancel);
|
|
@@ -459,57 +467,12 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
|
|
|
459
467
|
.map((state) => ({ nodeId: state.id, budgetBasis: nodeBudgetBasisMs(contract, /** @type {ValidatedNode} */ (heartbeatNodes.get(state.id))) }));
|
|
460
468
|
// A programmer error surfacing inside a background settlement must still
|
|
461
469
|
// crash the whole run, exactly as an unguarded `await finalizeClosedJobs`
|
|
462
|
-
// used to:
|
|
463
|
-
//
|
|
464
|
-
// block a sibling's dispatch is
|
|
470
|
+
// used to: the overlap records it (`settlements.failure()`) and it is
|
|
471
|
+
// thrown from the top of the loop on the very next tick, rather than
|
|
472
|
+
// immediately, so it cannot itself become the block a sibling's dispatch is
|
|
473
|
+
// waiting behind. A lost lock is not this --
|
|
465
474
|
// the loop's own `lock.assert()` calls surface that same condition on their
|
|
466
475
|
// own schedule, so it is left for them.
|
|
467
|
-
/** @type {unknown} */
|
|
468
|
-
let backgroundSettlementFailure = null;
|
|
469
|
-
// Mark every job that closed this tick as settling, and free its slot,
|
|
470
|
-
// without waiting for any of them: that alone is what lets an eligible
|
|
471
|
-
// sibling dispatch into the freed slot while this node's minutes-long
|
|
472
|
-
// controller verification or judge round is still running. The actual
|
|
473
|
-
// settlement work is chained onto `settlementQueue`, one node at a time in
|
|
474
|
-
// the order its job closed, so it still runs exactly as serialized against
|
|
475
|
-
// every *other* node's settlement as it did when this loop awaited
|
|
476
|
-
// `finalizeClosedJobs` directly -- `finalizeClosedJobs` runs against a
|
|
477
|
-
// one-entry map per node, so this is one call per node rather than the one
|
|
478
|
-
// batched call it used to be, but the chain still runs them one at a time.
|
|
479
|
-
// A settlement may itself dispatch the node's next phase (a judge, a
|
|
480
|
-
// revision) through the same `startJudge`/`startWorker` calls dispatch below
|
|
481
|
-
// uses, so it is handed the real `running` to dispatch into: a job it starts
|
|
482
|
-
// is counted against `maxParallel` from the instant the process exists, and
|
|
483
|
-
// `applyRejection` reads that same map to decide whether the run has a slot
|
|
484
|
-
// for the revision at all. It used to dispatch into the throwaway one-entry
|
|
485
|
-
// map instead, copied back only once the settlement returned, which is how
|
|
486
|
-
// run state-location-and-routing-economics-13 came to hold two workers under
|
|
487
|
-
// `maxParallel: 1` on 2026-09-21. What it may *settle* is still only its own
|
|
488
|
-
// node: the one-entry map below is the job, not the run. The node's own
|
|
489
|
-
// steps stay ordered by never starting a second settlement for a node whose
|
|
490
|
-
// first has not yet cleared `pendingSettlements`.
|
|
491
|
-
const settleClosedJobsInBackground = () => {
|
|
492
|
-
for (const [nodeId, job] of [...running]) {
|
|
493
|
-
if (pendingSettlements.has(nodeId) || !job.closed || invocationAlive(job.invocation)) continue;
|
|
494
|
-
running.delete(nodeId);
|
|
495
|
-
const settlement = settlementQueue
|
|
496
|
-
.then(() => finalizeClosedJobs(contract, runDir, states, new Map([[nodeId, job]]), lock, campaign.path, running))
|
|
497
|
-
.finally(() => pendingSettlements.delete(nodeId));
|
|
498
|
-
// The queue itself must never reject -- a rejected settlement (a lost
|
|
499
|
-
// lock, a programmer error) would otherwise wedge every node queued
|
|
500
|
-
// behind it. The rejection still reaches whoever awaits the real
|
|
501
|
-
// `settlement` promise (`pendingSettlements`, below).
|
|
502
|
-
settlementQueue = settlement.catch(() => {});
|
|
503
|
-
// Handled here so an in-flight settlement never becomes an unhandled
|
|
504
|
-
// rejection when nobody happens to await `pendingSettlements` before the
|
|
505
|
-
// process exits; the original promise, still held below, carries the
|
|
506
|
-
// rejection to whichever checkpoint (cancel, shutdown) awaits it.
|
|
507
|
-
settlement.catch((error) => {
|
|
508
|
-
if (!(error instanceof LockLostError) && backgroundSettlementFailure === null) backgroundSettlementFailure = error;
|
|
509
|
-
});
|
|
510
|
-
pendingSettlements.set(nodeId, settlement);
|
|
511
|
-
}
|
|
512
|
-
};
|
|
513
476
|
// Captured once, before the loop, rather than re-derived every tick: it is
|
|
514
477
|
// "parked when this controller invocation started" (autoRetryParkedNodes's
|
|
515
478
|
// own contract), and a node a background settlement parks between two ticks
|
|
@@ -531,7 +494,7 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
|
|
|
531
494
|
// before the loop is allowed to end.
|
|
532
495
|
if (anyUnsettled()) for (;;) {
|
|
533
496
|
lock.assert();
|
|
534
|
-
if (
|
|
497
|
+
if (settlements.failure() !== null) throw settlements.failure();
|
|
535
498
|
if (existsSync(join(runDir, "cancel.request.json"))) canceled = true;
|
|
536
499
|
if (canceled) {
|
|
537
500
|
// A settlement already in flight owns the one decision a cancellation
|
|
@@ -569,9 +532,34 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
|
|
|
569
532
|
// running, not only once it closes. `finalizeClosedJobs` used to open
|
|
570
533
|
// with this same check; calling it once here, rather than once per
|
|
571
534
|
// node settled this tick, is what keeps it at one pass per tick now
|
|
572
|
-
// that settlement runs per node
|
|
535
|
+
// that settlement runs per node below.
|
|
573
536
|
await emitNodeAdvisories(contract, runDir, states);
|
|
574
|
-
|
|
537
|
+
// Mark every job that closed this tick as settling -- its admission slot
|
|
538
|
+
// moves from `running` into `settling` (see the settle window above) --
|
|
539
|
+
// without waiting for any of them. The settlement work is handed to the
|
|
540
|
+
// overlap one node at a time in the order its job closed: its sections
|
|
541
|
+
// are serialized against every *other* node's exactly as they were when
|
|
542
|
+
// this loop awaited `finalizeClosedJobs` directly, and only its
|
|
543
|
+
// verification proof is parked out of the slot while it runs (see
|
|
544
|
+
// `engine/settlement-overlap.mjs`). A settlement
|
|
545
|
+
// may itself dispatch the node's next phase (a judge, a revision)
|
|
546
|
+
// through the same `startJudge`/`startWorker` calls dispatch below uses,
|
|
547
|
+
// so it is handed the real `running` to dispatch into: a job it starts
|
|
548
|
+
// is counted against `maxParallel` from the instant the process exists,
|
|
549
|
+
// and `applyRejection` reads that same map to decide whether the run has
|
|
550
|
+
// a slot for the revision at all. It used to dispatch into the throwaway
|
|
551
|
+
// one-entry map instead, copied back only once the settlement returned,
|
|
552
|
+
// which is how run state-location-and-routing-economics-13 came to hold
|
|
553
|
+
// two workers under `maxParallel: 1` on 2026-09-21. What it may *settle*
|
|
554
|
+
// is still only its own node: the one-entry map below is the job, not
|
|
555
|
+
// the run. The node's own steps stay ordered by never starting a second
|
|
556
|
+
// settlement for a node whose first has not yet cleared
|
|
557
|
+
// `pendingSettlements`.
|
|
558
|
+
for (const [nodeId, job] of [...running]) {
|
|
559
|
+
if (pendingSettlements.has(nodeId) || !job.closed || invocationAlive(job.invocation)) continue;
|
|
560
|
+
running.delete(nodeId);
|
|
561
|
+
settlements.enqueue(nodeId, job, () => finalizeClosedJobs(contract, runDir, states, new Map([[nodeId, job]]), lock, campaign.path, running));
|
|
562
|
+
}
|
|
575
563
|
await detectStalls(contract, running, async (job, status, error) => {
|
|
576
564
|
const envelope = recordInvocationUsage(job);
|
|
577
565
|
job.state.usage = invocationUsage(job.state);
|
|
@@ -642,44 +630,39 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
|
|
|
642
630
|
}, lock);
|
|
643
631
|
await raiseNodeAttention(campaign.path, runDir, /** @type {NodeSnapshot} */ (states.get(node.id)), HUMAN_STEP_ERROR_CODE);
|
|
644
632
|
}
|
|
645
|
-
|
|
646
|
-
|
|
633
|
+
// F5 admission is judged against every attempt holding a slot: the live
|
|
634
|
+
// jobs plus the closed jobs still inside their settle window. The
|
|
635
|
+
// contract's `maxParallel` bounds the total; a spawn the settlement
|
|
636
|
+
// starts itself (the node's judge) runs inside the node's own held slot.
|
|
637
|
+
const admission = admissionPass(contract, running, settling, states);
|
|
638
|
+
if (admission.slots > 0) {
|
|
647
639
|
const dispatchable = ready.filter((node) => !node.humanStep);
|
|
648
|
-
// Per-runtime capacity is judged per dispatch, not per tick: the
|
|
649
|
-
// counts include what this tick has already started, and a runtime a
|
|
650
|
-
// sibling is waiting out a quota reset on accepts nothing new.
|
|
651
|
-
const counts = runningPerRuntime(running.values());
|
|
652
|
-
const held = quotaHeldRuntimes(states.values(), Date.now());
|
|
653
640
|
let dispatched = 0;
|
|
654
641
|
for (const node of dispatchable) {
|
|
655
|
-
if (dispatched >= slots) break;
|
|
642
|
+
if (dispatched >= admission.slots) break;
|
|
656
643
|
const state = states.get(node.id);
|
|
657
644
|
if (!state || routingBackoffActive(state, state.phase)) continue;
|
|
658
645
|
const routed = routeRuntimeForState(contract, node, state, state.phase === "judge" ? "judge" : "worker");
|
|
659
|
-
if (!
|
|
660
|
-
counts.set(routed.id, (counts.get(routed.id) ?? 0) + 1);
|
|
646
|
+
if (!admission.take(routed.id)) continue;
|
|
661
647
|
dispatched += 1;
|
|
662
648
|
// A node recovered pending a re-ask judge (its own worker attempt
|
|
663
649
|
// already accepted, `state.result` durable) reaches `settleDone` /
|
|
664
650
|
// `integrateAttempt` exactly like a closed job's own settlement
|
|
665
|
-
// does, on the same per-run candidate ref and worktree
|
|
666
|
-
//
|
|
667
|
-
// same way:
|
|
651
|
+
// does, on the same per-run candidate ref and worktree the
|
|
652
|
+
// overlap's serialized sections exist to protect -- so it is
|
|
653
|
+
// dispatched the same way: handed to the overlap rather than
|
|
654
|
+
// awaited here, using
|
|
668
655
|
// `running` itself as its dispatch map, so the judge it starts is
|
|
669
656
|
// counted the instant it exists. The node's own order is untouched
|
|
670
657
|
// (still one entry at a time, gated by `pendingSettlements`); only
|
|
671
658
|
// the tick stops waiting behind it.
|
|
672
659
|
if (state.phase === "judge" && state.result) {
|
|
673
660
|
const workerResult = state.result;
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
settlement.catch((error) => {
|
|
680
|
-
if (!(error instanceof LockLostError) && backgroundSettlementFailure === null) backgroundSettlementFailure = error;
|
|
681
|
-
});
|
|
682
|
-
pendingSettlements.set(node.id, settlement);
|
|
661
|
+
// Counted twice while the judge job is live -- in `settling` and
|
|
662
|
+
// in `running` -- which can only hold a sibling back.
|
|
663
|
+
settlements.enqueue(node.id, { runtime: routed }, () =>
|
|
664
|
+
startJudge(contract, node, state, runDir, running, workerResult, lock, states, campaign.path)
|
|
665
|
+
.then((round) => applyJudgeRound(round, contract, node, state, runDir, running, lock, states, campaign.path, workerResult)));
|
|
683
666
|
continue;
|
|
684
667
|
}
|
|
685
668
|
state.attempt += 1;
|
|
@@ -734,12 +717,12 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
|
|
|
734
717
|
// instant it settles, win or lose -- so a rejection recorded here can
|
|
735
718
|
// already be gone from the map above by the time this line runs, with
|
|
736
719
|
// nothing left to await it. The loop's own top-of-tick check
|
|
737
|
-
// (`if (
|
|
720
|
+
// (`if (settlements.failure() !== null) throw ...`) cannot save
|
|
738
721
|
// that case either: the loop has already exited. Checked again here,
|
|
739
722
|
// once, so a background settlement failure can never read as a clean
|
|
740
723
|
// finish just because it settled on the same tick the run's last node
|
|
741
724
|
// did.
|
|
742
|
-
if (
|
|
725
|
+
if (settlements.failure() !== null) throw settlements.failure();
|
|
743
726
|
} catch (error) {
|
|
744
727
|
if (!(error instanceof LockLostError)) throw error;
|
|
745
728
|
// A settlement still merges whatever it started into `running` in its own
|
package/src/engine/settle.mjs
CHANGED
|
@@ -16,6 +16,7 @@ import { candidateOnlyFailures, resetPhaseRouting, verificationFailureVerdict }
|
|
|
16
16
|
import { retryPrompt } from "./prompts.mjs";
|
|
17
17
|
import { renderPreviousAttemptSection } from "./retry.mjs";
|
|
18
18
|
import { startWorker } from "./dispatch.mjs";
|
|
19
|
+
import { auxSpawnAdmission, recordAuxRefusal } from "./aux-admission.mjs";
|
|
19
20
|
import { ensureTerminalEvent, transition, writeNode } from "./state.mjs";
|
|
20
21
|
import { verificationFailureWithScope } from "../contract/scope-findings.mjs";
|
|
21
22
|
import { alreadyNotified, notifyQueueFor } from "./notify-queue.mjs";
|
|
@@ -70,14 +71,18 @@ export function applyRejection(contract, node, state, runDir, running, lock, sta
|
|
|
70
71
|
// path) carry it.
|
|
71
72
|
state.previousAttempt = renderPreviousAttemptSection(state) ?? state.previousAttempt;
|
|
72
73
|
// A revision is a new attempt, and a new attempt is a dispatch: it starts
|
|
73
|
-
// here only against a slot the run actually has free
|
|
74
|
-
//
|
|
75
|
-
//
|
|
76
|
-
//
|
|
77
|
-
// regardless is how run
|
|
78
|
-
//
|
|
79
|
-
//
|
|
80
|
-
|
|
74
|
+
// here only against a slot the run actually has free -- the global
|
|
75
|
+
// `maxParallel` and the routed runtime's own ceiling, the same judgment a
|
|
76
|
+
// dispatch pass applies (`auxSpawnAdmission`). `running` is the
|
|
77
|
+
// scheduler's own map, with this node's own closed job already out of it.
|
|
78
|
+
// A settlement that dispatched regardless is how run
|
|
79
|
+
// state-location-and-routing-economics-13 ran two workers under
|
|
80
|
+
// `maxParallel: 1` on 2026-09-21, and two of that day's three OOM kills
|
|
81
|
+
// happened with more running than the contract declared.
|
|
82
|
+
const admission = running
|
|
83
|
+
? auxSpawnAdmission(contract, node, state, running, states, "worker")
|
|
84
|
+
: null;
|
|
85
|
+
if (admission?.admitted && running) {
|
|
81
86
|
// Dispatching here owns the increment, because `startWorker` expects the
|
|
82
87
|
// attempt number it is about to run under.
|
|
83
88
|
state.attempt += 1;
|
|
@@ -92,7 +97,9 @@ export function applyRejection(contract, node, state, runDir, running, lock, sta
|
|
|
92
97
|
// killed controller — a node that ran twice reported attempt 3, with no
|
|
93
98
|
// `…2.*` logs and a `worktree.previousAttempt` naming an attempt that
|
|
94
99
|
// never existed.
|
|
100
|
+
const from = state.status;
|
|
95
101
|
transition(runDir, state, "pending", { phase: "worker", error: null }, lock);
|
|
102
|
+
if (admission) recordAuxRefusal(runDir, state, from, lock, "worker", admission);
|
|
96
103
|
return;
|
|
97
104
|
}
|
|
98
105
|
transition(runDir, state, node.gate.enabled ? "exhausted" : "failed", {
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The settlement overlap: what makes two closed nodes' verification proofs run
|
|
3
|
+
* at the same time through one real run, while everything a concurrent second
|
|
4
|
+
* settlement would corrupt stays serialized.
|
|
5
|
+
*
|
|
6
|
+
* The verify layer's proof gate (`engine/verify.mjs`) already bounds concurrent
|
|
7
|
+
* passes by the contract's `maxParallel` and serializes passes that share a
|
|
8
|
+
* tree; what held the engine back was the caller. `driveRun` chained whole
|
|
9
|
+
* settlements onto one promise queue, so one node's controller verification --
|
|
10
|
+
* minutes of child processes -- held every other node's settlement behind it
|
|
11
|
+
* and two independent worktree proofs never overlapped (phase-5 d4.1). This
|
|
12
|
+
* module replaces that queue: a settlement is a sequence of serialized
|
|
13
|
+
* *sections* separated by *proofs*. A section holds the run's one settlement
|
|
14
|
+
* slot; `parkDuringProof`, called around the controller verification pass,
|
|
15
|
+
* hands the slot back while the proof's child processes run and re-acquires it
|
|
16
|
+
* before the settlement's next decision. Dispatch decisions, the
|
|
17
|
+
* phase-continuation choice in `finalizeClosedJobs`, and ref updates and the
|
|
18
|
+
* whole integration transaction therefore never interleave between two nodes,
|
|
19
|
+
* while the proofs between them do. The integration candidate's own
|
|
20
|
+
* verification is deliberately never parked: it runs inside
|
|
21
|
+
* `repo/integrate.mjs`'s candidate lock, and parking it would deadlock -- its
|
|
22
|
+
* settlement would wait to re-acquire the slot behind a sibling whose section
|
|
23
|
+
* is itself waiting on the candidate lock this settlement holds -- and
|
|
24
|
+
* integration is exactly what phase 5 keeps serialized.
|
|
25
|
+
*
|
|
26
|
+
* Per-node ordering needs no lock here: the scheduler never starts a second
|
|
27
|
+
* settlement for a node whose first is still in `pending`, and the admission
|
|
28
|
+
* settle window (`settling`) holds each node's slot from its job's close until
|
|
29
|
+
* its settlement finishes, parked proof included.
|
|
30
|
+
*/
|
|
31
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
32
|
+
import { LockLostError } from "../run/lock.mjs";
|
|
33
|
+
|
|
34
|
+
/** @typedef {import("./lifecycle.mjs").Job} Job */
|
|
35
|
+
|
|
36
|
+
/** @typedef {{park: (factory: () => Promise<unknown>) => Promise<unknown>}} SettlementContext */
|
|
37
|
+
|
|
38
|
+
/** The overlap of the settlement currently running on this controller, if any. @type {AsyncLocalStorage<SettlementContext>} */
|
|
39
|
+
const settlementContext = new AsyncLocalStorage();
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Run a verification proof outside the caller's serialized section: while the
|
|
43
|
+
* proof's child processes run, the settlement's slot is handed back and a
|
|
44
|
+
* sibling's settlement advances. Outside a settlement -- a recovery path, a
|
|
45
|
+
* test driving `executeControllerVerification` directly -- the proof runs
|
|
46
|
+
* exactly inline, unchanged.
|
|
47
|
+
*
|
|
48
|
+
* @template T
|
|
49
|
+
* @param {() => Promise<T>|T} factory
|
|
50
|
+
* @returns {Promise<T>}
|
|
51
|
+
*/
|
|
52
|
+
export async function parkDuringProof(factory) {
|
|
53
|
+
const overlap = settlementContext.getStore();
|
|
54
|
+
if (!overlap) return /** @type {T} */ (await factory());
|
|
55
|
+
return /** @type {Promise<T>} */ (overlap.park(/** @type {() => Promise<unknown>} */ (factory)));
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* @returns {{pending: Map<string, Promise<void>>, settling: Map<string, Pick<Job, "runtime">>, enqueue: (nodeId: string, job: Pick<Job, "runtime">, work: () => Promise<void>) => void, failure: () => unknown}}
|
|
60
|
+
*/
|
|
61
|
+
export function createSettlementOverlap() {
|
|
62
|
+
/** @type {Map<string, Promise<void>>} */
|
|
63
|
+
const pending = new Map();
|
|
64
|
+
/** @type {Map<string, Pick<Job, "runtime">>} */
|
|
65
|
+
const settling = new Map();
|
|
66
|
+
/** @type {unknown} */
|
|
67
|
+
let failure = null;
|
|
68
|
+
let sectionBusy = false;
|
|
69
|
+
/** @type {(() => void)[]} */
|
|
70
|
+
const sectionWaiters = [];
|
|
71
|
+
/** @returns {Promise<void>} */
|
|
72
|
+
const acquireSection = () => new Promise((satisfy) => {
|
|
73
|
+
if (!sectionBusy) {
|
|
74
|
+
sectionBusy = true;
|
|
75
|
+
satisfy();
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
sectionWaiters.push(satisfy);
|
|
79
|
+
});
|
|
80
|
+
// Hand-off, not vacancy: the next waiter is granted the slot directly, so a
|
|
81
|
+
// caller that acquires between the release and the grant joins the queue
|
|
82
|
+
// behind it instead of slipping past.
|
|
83
|
+
/** @returns {void} */
|
|
84
|
+
const releaseSection = () => {
|
|
85
|
+
const next = sectionWaiters.shift();
|
|
86
|
+
if (next) next();
|
|
87
|
+
else sectionBusy = false;
|
|
88
|
+
};
|
|
89
|
+
/**
|
|
90
|
+
* Hand the slot back while a proof runs and take it again before the next
|
|
91
|
+
* decision. Releasing is safe exactly because the proof holds no shared
|
|
92
|
+
* settlement state: its writes go to its own node's snapshot, its child
|
|
93
|
+
* processes run in their own worktree, and its overlap with sibling proofs
|
|
94
|
+
* is bounded by the verify layer's proof gate.
|
|
95
|
+
*
|
|
96
|
+
* @template T
|
|
97
|
+
* @param {() => Promise<T>} factory
|
|
98
|
+
* @returns {Promise<T>}
|
|
99
|
+
*/
|
|
100
|
+
const park = async (factory) => {
|
|
101
|
+
releaseSection();
|
|
102
|
+
try {
|
|
103
|
+
return await factory();
|
|
104
|
+
} finally {
|
|
105
|
+
await acquireSection();
|
|
106
|
+
}
|
|
107
|
+
};
|
|
108
|
+
return {
|
|
109
|
+
pending,
|
|
110
|
+
settling,
|
|
111
|
+
/**
|
|
112
|
+
* Chain one node's settlement onto the overlap. The reservation semantics
|
|
113
|
+
* are the scheduler's old `enqueueSettlement` verbatim: the settle-window
|
|
114
|
+
* slot is reserved before the work starts, released in the same `finally`
|
|
115
|
+
* that retires `pending`, and the first rejection that is not a lost lock
|
|
116
|
+
* is recorded for the top of the next tick to throw.
|
|
117
|
+
*
|
|
118
|
+
* @param {string} nodeId
|
|
119
|
+
* @param {Pick<Job, "runtime">} job
|
|
120
|
+
* @param {() => Promise<void>} work
|
|
121
|
+
*/
|
|
122
|
+
enqueue(nodeId, job, work) {
|
|
123
|
+
settling.set(nodeId, job);
|
|
124
|
+
const settlement = settlementContext.run({ park }, async () => {
|
|
125
|
+
await acquireSection();
|
|
126
|
+
try {
|
|
127
|
+
return await work();
|
|
128
|
+
} finally {
|
|
129
|
+
releaseSection();
|
|
130
|
+
}
|
|
131
|
+
}).finally(() => {
|
|
132
|
+
pending.delete(nodeId);
|
|
133
|
+
settling.delete(nodeId);
|
|
134
|
+
});
|
|
135
|
+
settlement.catch((error) => {
|
|
136
|
+
if (!(error instanceof LockLostError) && failure === null) failure = error;
|
|
137
|
+
});
|
|
138
|
+
pending.set(nodeId, settlement);
|
|
139
|
+
},
|
|
140
|
+
/** @returns {unknown} */
|
|
141
|
+
failure: () => failure,
|
|
142
|
+
};
|
|
143
|
+
}
|
package/src/engine/verify.mjs
CHANGED
|
@@ -13,12 +13,14 @@ import { boundedUtf8, errorMessage } from "../util.mjs";
|
|
|
13
13
|
import { compactVerification } from "../contract/verification.mjs";
|
|
14
14
|
import { MAX_SCOPE_FINDING_PATHS } from "../contract/scope-findings.mjs";
|
|
15
15
|
import { finalVerificationCommands, phaseTerminalNode, sharedVerificationCommands } from "../contract/final-verification.mjs";
|
|
16
|
-
import {
|
|
16
|
+
import { realpathSync } from "node:fs";
|
|
17
|
+
import { join, resolve } from "node:path";
|
|
17
18
|
import { listNodeSnapshots, readNodeSnapshot } from "../run/node-store.mjs";
|
|
18
19
|
import { SETTLED } from "./prompts.mjs";
|
|
19
20
|
import { terminateInvocation } from "./process.mjs";
|
|
20
21
|
import { writeNode } from "./state.mjs";
|
|
21
22
|
import { runVerification } from "./run-command.mjs";
|
|
23
|
+
import { parkDuringProof } from "./settlement-overlap.mjs";
|
|
22
24
|
|
|
23
25
|
/** @typedef {import("../repo/integrate.mjs").CandidateEvidence} CandidateEvidence */
|
|
24
26
|
/** @typedef {import("../cli.mjs").LockHandle} LockHandle */
|
|
@@ -31,6 +33,119 @@ import { runVerification } from "./run-command.mjs";
|
|
|
31
33
|
/** @typedef {import("../contract/index.mjs").VerificationState} VerificationState */
|
|
32
34
|
/** @typedef {{index: number, total: number, argv: string}} VerificationProgress */
|
|
33
35
|
|
|
36
|
+
/**
|
|
37
|
+
* The proof gate: the verification layer's own admission for concurrent
|
|
38
|
+
* passes. R7 (phase 5) allows proofs in independent worktrees to advance
|
|
39
|
+
* together, bounded by the contract's `maxParallel` — the ceiling the planner
|
|
40
|
+
* already sized from this machine, and nothing higher (phase-5 restriction:
|
|
41
|
+
* the host is measured before concurrency is raised). Passes that share a
|
|
42
|
+
* tree can interfere, so they never overlap no matter what the budget allows:
|
|
43
|
+
* the base checkout (`contract.cwd`, where a node without an attempt worktree
|
|
44
|
+
* runs) and the run's one candidate worktree are each serialized under their
|
|
45
|
+
* own key. The settlement overlap (`engine/settlement-overlap.mjs`) is the
|
|
46
|
+
* caller that puts the budget to work: it parks a settlement for the length
|
|
47
|
+
* of each pass, so two nodes' independent worktree proofs overlap through a
|
|
48
|
+
* real run while this gate holds the bound, shared-tree passes, and the
|
|
49
|
+
* candidate worktree's pass serialized.
|
|
50
|
+
*
|
|
51
|
+
* @typedef {{limit: number, active: number, held: Set<string>, waiters: {key: string, satisfy: () => void}[]}} ProofGate
|
|
52
|
+
*/
|
|
53
|
+
|
|
54
|
+
/** @type {WeakMap<ValidatedContract, ProofGate>} */
|
|
55
|
+
const proofGates = new WeakMap();
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* @param {ValidatedContract} contract
|
|
59
|
+
* @returns {ProofGate}
|
|
60
|
+
*/
|
|
61
|
+
function proofGateFor(contract) {
|
|
62
|
+
let gate = proofGates.get(contract);
|
|
63
|
+
if (!gate) {
|
|
64
|
+
gate = { limit: Math.max(1, contract.maxParallel ?? 1), active: 0, held: new Set(), waiters: [] };
|
|
65
|
+
proofGates.set(contract, gate);
|
|
66
|
+
}
|
|
67
|
+
return gate;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* The tree a pass runs in. `realpath` because a worktree path reached through
|
|
72
|
+
* a symlinked parent (/var vs /private/var on darwin) must read as the same
|
|
73
|
+
* key on every call.
|
|
74
|
+
*
|
|
75
|
+
* @param {string} workspace
|
|
76
|
+
* @returns {string}
|
|
77
|
+
*/
|
|
78
|
+
function proofWorkspaceKey(workspace) {
|
|
79
|
+
try {
|
|
80
|
+
return realpathSync(workspace);
|
|
81
|
+
} catch {
|
|
82
|
+
// An absent workspace has no realpath; nothing can run in it anyway, so
|
|
83
|
+
// the lexical resolution is as good a key as any.
|
|
84
|
+
return resolve(workspace);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** @param {ProofGate} gate @returns {void} */
|
|
89
|
+
function drainProofWaiters(gate) {
|
|
90
|
+
for (let index = 0; index < gate.waiters.length && gate.active < gate.limit;) {
|
|
91
|
+
const waiter = /** @type {{key: string, satisfy: () => void}} */ (gate.waiters[index]);
|
|
92
|
+
if (gate.held.has(waiter.key)) {
|
|
93
|
+
index += 1;
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
gate.waiters.splice(index, 1);
|
|
97
|
+
waiter.satisfy();
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* @param {ProofGate} gate
|
|
103
|
+
* @param {string} key
|
|
104
|
+
* @returns {Promise<() => void>}
|
|
105
|
+
*/
|
|
106
|
+
function acquireProofSlot(gate, key) {
|
|
107
|
+
return new Promise((resolveSlot) => {
|
|
108
|
+
/** @returns {void} */
|
|
109
|
+
const satisfy = () => {
|
|
110
|
+
gate.active += 1;
|
|
111
|
+
gate.held.add(key);
|
|
112
|
+
let released = false;
|
|
113
|
+
resolveSlot(() => {
|
|
114
|
+
if (released) return;
|
|
115
|
+
released = true;
|
|
116
|
+
gate.active -= 1;
|
|
117
|
+
gate.held.delete(key);
|
|
118
|
+
drainProofWaiters(gate);
|
|
119
|
+
});
|
|
120
|
+
};
|
|
121
|
+
if (gate.active < gate.limit && !gate.held.has(key)) {
|
|
122
|
+
satisfy();
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
gate.waiters.push({ key, satisfy });
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Run one verification pass holding its proof slot: the pass overlaps other
|
|
131
|
+
* passes only in worktrees independent of its own, and only while the run's
|
|
132
|
+
* budget has room.
|
|
133
|
+
*
|
|
134
|
+
* @template T
|
|
135
|
+
* @param {ValidatedContract} contract
|
|
136
|
+
* @param {string} workspace
|
|
137
|
+
* @param {() => Promise<T>} run
|
|
138
|
+
* @returns {Promise<T>}
|
|
139
|
+
*/
|
|
140
|
+
async function withProofSlot(contract, workspace, run) {
|
|
141
|
+
const release = await acquireProofSlot(proofGateFor(contract), proofWorkspaceKey(workspace));
|
|
142
|
+
try {
|
|
143
|
+
return await run();
|
|
144
|
+
} finally {
|
|
145
|
+
release();
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
34
149
|
/**
|
|
35
150
|
* The sibling phase-terminal node ids already settled, read straight off
|
|
36
151
|
* disk: every node snapshot is persisted from the run's first tick (see
|
|
@@ -147,7 +262,16 @@ export async function executeControllerVerification(contract, runDir, node, stat
|
|
|
147
262
|
/** @param {VerificationAttempt} attempt @returns {VerificationProgress} */
|
|
148
263
|
const progressFor = (attempt) => verificationProgress(attempt.commandIndex + 1, commands.length, /** @type {VerificationCommand|undefined} */ (commands[attempt.commandIndex])?.argv);
|
|
149
264
|
try {
|
|
150
|
-
|
|
265
|
+
// Parked for the length of the pass: the settlement's section slot goes
|
|
266
|
+
// back to the overlap while these child processes run, so a sibling's
|
|
267
|
+
// settlement -- and its own proof -- advances. Everything after this
|
|
268
|
+
// await (judge dispatch, gate verdicts, integration) runs in the
|
|
269
|
+
// re-acquired section; the proof gate below is what bounds the overlap.
|
|
270
|
+
// The candidate pass in `verifyCandidateWorkspace` is deliberately not
|
|
271
|
+
// parked: it runs inside the integration transaction's candidate lock,
|
|
272
|
+
// and parking it would wait to re-acquire a section a sibling holds
|
|
273
|
+
// while that sibling waits on this same candidate lock.
|
|
274
|
+
const result = await parkDuringProof(() => withProofSlot(contract, workspace, () => runVerification(commands, workspace, {
|
|
151
275
|
logDir: join(runDir, "logs", `${node.id}.${state.attempt}.verification`),
|
|
152
276
|
writeFiles: node.taskPacket.writeFiles ?? [],
|
|
153
277
|
onAttemptStart: (attempt) => persistVerificationAttempt(runDir, state, lock, attempt, progressFor(attempt)),
|
|
@@ -156,7 +280,7 @@ export async function executeControllerVerification(contract, runDir, node, stat
|
|
|
156
280
|
...attempt,
|
|
157
281
|
result: boundedVerificationAttemptResult(attempt.result),
|
|
158
282
|
}),
|
|
159
|
-
});
|
|
283
|
+
})));
|
|
160
284
|
state.verification = {
|
|
161
285
|
...compactVerification(result),
|
|
162
286
|
completed: true,
|
|
@@ -318,19 +442,19 @@ export async function verifyCandidateWorkspace(contract, node, state, runDir, wo
|
|
|
318
442
|
const commands = [...node.taskPacket.verification, ...sharedVerificationCommands(contract), ...finalVerificationCommands(contract, node, settledSiblingIds(runDir, contract, node))];
|
|
319
443
|
markCandidateVerification(runDir, state, lock, true);
|
|
320
444
|
try {
|
|
321
|
-
const result = await runVerification(commands, workspace, {
|
|
445
|
+
const result = await withProofSlot(contract, workspace, () => runVerification(commands, workspace, {
|
|
322
446
|
logDir: join(runDir, "logs", `${node.id}.${state.attempt}.candidate-verification`),
|
|
323
447
|
writeFiles: node.taskPacket.writeFiles ?? [],
|
|
324
|
-
});
|
|
448
|
+
}));
|
|
325
449
|
/** @type {import("../contract/verification.mjs").VerificationResult & {retried?: number[]}} */
|
|
326
450
|
let settled = result;
|
|
327
451
|
if (!result.passed) {
|
|
328
452
|
settled = await retryDivergentCandidateCommands(result, state.verification, async (indexes) => {
|
|
329
453
|
const subset = indexes.map((index) => commands[index]);
|
|
330
|
-
const rerun = await runVerification(subset, workspace, {
|
|
454
|
+
const rerun = await withProofSlot(contract, workspace, () => runVerification(subset, workspace, {
|
|
331
455
|
logDir: join(runDir, "logs", `${node.id}.${state.attempt}.candidate-retry`),
|
|
332
456
|
writeFiles: node.taskPacket.writeFiles ?? [],
|
|
333
|
-
});
|
|
457
|
+
}));
|
|
334
458
|
return rerun.commands;
|
|
335
459
|
});
|
|
336
460
|
}
|