faberun 0.30.0 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/campaign/campaign-brief.mjs +17 -6
- package/src/campaign/chain.mjs +124 -8
- package/src/campaign/ledger.mjs +19 -7
- package/src/campaign/metrics-command.mjs +68 -9
- package/src/campaign/metrics.mjs +75 -45
- package/src/campaign/record.mjs +8 -1
- package/src/campaign/reserve.mjs +340 -0
- package/src/campaign/spend.mjs +229 -0
- package/src/campaign/unpark.mjs +54 -1
- package/src/campaign/watch.mjs +314 -37
- package/src/cli/brand.mjs +2 -1
- package/src/cli.mjs +52 -2
- package/src/contract/judge-envelope.mjs +17 -2
- package/src/contract/scope-findings.mjs +3 -1
- package/src/contract/task-packet.mjs +40 -5
- package/src/contract/verification.mjs +67 -1
- package/src/engine/aux-admission.mjs +124 -0
- package/src/engine/capacity.mjs +78 -0
- package/src/engine/dispatch.mjs +116 -23
- package/src/engine/failure-cause.mjs +59 -0
- package/src/engine/lifecycle.mjs +3 -2
- package/src/engine/phase-session.mjs +56 -10
- package/src/engine/prompts.mjs +57 -2
- package/src/engine/reserve-admission.mjs +99 -0
- package/src/engine/resume.mjs +3 -1
- package/src/engine/review.mjs +21 -0
- package/src/engine/run-command.mjs +166 -11
- package/src/engine/scheduler.mjs +92 -88
- package/src/engine/settle.mjs +15 -8
- package/src/engine/settlement-overlap.mjs +143 -0
- package/src/engine/verify.mjs +131 -7
- package/src/harnesses/session-metrics.mjs +112 -20
- package/src/notify/index.mjs +86 -10
- package/src/plan/freeze.mjs +7 -2
- package/src/plan/human-step.mjs +36 -0
- package/src/plan/pipeline.mjs +53 -22
- package/src/plan/preflight.mjs +81 -5
- package/src/plan/progress.mjs +101 -1
- package/src/plan/repo-facts.mjs +403 -49
- package/src/plan/resolve.mjs +7 -0
- package/src/plan/revise-context.mjs +108 -0
- package/src/plan/rounds.mjs +180 -45
- package/src/plan/sizing.mjs +33 -1
- package/src/plan/template.mjs +134 -22
- package/src/repo/integrate.mjs +47 -5
- package/src/report/campaign-brief-estimate.mjs +47 -1
- package/src/report/campaign-brief.mjs +31 -2
- package/src/report/locale.mjs +20 -0
- package/src/report/message.mjs +13 -1
- package/src/report/metrics-report.mjs +38 -0
- package/src/report/progress.mjs +194 -10
- package/src/report/render.mjs +1 -1
- package/src/report/report-html.mjs +235 -0
- package/src/report/report-video.mjs +323 -0
- package/src/run/usage-windows.mjs +99 -4
- package/src/run/usage.mjs +69 -7
- package/src/web/api.mjs +179 -24
- package/src/web/app.mjs +27 -2
- package/src/web/server.mjs +32 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Buffer } from "node:buffer";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
2
3
|
import { realpathSync, statSync } from "node:fs";
|
|
3
4
|
import { isAbsolute, resolve } from "node:path";
|
|
4
5
|
import { fail, isContained, tailText } from "../util.mjs";
|
|
@@ -88,7 +89,10 @@ export const MUTATION_TIERS = Object.freeze({
|
|
|
88
89
|
*/
|
|
89
90
|
|
|
90
91
|
/**
|
|
91
|
-
* Callbacks and options for {@link runVerification}.
|
|
92
|
+
* Callbacks and options for {@link runVerification}. `logDir` is both the
|
|
93
|
+
* per-command evidence log and the checkpoint store a resumed pass serves
|
|
94
|
+
* from; the serving is `engine/run-command.mjs`'s, the record's shape is
|
|
95
|
+
* validated here.
|
|
92
96
|
*
|
|
93
97
|
* @typedef {{signal?: AbortSignal, logDir?: string, writeFiles?: string[], onAttemptStart?: (attempt: VerificationAttempt) => void, onAttemptSpawn?: (attempt: VerificationAttempt) => void, onAttemptComplete?: (attempt: VerificationAttempt) => void}} VerificationOptions
|
|
94
98
|
*/
|
|
@@ -225,6 +229,68 @@ export function compactVerification(result) {
|
|
|
225
229
|
return { passed: Boolean(result?.passed), commands };
|
|
226
230
|
}
|
|
227
231
|
|
|
232
|
+
/**
|
|
233
|
+
* A per-command checkpoint: the identity hash of everything the recorded proof
|
|
234
|
+
* depends on, plus the recorded result bounded exactly like persisted state.
|
|
235
|
+
* It lives beside the pass's evidence log (`checkpoint-<ordinal>.json`) and is
|
|
236
|
+
* what a resumed verification pass serves instead of re-running a command.
|
|
237
|
+
*
|
|
238
|
+
* @typedef {{identity: string, result: VerificationCommandResult}} CommandCheckpoint
|
|
239
|
+
*/
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* The identity of one command's proof: a hash over the workspace tree — the
|
|
243
|
+
* dirty fingerprint *and* the HEAD tree id, because the fingerprint hashes
|
|
244
|
+
* only the difference from HEAD and two clean trees at different commits share
|
|
245
|
+
* one (review 2026-10-02: a stale passing result was served across a commit) —
|
|
246
|
+
* plus the validated command, the exact child environment and the
|
|
247
|
+
* dependency-inputs hash. The tree, the command, the environment and the
|
|
248
|
+
* dependencies all participate — never the argv alone, which is the proof
|
|
249
|
+
* cache phase 5 refuses. Key order is fixed by construction (`env` arrives in
|
|
250
|
+
* `VERIFICATION_ENV_BASE_NAMES`-then-declared order and the canonical object
|
|
251
|
+
* is one literal), so the hash is stable across the process restart a resume
|
|
252
|
+
* is.
|
|
253
|
+
*
|
|
254
|
+
* @param {{treeFingerprint: string|null, headTree: string|null, command: VerificationCommand, env: Record<string, string>, dependencyHash: string}} parts
|
|
255
|
+
* @returns {string}
|
|
256
|
+
*/
|
|
257
|
+
export function commandCheckpointIdentity({ treeFingerprint, headTree, command, env, dependencyHash }) {
|
|
258
|
+
const canonical = { dependencyHash, command, env, headTree, treeFingerprint };
|
|
259
|
+
return createHash("sha256").update(JSON.stringify(canonical)).digest("hex");
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Read back a written checkpoint, failing closed: anything malformed throws
|
|
264
|
+
* and the caller re-runs the command rather than serving or crashing. The
|
|
265
|
+
* recorded result is re-compacted on read, so a hand-edited or oversized
|
|
266
|
+
* record is either bounded to exactly what persisted state may carry or
|
|
267
|
+
* rejected.
|
|
268
|
+
*
|
|
269
|
+
* @param {unknown} value
|
|
270
|
+
* @param {string} [label]
|
|
271
|
+
* @returns {CommandCheckpoint}
|
|
272
|
+
*/
|
|
273
|
+
export function parseCommandCheckpoint(value, label = "command checkpoint") {
|
|
274
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new TypeError(`${label} must be an object`);
|
|
275
|
+
const record = /** @type {Record<string, unknown>} */ (value);
|
|
276
|
+
for (const key of Object.keys(record)) {
|
|
277
|
+
if (key !== "identity" && key !== "result") throw new TypeError(`${label} has unexpected field ${key}`);
|
|
278
|
+
}
|
|
279
|
+
if (typeof record.identity !== "string" || !record.identity || Buffer.byteLength(record.identity, "utf8") > 128) {
|
|
280
|
+
throw new TypeError(`${label}.identity must be a hash of at most 128 bytes`);
|
|
281
|
+
}
|
|
282
|
+
const result = /** @type {Record<string, unknown>} */ (record.result);
|
|
283
|
+
if (!result || typeof result !== "object" || Array.isArray(result) || typeof result.passed !== "boolean") {
|
|
284
|
+
throw new TypeError(`${label}.result must be a single-command verification result`);
|
|
285
|
+
}
|
|
286
|
+
if (!Array.isArray(result.commands) || result.commands.length !== 1) {
|
|
287
|
+
throw new TypeError(`${label}.result must carry exactly one command`);
|
|
288
|
+
}
|
|
289
|
+
const compacted = compactVerification({ passed: result.passed, commands: /** @type {VerificationCommandResult[]} */ (result.commands) });
|
|
290
|
+
if (compacted.commands.length !== 1) throw new TypeError(`${label}.result exceeds the persisted command bounds`);
|
|
291
|
+
return { identity: record.identity, result: compacted.commands[0] };
|
|
292
|
+
}
|
|
293
|
+
|
|
228
294
|
|
|
229
295
|
/**
|
|
230
296
|
* One proof, one source. A command that declares `requirementId` says it is
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Admission for the paid spawns that start outside the scheduler's dispatch
|
|
3
|
+
* pass: the node's own judge after its verification gate, the bounded judge
|
|
4
|
+
* re-ask, and the gate revision. `capacity.mjs`'s `admissionPass` judges the
|
|
5
|
+
* spawns the dispatch pass issues itself and owns the pass's reservations;
|
|
6
|
+
* these three start inside a settlement or a stall handler, take over the
|
|
7
|
+
* slot the node already holds, and need one yes/no judgment against the live
|
|
8
|
+
* running set. On a refusal the node hands back to the scheduler -- the one
|
|
9
|
+
* place that knows when the run stops being full -- and the verdict is
|
|
10
|
+
* recorded on the node's event journal.
|
|
11
|
+
*
|
|
12
|
+
* Measured 2026-09-29 (ACHADOS-PRODUTO achado 2): these spawns ran with no
|
|
13
|
+
* per-runtime check, and two invocations of one runtime were seen alive at
|
|
14
|
+
* once under a contract declaring `maxParallel: 1`. The judgment here is the
|
|
15
|
+
* dispatch pass's own rule (`runtimeHasCapacity`) over the same running set
|
|
16
|
+
* the spawn is about to join, plus the pass's global bound: the settle window
|
|
17
|
+
* keeps `running` under `maxParallel` while a settlement spawns, and the
|
|
18
|
+
* stall-handler re-ask -- the one aux spawn outside any settle window -- takes
|
|
19
|
+
* the same check instead of trusting that.
|
|
20
|
+
*/
|
|
21
|
+
import { quotaHeldRuntimes, runningPerRuntime, runtimeHasCapacity } from "./capacity.mjs";
|
|
22
|
+
import { routeRuntimeForState } from "./failover.mjs";
|
|
23
|
+
import { startJudge } from "./dispatch.mjs";
|
|
24
|
+
import { appendTransitionEvent, transition } from "./state.mjs";
|
|
25
|
+
|
|
26
|
+
/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
|
|
27
|
+
/** @typedef {import("../contract/index.mjs").ValidatedNode} ValidatedNode */
|
|
28
|
+
/** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
|
|
29
|
+
/** @typedef {import("../run/lock.mjs").LockRecord} LockRecord */
|
|
30
|
+
/** @typedef {ReturnType<typeof import("../run/lock.mjs").acquire>} LockHandle */
|
|
31
|
+
/** @typedef {import("./dispatch.mjs").JudgeRound} JudgeRound */
|
|
32
|
+
/** @typedef {import("./process.mjs").Job} Job */
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* One auxiliary-spawn admission: may this spawn start right now? The spawn
|
|
36
|
+
* joins the caller's `running` map, so that map is what is judged: the
|
|
37
|
+
* contract's global `maxParallel`, and the routed runtime's `maxConcurrent`
|
|
38
|
+
* over the attempts it already holds. A runtime some node is waiting out a
|
|
39
|
+
* quota reset on refuses too, exactly as a dispatch pass would. A closed job
|
|
40
|
+
* still inside its settle window holds no live process and is not counted:
|
|
41
|
+
* the slot it holds is this node's own, and this spawn is the reason the slot
|
|
42
|
+
* existed.
|
|
43
|
+
*
|
|
44
|
+
* @param {ValidatedContract} contract
|
|
45
|
+
* @param {ValidatedNode} node
|
|
46
|
+
* @param {NodeSnapshot} state
|
|
47
|
+
* @param {Map<string, Job>} running the live dispatch map the spawn would join
|
|
48
|
+
* @param {Map<string, NodeSnapshot>} states
|
|
49
|
+
* @param {"worker"|"judge"} role
|
|
50
|
+
* @param {number} [now] epoch milliseconds
|
|
51
|
+
* @returns {{admitted: boolean, runtimeId: string, reason: string|null}}
|
|
52
|
+
*/
|
|
53
|
+
export function auxSpawnAdmission(contract, node, state, running, states, role, now = Date.now()) {
|
|
54
|
+
const runtime = routeRuntimeForState(contract, node, state, role);
|
|
55
|
+
if (running.size >= contract.maxParallel) {
|
|
56
|
+
return { admitted: false, runtimeId: runtime.id, reason: `run holds ${running.size} of maxParallel ${contract.maxParallel} slots` };
|
|
57
|
+
}
|
|
58
|
+
const held = quotaHeldRuntimes(states.values(), now);
|
|
59
|
+
if (!runtimeHasCapacity(runtime.id, contract, runningPerRuntime(running.values()), held)) {
|
|
60
|
+
return {
|
|
61
|
+
admitted: false,
|
|
62
|
+
runtimeId: runtime.id,
|
|
63
|
+
reason: held.has(runtime.id)
|
|
64
|
+
? `runtime ${runtime.id} is waiting out a quota reset`
|
|
65
|
+
: `runtime ${runtime.id} holds its maxConcurrent of ${contract.runtimes[runtime.id]?.maxConcurrent}`,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
return { admitted: true, runtimeId: runtime.id, reason: null };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Record a refused auxiliary spawn on the node's event journal: the verdict
|
|
73
|
+
* the deferral is gated on, read back by the phase proof and by an operator
|
|
74
|
+
* asking why a node went back to pending without a new attempt. The event is
|
|
75
|
+
* written only for a refusal, so its existence is the verdict. Measured
|
|
76
|
+
* 2026-10-02: `appendTransitionEvent` builds the event from the node snapshot
|
|
77
|
+
* and its own fields win every colliding name -- the event's `runtime` and
|
|
78
|
+
* `summary` are the state's, not ours -- so the refusal rides the one field
|
|
79
|
+
* the transition leaves unset: `error`, with the node's own error null
|
|
80
|
+
* (the deferral parks nothing on the node).
|
|
81
|
+
*
|
|
82
|
+
* @param {string} runDir
|
|
83
|
+
* @param {NodeSnapshot} state
|
|
84
|
+
* @param {string} from the node status the refusal found
|
|
85
|
+
* @param {LockHandle|null} lock
|
|
86
|
+
* @param {"worker"|"judge"} role
|
|
87
|
+
* @param {{runtimeId: string, reason: string|null}} admission
|
|
88
|
+
*/
|
|
89
|
+
export function recordAuxRefusal(runDir, state, from, lock, role, admission) {
|
|
90
|
+
appendTransitionEvent(runDir, state, from, "pending", {
|
|
91
|
+
type: "aux.admission",
|
|
92
|
+
role,
|
|
93
|
+
error: { code: "aux_spawn_refused", message: admission.reason ?? "aux spawn refused" },
|
|
94
|
+
}, lock);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* The judge a settlement starts itself: admit it against the live running
|
|
99
|
+
* set, and on a refusal hand the node back to the scheduler as a pending
|
|
100
|
+
* judge with its accepted result intact -- the drive loop dispatches it
|
|
101
|
+
* through the ordinary admission pass once the runtime frees. Returns what
|
|
102
|
+
* `startJudge` would have, so the caller's `applyJudgeRound` decides nothing
|
|
103
|
+
* new: a refused round is already parked on the node.
|
|
104
|
+
*
|
|
105
|
+
* @param {ValidatedContract} contract
|
|
106
|
+
* @param {ValidatedNode} node
|
|
107
|
+
* @param {NodeSnapshot} state
|
|
108
|
+
* @param {string} runDir
|
|
109
|
+
* @param {Map<string, Job>} running
|
|
110
|
+
* @param {Map<string, NodeSnapshot>} states
|
|
111
|
+
* @param {LockHandle} lock
|
|
112
|
+
* @param {string} campaignPath
|
|
113
|
+
* @returns {Promise<JudgeRound>}
|
|
114
|
+
*/
|
|
115
|
+
export async function auxJudgeRound(contract, node, state, runDir, running, states, lock, campaignPath) {
|
|
116
|
+
const admission = auxSpawnAdmission(contract, node, state, running, states, "judge");
|
|
117
|
+
if (!admission.admitted) {
|
|
118
|
+
const from = state.status;
|
|
119
|
+
transition(runDir, state, "pending", { phase: "judge", error: null, blockedBy: [] }, lock);
|
|
120
|
+
recordAuxRefusal(runDir, state, from, lock, "judge", admission);
|
|
121
|
+
return { kind: "refused" };
|
|
122
|
+
}
|
|
123
|
+
return startJudge(contract, node, state, runDir, running, state.result, lock, states, campaignPath);
|
|
124
|
+
}
|
package/src/engine/capacity.mjs
CHANGED
|
@@ -11,6 +11,25 @@
|
|
|
11
11
|
* measured 2026-09-20: `maxParallel` was the one concurrency knob, global to
|
|
12
12
|
* the run; nothing reduced dispatch to a provider that had just refused a
|
|
13
13
|
* sibling on quota, and 14 of 58 stored contracts ran with maxParallel 2.
|
|
14
|
+
*
|
|
15
|
+
* measured 2026-09-29: this gate was consulted by the scheduler's own
|
|
16
|
+
* dispatches only, and a run declaring `maxParallel: 1` and `maxConcurrent: 1`
|
|
17
|
+
* on each role's runtime was measured with a judge and a sibling's worker
|
|
18
|
+
* alive at once -- peak 2, one process per role, two on the single runtime
|
|
19
|
+
* (`test/engine/max-parallel.test.mjs`). The slot a node's closed job had
|
|
20
|
+
* freed was spendable while the node was still mid-flight.
|
|
21
|
+
*
|
|
22
|
+
* F5 admission (2026-10-02): the scheduler now counts a node from before its
|
|
23
|
+
* spawn until its settle completes (`admissionHold`, fed by the settle window
|
|
24
|
+
* in `scheduler.mjs`), which closes the freed-slot race for every spawn the
|
|
25
|
+
* scheduler issues. The dispatch pass opens its turn with `admissionPass`,
|
|
26
|
+
* which folds the live jobs, the settle window and the pass's own
|
|
27
|
+
* reservations into one judgment per node. The judge a settlement starts
|
|
28
|
+
* itself, the bounded re-ask
|
|
29
|
+
* (`review.mjs`) and the gate revision (`settle.mjs` `applyRejection`) still
|
|
30
|
+
* spawn with no call here: the node's own settle-window hold keeps the global
|
|
31
|
+
* `maxParallel` true while they run, but their per-runtime ceiling is not
|
|
32
|
+
* re-checked -- wiring those spawns through this gate is the follow-up node's.
|
|
14
33
|
*/
|
|
15
34
|
|
|
16
35
|
/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
|
|
@@ -33,6 +52,65 @@ export function runningPerRuntime(running) {
|
|
|
33
52
|
return counts;
|
|
34
53
|
}
|
|
35
54
|
|
|
55
|
+
/**
|
|
56
|
+
* The admission a dispatch pass judges a spawn against: every attempt holding
|
|
57
|
+
* a run slot right now. A live job holds one, and so does a closed job whose
|
|
58
|
+
* settlement has not finished -- a node whose worker exited but whose
|
|
59
|
+
* controller verification, judge round or integration is still running keeps
|
|
60
|
+
* the global slot and the runtime slot its dispatch reserved, because the
|
|
61
|
+
* judge that settlement is about to start is the reason the slot existed.
|
|
62
|
+
* Releasing at close instead is the measured excess (2026-09-29, achado 2 of
|
|
63
|
+
* ACHADOS-PRODUTO.md): two workers alive on one runtime under `maxParallel:
|
|
64
|
+
* 1`, and a sibling dispatched while the first node sat blocked in its
|
|
65
|
+
* verification gate.
|
|
66
|
+
*
|
|
67
|
+
* @param {Map<string, {runtime: {id: string|null}}>} running live jobs the tick is holding
|
|
68
|
+
* @param {Map<string, {runtime: {id: string|null}}>} settling closed jobs whose settlement has not finished
|
|
69
|
+
* @returns {{count: number, perRuntime: Map<string, number>}} the global slot
|
|
70
|
+
* count and the per-runtime counts; the dispatch pass grows both as it
|
|
71
|
+
* reserves, exactly as it grew `runningPerRuntime`'s map before
|
|
72
|
+
*/
|
|
73
|
+
export function admissionHold(running, settling) {
|
|
74
|
+
const holders = [...running.values(), ...settling.values()];
|
|
75
|
+
return { count: holders.length, perRuntime: runningPerRuntime(holders) };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* One dispatch pass's admission: the slots the contract's `maxParallel` still
|
|
80
|
+
* grants, judged against every attempt holding one -- the live jobs plus the
|
|
81
|
+
* closed jobs still inside their settle window -- and the per-runtime
|
|
82
|
+
* reservation the pass takes as it dispatches. `take` is consulted per node
|
|
83
|
+
* and refuses a runtime a sibling is waiting out a quota reset on, a runtime
|
|
84
|
+
* at its `maxConcurrent`, and one already holding its share of this pass's
|
|
85
|
+
* reservations; on success it reserves the slot in the same breath, because a
|
|
86
|
+
* check without its reservation would admit two nodes on one judgment. A
|
|
87
|
+
* spawn the settlement starts itself (the node's judge) runs inside the
|
|
88
|
+
* node's own held slot and never reaches here.
|
|
89
|
+
*
|
|
90
|
+
* @param {ValidatedContract} contract
|
|
91
|
+
* @param {Map<string, {runtime: {id: string|null}}>} running live jobs the tick is holding
|
|
92
|
+
* @param {Map<string, {runtime: {id: string|null}}>} settling closed jobs whose settlement has not finished
|
|
93
|
+
* @param {Map<string, NodeSnapshot>} states
|
|
94
|
+
* @returns {{slots: number, take: (runtimeId: string) => boolean}} the free
|
|
95
|
+
* slot count, read once per pass, and the per-node gate
|
|
96
|
+
*/
|
|
97
|
+
export function admissionPass(contract, running, settling, states) {
|
|
98
|
+
const hold = admissionHold(running, settling);
|
|
99
|
+
const held = quotaHeldRuntimes(states.values(), Date.now());
|
|
100
|
+
return {
|
|
101
|
+
slots: contract.maxParallel - hold.count,
|
|
102
|
+
/**
|
|
103
|
+
* @param {string} runtimeId
|
|
104
|
+
* @returns {boolean}
|
|
105
|
+
*/
|
|
106
|
+
take: (runtimeId) => {
|
|
107
|
+
if (!runtimeHasCapacity(runtimeId, contract, hold.perRuntime, held)) return false;
|
|
108
|
+
hold.perRuntime.set(runtimeId, (hold.perRuntime.get(runtimeId) ?? 0) + 1);
|
|
109
|
+
return true;
|
|
110
|
+
},
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
36
114
|
/**
|
|
37
115
|
* Runtimes some node is waiting to use again after a provider exhaustion:
|
|
38
116
|
* the node's current routing override still has a `backoffUntil` in the
|
package/src/engine/dispatch.mjs
CHANGED
|
@@ -4,15 +4,22 @@
|
|
|
4
4
|
* record the invocation intent before a token is spent.
|
|
5
5
|
*
|
|
6
6
|
* It decides nothing about outcomes. `startJudge` runs the mechanical gate and
|
|
7
|
-
* returns what it found -- `rejected`, `settle` or `dispatched` --
|
|
8
|
-
* caller to act on. It used to call `applyRejection` and `settleDone`
|
|
9
|
-
* which made dispatch depend on review policy and on settlement, and
|
|
10
|
-
* the shape that kept `engine/` a web instead of a stack.
|
|
7
|
+
* returns what it found -- `rejected`, `refused`, `settle` or `dispatched` --
|
|
8
|
+
* for the caller to act on. It used to call `applyRejection` and `settleDone`
|
|
9
|
+
* itself, which made dispatch depend on review policy and on settlement, and
|
|
10
|
+
* that is the shape that kept `engine/` a web instead of a stack.
|
|
11
|
+
*
|
|
12
|
+
* Every new paid dispatch is also admitted through the campaign's optional
|
|
13
|
+
* reserve (ADR 0011) before the provider starts; the admission decision lives
|
|
14
|
+
* in `reserve-admission.mjs`, while the release and the reconciliation
|
|
15
|
+
* against the real charge stay here because the reservation lives exactly as
|
|
16
|
+
* long as the call. One-turn result
|
|
17
|
+
* materialization continues an already-running session and stays on its
|
|
18
|
+
* existing path.
|
|
11
19
|
*/
|
|
12
|
-
import { JUDGE_SCHEMA, appendSandboxNotice, judgePrompt } from "./prompts.mjs";
|
|
20
|
+
import { JUDGE_SCHEMA, appendSandboxNotice, judgePrompt, splitPreviousAttemptSection } from "./prompts.mjs";
|
|
13
21
|
import { LockLostError } from "../run/lock.mjs";
|
|
14
22
|
import { TOOL_OUTPUT_LIMIT_BYTES } from "../harnesses/exec-jsonl/index.mjs";
|
|
15
|
-
import { appendPreviousAttempt } from "./retry.mjs";
|
|
16
23
|
import {
|
|
17
24
|
RESULT_MATERIALIZATION_PROMPT_HEADER,
|
|
18
25
|
attemptWorkerResultPath,
|
|
@@ -30,16 +37,18 @@ import { captureWorkspaceScope, captureWorkspaceSnapshot } from "../repo/workspa
|
|
|
30
37
|
import { deterministicGate, judgeReaskReason, judgeRequired, judgeSkippedByScope } from "./judge-gate.mjs";
|
|
31
38
|
import { emptyScope, persistedScopeBoundary, workerScope } from "./scope.mjs";
|
|
32
39
|
import { hasOperationIntent, hasOperationSettlement, operationNeedsRecovery, operationNextState, persistInvocationIntent, providerReceipts, settleInvocation } from "../run/operations.mjs";
|
|
33
|
-
import { invocationCost, invocationUsage } from "../run/usage.mjs";
|
|
40
|
+
import { invocationCost, invocationUsage, priceUsage } from "../run/usage.mjs";
|
|
34
41
|
import { logPaths, startProcess } from "./process.mjs";
|
|
35
42
|
import { readBoundedTail } from "./transcript.mjs";
|
|
36
43
|
import { mkdirSync, statSync } from "node:fs";
|
|
44
|
+
import { reconcileCampaignReservation, releaseCampaignReservation } from "../campaign/reserve.mjs";
|
|
37
45
|
import { READ_BYTE_LIMIT, READ_LINE_LIMIT, harnessCapabilities, normalizeProviderResult, writesWorkspace } from "../harnesses/index.mjs";
|
|
38
46
|
import { writeJsonAtomic } from "../run/store.mjs";
|
|
39
47
|
import { judgeReaskInstruction, reviewMode } from "../contract/review-modes.mjs";
|
|
40
48
|
import { isSameProviderReviewPair } from "../contract/judge-independence.mjs";
|
|
41
49
|
import { previousAttemptRuntimeId, routeRuntimeForState, runtimeSnapshot } from "./failover.mjs";
|
|
42
50
|
import { fingerprintRuntime, forceFreshSession, phaseInvocationPlan } from "./phase-session.mjs";
|
|
51
|
+
import { releaseReservation, reserveAdmission } from "./reserve-admission.mjs";
|
|
43
52
|
|
|
44
53
|
/** @typedef {import("./phase-session.mjs").SessionPolicy} SessionPolicy */
|
|
45
54
|
import { transition, writeNode } from "./state.mjs";
|
|
@@ -50,8 +59,12 @@ import { transition, writeNode } from "./state.mjs";
|
|
|
50
59
|
* `settle`: the gate passed and no judgment item needs arbitrating.
|
|
51
60
|
* `dispatched`: a judge is running (or failed to start and the node is already
|
|
52
61
|
* marked failed).
|
|
62
|
+
* `refused`: the dispatch admission declined the new judge dispatch -- the
|
|
63
|
+
* campaign reserve cannot hold it, or the campaign record cannot be read or
|
|
64
|
+
* validated; the node is
|
|
65
|
+
* already parked blocked and no judge will run this round.
|
|
53
66
|
*
|
|
54
|
-
* @typedef {{kind: "rejected", verdict: JudgeVerdict}|{kind: "settle", gate: JudgeVerdict|null}|{kind: "dispatched"}} JudgeRound
|
|
67
|
+
* @typedef {{kind: "rejected", verdict: JudgeVerdict}|{kind: "settle", gate: JudgeVerdict|null}|{kind: "dispatched"}|{kind: "refused"}} JudgeRound
|
|
55
68
|
*/
|
|
56
69
|
/** @typedef {import("./prompts.mjs").JudgeVerdict} JudgeVerdict */
|
|
57
70
|
|
|
@@ -232,6 +245,41 @@ function ensureAttemptWorkspace(contract, node, state, runDir, lock) {
|
|
|
232
245
|
writeNode(runDir, state, lock);
|
|
233
246
|
return worktree.path;
|
|
234
247
|
}
|
|
248
|
+
/**
|
|
249
|
+
* The final assembly every dispatched prompt goes through: the stable
|
|
250
|
+
* instruction sections — the caller's stable appends, run on the stable
|
|
251
|
+
* remainder (the worker's sandbox notice and result-file protocol, the
|
|
252
|
+
* judge's re-ask instruction) — precede the variable previous-attempt
|
|
253
|
+
* state, so the cacheable prefix survives across attempts. A base that
|
|
254
|
+
* already carries the section (a candidate reused from an earlier attempt)
|
|
255
|
+
* has it lifted out and re-appended last; a base without one gets the state's
|
|
256
|
+
* own section appended, and a state that carries none appends nothing. When
|
|
257
|
+
* both are present the state's section wins: an embedded one is the stale
|
|
258
|
+
* copy the reused candidate carries, from an earlier record than
|
|
259
|
+
* `state.previousAttempt`, and it is lifted out rather than re-appended. The
|
|
260
|
+
* append is a plain join, not `appendPreviousAttempt`: that helper refuses
|
|
261
|
+
* any base carrying a `Previous attempt` markdown heading — the same false
|
|
262
|
+
* positive the splitter just learned to see through, since a packet may
|
|
263
|
+
* legitimately head its own instructions `## Previous attempt notes:` — and
|
|
264
|
+
* silently dropped the state's retry record on exactly those prompts. The
|
|
265
|
+
* splitter guarantees the stable remainder carries no lifted record, so the
|
|
266
|
+
* join cannot double-append. The 64 KiB dispatch budget is enforced by the
|
|
267
|
+
* caller right after this returns, on the assembled prompt.
|
|
268
|
+
*
|
|
269
|
+
* @param {string} basePrompt
|
|
270
|
+
* @param {{sandbox?: (prompt: string) => string, protocol?: (prompt: string) => string, stable?: (prompt: string) => string, section?: string}} assembly
|
|
271
|
+
* @returns {string}
|
|
272
|
+
*/
|
|
273
|
+
export function assembleDispatchPrompt(basePrompt, { sandbox, protocol, stable, section } = {}) {
|
|
274
|
+
const { stable: head, section: embedded } = splitPreviousAttemptSection(basePrompt);
|
|
275
|
+
let ordered = head;
|
|
276
|
+
if (sandbox) ordered = sandbox(ordered);
|
|
277
|
+
if (protocol) ordered = protocol(ordered);
|
|
278
|
+
if (stable) ordered = stable(ordered);
|
|
279
|
+
const variable = section ?? embedded;
|
|
280
|
+
return variable ? `${ordered}\n\n${variable}` : ordered;
|
|
281
|
+
}
|
|
282
|
+
|
|
235
283
|
/**
|
|
236
284
|
* @param {ValidatedContract} contract
|
|
237
285
|
* @param {ValidatedNode} node
|
|
@@ -261,17 +309,20 @@ export function startWorker(contract, node, state, runDir, running, prompt, lock
|
|
|
261
309
|
}
|
|
262
310
|
const runtime = routeRuntimeForState(contract, node, state, "worker");
|
|
263
311
|
const phasePlan = phaseInvocationPlan(contract, node, state, runDir, "worker", prompt, sessionPolicy);
|
|
264
|
-
// The previous-attempt section still has to survive on a retried attempt,
|
|
265
|
-
// so it is appended to the resolved prompt rather than the candidate handed
|
|
266
|
-
// to phaseInvocationPlan.
|
|
267
|
-
phasePlan.prompt = appendPreviousAttempt(phasePlan.prompt, state.previousAttempt);
|
|
268
|
-
// The resolved worker's sandbox is a dispatch-time fact, not a packet one.
|
|
269
|
-
phasePlan.prompt = appendSandboxNotice(phasePlan.prompt, runtime);
|
|
270
312
|
// The worker prompt directs the provider to write the canonical result file;
|
|
271
313
|
// make sure the directory exists before the provider is asked to.
|
|
272
314
|
const resultPath = attemptWorkerResultPath(runDir, node.id, workspace);
|
|
273
315
|
mkdirSync(dirname(resultPath), { recursive: true });
|
|
274
|
-
|
|
316
|
+
// The resolved worker's sandbox is a dispatch-time fact, not a packet one,
|
|
317
|
+
// and the previous-attempt section has to survive on a retried attempt.
|
|
318
|
+
// Both go through the shared assembly: stable instructions (packet prompt,
|
|
319
|
+
// sandbox notice, result-file protocol) precede the variable section, which
|
|
320
|
+
// is lifted out of a base that already carries one and re-appended last.
|
|
321
|
+
const effectivePrompt = assembleDispatchPrompt(phasePlan.prompt, {
|
|
322
|
+
sandbox: (ordered) => appendSandboxNotice(ordered, runtime),
|
|
323
|
+
protocol: (ordered) => workerProtocolPrompt(ordered, resultPath, writesWorkspace(runtime)),
|
|
324
|
+
section: state.previousAttempt,
|
|
325
|
+
});
|
|
275
326
|
const paths = logPaths(runDir, node.id, "worker", state.attempt);
|
|
276
327
|
if (Buffer.byteLength(effectivePrompt, "utf8") > 64 * 1024) {
|
|
277
328
|
transition(runDir, state, "failed", { phase: "worker", error: { code: "worker_prompt_too_large", message: "worker prompt exceeds 65536 bytes" } }, lock);
|
|
@@ -323,6 +374,11 @@ export function startWorker(contract, node, state, runDir, running, prompt, lock
|
|
|
323
374
|
state.error = null;
|
|
324
375
|
state.scope = emptyScope(boundary);
|
|
325
376
|
writeNode(runDir, state, lock);
|
|
377
|
+
const admission = reserveAdmission(campaignPath, state, basename(runDir), node.id, "worker");
|
|
378
|
+
if (admission.refused) {
|
|
379
|
+
transition(runDir, state, "blocked", { phase: "worker", error: admission.error }, lock);
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
326
382
|
try {
|
|
327
383
|
const job = startProcess({
|
|
328
384
|
contract, node, state, runtime, workspace, prompt: effectivePrompt, paths, phase: "worker",
|
|
@@ -336,7 +392,7 @@ export function startWorker(contract, node, state, runDir, running, prompt, lock
|
|
|
336
392
|
stampInvocation(invocation, contract, node, runtime, state, runDir, "worker", phasePlan.mode, phasePlan.continuationId);
|
|
337
393
|
invocation.snapshotPath = snapshotPath;
|
|
338
394
|
currentJob.scopeBaseline = baseline;
|
|
339
|
-
persistInvocation(runDir, state, invocation, currentJob, lock);
|
|
395
|
+
persistInvocation(runDir, state, invocation, currentJob, lock, campaignPath, admission.reservation);
|
|
340
396
|
persistInvocationIntent(runDir, invocation, {
|
|
341
397
|
nodeId: node.id,
|
|
342
398
|
role: "worker",
|
|
@@ -351,6 +407,7 @@ export function startWorker(contract, node, state, runDir, running, prompt, lock
|
|
|
351
407
|
transition(runDir, state, "running", { phase: "worker", runtime, error: null }, lock);
|
|
352
408
|
running.set(node.id, job);
|
|
353
409
|
} catch (error) {
|
|
410
|
+
releaseReservation(campaignPath, admission.reservation);
|
|
354
411
|
const invocation = state.invocations?.at(-1);
|
|
355
412
|
if (invocation && hasOperationIntent(runDir, invocation.id) && operationNeedsRecovery(runDir, invocation.id)) {
|
|
356
413
|
settleInvocation(runDir, invocation, {
|
|
@@ -527,18 +584,31 @@ export async function startJudge(contract, node, state, runDir, running, workerR
|
|
|
527
584
|
if (previousInvocation && hasOperationSettlement(runDir, previousInvocation.id)) {
|
|
528
585
|
settleInvocation(runDir, previousInvocation, { nextState: operationNextState(state) });
|
|
529
586
|
}
|
|
587
|
+
const admission = reserveAdmission(campaignPath, state, basename(runDir), node.id, "judge");
|
|
588
|
+
if (admission.refused) {
|
|
589
|
+
transition(runDir, state, "blocked", { phase: "judge", error: admission.error }, lock);
|
|
590
|
+
return { kind: "refused" };
|
|
591
|
+
}
|
|
530
592
|
try {
|
|
531
|
-
const prompt =
|
|
593
|
+
const prompt = judgePrompt(node, workerResult, {
|
|
532
594
|
diff: state.scope?.changedPaths,
|
|
533
595
|
verification: state.verification,
|
|
534
596
|
deterministic: results,
|
|
535
597
|
scopeFindings: state.scopeFindings,
|
|
536
598
|
previousAttempt: state.previousAttempt,
|
|
537
|
-
})
|
|
599
|
+
});
|
|
538
600
|
const phasePlan = phaseInvocationPlan(contract, node, state, runDir, "judge", prompt);
|
|
539
|
-
//
|
|
540
|
-
//
|
|
541
|
-
|
|
601
|
+
// The same ordering invariant as the worker prompt: the variable section
|
|
602
|
+
// stays the trailing block, lifted out of a reused candidate and
|
|
603
|
+
// re-appended last — byte-stable when judgePrompt already trailed it. The
|
|
604
|
+
// re-ask instruction rides the stable remainder like any other stable
|
|
605
|
+
// append: appended to the raw base instead, it landed after the trailing
|
|
606
|
+
// section and the assembly's state-section replacement dropped it with
|
|
607
|
+
// the lifted block — a re-asked judge dispatched without its instruction.
|
|
608
|
+
phasePlan.prompt = assembleDispatchPrompt(phasePlan.prompt, {
|
|
609
|
+
stable: reask ? (ordered) => `${ordered}${judgeReaskInstruction(reaskReason)}` : undefined,
|
|
610
|
+
section: state.previousAttempt,
|
|
611
|
+
});
|
|
542
612
|
if (Buffer.byteLength(phasePlan.prompt, "utf8") > 64 * 1024) {
|
|
543
613
|
const error = /** @type {Error & {code: string}} */ (new Error("judge prompt exceeds 65536 bytes"));
|
|
544
614
|
error.code = "judge_prompt_too_large";
|
|
@@ -571,7 +641,7 @@ export async function startJudge(contract, node, state, runDir, running, workerR
|
|
|
571
641
|
// worker and a judge, and this field carries no persisted shape of
|
|
572
642
|
// its own that a judge borrowing it would have to match.
|
|
573
643
|
currentJob.scopeBaseline = judgeBaseline;
|
|
574
|
-
persistInvocation(runDir, state, invocation, currentJob, lock);
|
|
644
|
+
persistInvocation(runDir, state, invocation, currentJob, lock, campaignPath, admission.reservation);
|
|
575
645
|
persistInvocationIntent(runDir, invocation, {
|
|
576
646
|
nodeId: node.id,
|
|
577
647
|
role: "judge",
|
|
@@ -585,6 +655,7 @@ export async function startJudge(contract, node, state, runDir, running, workerR
|
|
|
585
655
|
transition(runDir, state, "running", { phase: "judge", runtime }, lock);
|
|
586
656
|
running.set(node.id, job);
|
|
587
657
|
} catch (error) {
|
|
658
|
+
releaseReservation(campaignPath, admission.reservation);
|
|
588
659
|
const invocation = state.invocations?.at(-1);
|
|
589
660
|
if (invocation && hasOperationIntent(runDir, invocation.id) && operationNeedsRecovery(runDir, invocation.id)) {
|
|
590
661
|
settleInvocation(runDir, invocation, {
|
|
@@ -604,8 +675,12 @@ export async function startJudge(contract, node, state, runDir, running, workerR
|
|
|
604
675
|
* @param {Invocation} invocation
|
|
605
676
|
* @param {Job} job
|
|
606
677
|
* @param {LockHandle} lock
|
|
678
|
+
* @param {string} [campaignPath] absent when the call took no admission
|
|
679
|
+
* @param {import("../campaign/reserve.mjs").Reservation|null} [reservation] the
|
|
680
|
+
* hold this dispatch was admitted under (ADR 0011); reconciled with the
|
|
681
|
+
* charge that actually arrived, or released when the call ended with none
|
|
607
682
|
*/
|
|
608
|
-
function persistInvocation(runDir, state, invocation, job, lock) {
|
|
683
|
+
function persistInvocation(runDir, state, invocation, job, lock, campaignPath, reservation) {
|
|
609
684
|
state.invocations = [...(state.invocations ?? []), invocation];
|
|
610
685
|
state.updatedAt = invocation.updatedAt;
|
|
611
686
|
writeNode(runDir, state, lock);
|
|
@@ -636,6 +711,24 @@ function persistInvocation(runDir, state, invocation, job, lock) {
|
|
|
636
711
|
state.costUsd = invocationCost(state);
|
|
637
712
|
state.updatedAt = closed.updatedAt;
|
|
638
713
|
writeNode(runDir, state, lock);
|
|
714
|
+
if (campaignPath && reservation) {
|
|
715
|
+
// The close envelope carries raw counters -- the priced charge only
|
|
716
|
+
// exists once settlement prices it. `priceUsage` is the same pure
|
|
717
|
+
// function the settlement ledger runs over the same transcript, so
|
|
718
|
+
// the reservation reconciles against the amount the invocation will
|
|
719
|
+
// carry rather than a second derivation of it. A charge the envelope
|
|
720
|
+
// cannot price releases the hold: the reserve gates new dispatches,
|
|
721
|
+
// and the measured truth stays in the usage ledger.
|
|
722
|
+
const charge = priceUsage(job.runtime, usage, typeof costUsd === "number" ? costUsd : null).costUsd;
|
|
723
|
+
try {
|
|
724
|
+
if (typeof charge === "number") reconcileCampaignReservation(campaignPath, reservation.id, charge);
|
|
725
|
+
else releaseCampaignReservation(campaignPath, reservation.id);
|
|
726
|
+
} catch {
|
|
727
|
+
// A hold that cannot settle stays visible in `readCampaignReserve`
|
|
728
|
+
// until an operator releases it; the node's own settlement must
|
|
729
|
+
// never fail for the campaign-level ledger.
|
|
730
|
+
}
|
|
731
|
+
}
|
|
639
732
|
settleInvocation(runDir, completed, {
|
|
640
733
|
status: envelopeStatus,
|
|
641
734
|
usage,
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The cause a failure that would fire a repair trigger failed for, drawn from
|
|
3
|
+
* a fixed vocabulary.
|
|
4
|
+
*
|
|
5
|
+
* The reauthor widens a refused packet and its round budget bounds only one
|
|
6
|
+
* invocation; what lets a later trigger refuse to repeat a repair that already
|
|
7
|
+
* ran to no effect is the durable record of which cause fired, on which node,
|
|
8
|
+
* at which artifact version -- the same cause on the same node and artifact
|
|
9
|
+
* version must never fire indefinite calls. The classes are read from the
|
|
10
|
+
* refused packet's own grants: a missing entry granted nowhere is a read gap,
|
|
11
|
+
* one readable but not writable is a write gap, and prose is requirement
|
|
12
|
+
* ambiguity, which no widening repairs and which becomes a durable question
|
|
13
|
+
* instead of another discovery round.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
/** The cause vocabulary `classifyFailureCause` draws from. */
|
|
17
|
+
export const FAILURE_CAUSE_CLASSES = ["missing_read_scope", "missing_write_scope", "ambiguous_requirement", "unclassified"];
|
|
18
|
+
|
|
19
|
+
/** One classified failure: the key a repair trigger is recorded under. */
|
|
20
|
+
/** @typedef {{cause: string, node: string, artifactVersion: string|null}} FailureCause */
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* A missing-context entry naming a path rather than prose: a path separator,
|
|
24
|
+
* or an extension at the end. `src/extra.mjs` and `README.md` are paths;
|
|
25
|
+
* "clearer acceptance criteria" is not.
|
|
26
|
+
*/
|
|
27
|
+
const PATH_LIKE = /[/\\]|\.[A-Za-z][A-Za-z0-9]{0,7}$/;
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Classify the failure of a node a reauthor would fire on -- a packet refused
|
|
31
|
+
* on `context_missing`. Ambiguity dominates, then read gaps: a discovery pass
|
|
32
|
+
* that cannot settle what is being asked cannot spend its budget usefully. A
|
|
33
|
+
* refusal naming nothing classifiable is recorded as `unclassified`, never
|
|
34
|
+
* dropped.
|
|
35
|
+
*
|
|
36
|
+
* @param {import("../contract/index.mjs").ValidatedNode} node
|
|
37
|
+
* @param {import("../contract/index.mjs").NodeSnapshot} state
|
|
38
|
+
* @returns {FailureCause}
|
|
39
|
+
*/
|
|
40
|
+
export function classifyFailureCause(node, state) {
|
|
41
|
+
const refused = /** @type {{missingContext?: string[]}|null|undefined} */ (state.result);
|
|
42
|
+
const missingContext = Array.isArray(refused?.missingContext) ? refused.missingContext : [];
|
|
43
|
+
const artifactVersion = state.packetHash ?? null;
|
|
44
|
+
if (missingContext.length === 0) return { cause: "unclassified", node: node.id, artifactVersion };
|
|
45
|
+
const read = new Set(node.taskPacket.readFiles ?? []);
|
|
46
|
+
const write = new Set(node.taskPacket.writeFiles ?? []);
|
|
47
|
+
let ambiguous = false;
|
|
48
|
+
let readGap = false;
|
|
49
|
+
let writeGap = false;
|
|
50
|
+
for (const entry of missingContext) {
|
|
51
|
+
if (typeof entry !== "string" || !PATH_LIKE.test(entry)) ambiguous = true;
|
|
52
|
+
else if (!read.has(entry) && !write.has(entry)) readGap = true;
|
|
53
|
+
else if (!write.has(entry)) writeGap = true;
|
|
54
|
+
}
|
|
55
|
+
let cause = "missing_read_scope";
|
|
56
|
+
if (writeGap && !readGap) cause = "missing_write_scope";
|
|
57
|
+
if (ambiguous) cause = "ambiguous_requirement";
|
|
58
|
+
return { cause, node: node.id, artifactVersion };
|
|
59
|
+
}
|
package/src/engine/lifecycle.mjs
CHANGED
|
@@ -55,7 +55,8 @@ import {
|
|
|
55
55
|
workerResultPath,
|
|
56
56
|
} from "./result-file.mjs";
|
|
57
57
|
import { canReuseResultEvidence, checkResultMaterializationScope, checkWorkerScope, recordScopeFinding, sourceWorkerRuntime } from "./scope.mjs";
|
|
58
|
-
import {
|
|
58
|
+
import { startMechanicalGate, startResultMaterialization } from "./dispatch.mjs";
|
|
59
|
+
import { auxJudgeRound } from "./aux-admission.mjs";
|
|
59
60
|
import { raiseNodeAttention, settleDone } from "./settle.mjs";
|
|
60
61
|
import { applyRejection, applyVerificationFailure } from "./settle.mjs";
|
|
61
62
|
import { emitNodeAdvisories } from "./notify-queue.mjs";
|
|
@@ -591,7 +592,7 @@ export async function finalizeClosedJobs(contract, runDir, states, closed, lock,
|
|
|
591
592
|
}
|
|
592
593
|
if (job.scopeViolation) recordScopeFinding(runDir, state, lock);
|
|
593
594
|
const round = job.node.gate.enabled
|
|
594
|
-
? await
|
|
595
|
+
? await auxJudgeRound(contract, job.node, state, runDir, running, states, lock, campaignPath)
|
|
595
596
|
: await startMechanicalGate(contract, job.node, state);
|
|
596
597
|
await applyJudgeRound(round, contract, job.node, state, runDir, running, lock, states, campaignPath, workerResult);
|
|
597
598
|
continue;
|