faberun 0.25.1 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/faberun/references/contract.md +30 -26
- package/src/campaign/campaign-brief-graph.mjs +218 -0
- package/src/campaign/campaign-brief.mjs +15 -167
- package/src/campaign/metrics-command.mjs +3 -1
- package/src/cli/brand.mjs +1 -0
- package/src/cli/plan.mjs +75 -3
- package/src/cli/setup.mjs +22 -5
- package/src/cli.mjs +35 -1
- package/src/contract/definition-of-done.mjs +48 -8
- package/src/contract/human-step.mjs +54 -0
- package/src/contract/index.mjs +25 -38
- package/src/contract/judge-independence.mjs +170 -0
- package/src/contract/judges.mjs +27 -0
- package/src/contract/provider.mjs +103 -0
- package/src/contract/runtime.mjs +19 -2
- package/src/contract/snapshot.mjs +31 -1
- package/src/engine/assignment.mjs +41 -5
- package/src/engine/backoff.mjs +30 -9
- package/src/engine/dispatch.mjs +19 -2
- package/src/engine/failover.mjs +10 -2
- package/src/engine/judge-gate.mjs +24 -4
- package/src/engine/judge-list.mjs +206 -0
- package/src/engine/lifecycle.mjs +19 -16
- package/src/engine/resume.mjs +46 -9
- package/src/engine/runtime-discovery.mjs +72 -11
- package/src/engine/scheduler.mjs +27 -6
- package/src/harnesses/catalogue.mjs +9 -2
- package/src/harnesses/index.mjs +2 -2
- package/src/harnesses/model-tiers.mjs +39 -0
- package/src/harnesses/protocol.mjs +4 -1
- package/src/host/config.mjs +7 -2
- package/src/plan/contest.mjs +78 -0
- package/src/plan/freeze.mjs +118 -7
- package/src/plan/human-step.mjs +38 -0
- package/src/plan/pipeline.mjs +224 -313
- package/src/plan/preflight.mjs +5 -3
- package/src/plan/proof-check.mjs +161 -0
- package/src/plan/proof-run.mjs +6 -2
- package/src/plan/proof-scope.mjs +175 -0
- package/src/plan/resolve.mjs +182 -0
- package/src/plan/reviewer.mjs +82 -0
- package/src/plan/rounds.mjs +446 -0
- package/src/plan/routing.mjs +11 -4
- package/src/plan/template.mjs +36 -14
- package/src/repo/signal-block.mjs +10 -3
- package/src/repo/source-identity.mjs +45 -5
- package/src/repo/worktree-prune.mjs +72 -0
- package/src/report/campaign-brief-estimate.mjs +1 -1
- package/src/report/campaign-brief.mjs +4 -0
- package/src/report/metrics-report.mjs +34 -1
- package/src/report/render.mjs +31 -10
package/src/engine/failover.mjs
CHANGED
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
import { harnessCapabilities } from "../harnesses/index.mjs";
|
|
18
18
|
import { isRuntimeAvailable, nextSameTierRuntime } from "./runtime-discovery.mjs";
|
|
19
19
|
import { routeRuntime } from "../contract/runtime.mjs";
|
|
20
|
+
import { effectiveProvider } from "../contract/provider.mjs";
|
|
20
21
|
|
|
21
22
|
/** @typedef {import("../contract/index.mjs").ValidatedNode} ValidatedNode */
|
|
22
23
|
/** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
|
|
@@ -188,11 +189,17 @@ export function routingBackoffActive(state, phase) {
|
|
|
188
189
|
* survives both a routed hop and the revision boundary: `resetPhaseRouting`
|
|
189
190
|
* clears the routing override between revisions, but never the invocations.
|
|
190
191
|
*
|
|
192
|
+
* Exported so a same-provider-review mark (R20) can be taken from the worker
|
|
193
|
+
* invocation that actually ran instead of `state.runtime`, which `startJudge`
|
|
194
|
+
* overwrites with the judge's own runtime as soon as it dispatches -- a re-ask
|
|
195
|
+
* or a judge-failure retry then calls `startJudge` again with the judge still
|
|
196
|
+
* in `state.runtime`, so comparing against it compares the judge to itself.
|
|
197
|
+
*
|
|
191
198
|
* @param {NodeSnapshot} state
|
|
192
199
|
* @param {"worker"|"judge"} role
|
|
193
200
|
* @returns {string|undefined}
|
|
194
201
|
*/
|
|
195
|
-
function previousAttemptRuntimeId(state, role) {
|
|
202
|
+
export function previousAttemptRuntimeId(state, role) {
|
|
196
203
|
const found = [...(state.invocations ?? [])].reverse().find((invocation) => invocation.phase === role)?.runtimeId;
|
|
197
204
|
return typeof found === "string" ? found : undefined;
|
|
198
205
|
}
|
|
@@ -223,7 +230,8 @@ function admitsAffinity(contract, state, role, candidate) {
|
|
|
223
230
|
if (availability && !isRuntimeAvailable(availability)) return false;
|
|
224
231
|
if (role !== "judge") return true;
|
|
225
232
|
const workerId = previousAttemptRuntimeId(state, "worker") ?? state.routing?.assignments?.worker;
|
|
226
|
-
|
|
233
|
+
const workerRuntime = workerId ? contract.runtimes[workerId] : undefined;
|
|
234
|
+
return effectiveProvider(runtime) !== (workerRuntime ? effectiveProvider(workerRuntime) : null);
|
|
227
235
|
}
|
|
228
236
|
|
|
229
237
|
/**
|
|
@@ -142,8 +142,9 @@ function proveVerification(id, proof, recorded) {
|
|
|
142
142
|
* `--test-skip-pattern`) is judged on more than its exit code, because the
|
|
143
143
|
* runner exits 0 whether its filter selected anything or not. The proof runs
|
|
144
144
|
* with the TAP reporter selected through `NODE_OPTIONS` and is refused when the
|
|
145
|
-
* output
|
|
146
|
-
*
|
|
145
|
+
* output says the filter selected nothing (`tapSelectedNothing`). Measured
|
|
146
|
+
* 2026-09-21 on node v26.8.1, in a single-file run: a filter that matches
|
|
147
|
+
* emits `1..0` zero times, a filter that matches nothing
|
|
147
148
|
* emits it exactly once, a run with no filter emits it zero times, and an empty
|
|
148
149
|
* suite under a matching filter emits no nested zero plan. The default reporter
|
|
149
150
|
* cannot make the distinction -- both cases print identical counters, because
|
|
@@ -154,7 +155,26 @@ function proveVerification(id, proof, recorded) {
|
|
|
154
155
|
* never sees it.
|
|
155
156
|
*/
|
|
156
157
|
const TEST_FILTER_FLAGS = ["--test-name-pattern", "--test-skip-pattern"];
|
|
157
|
-
const TAP_ZERO_PLAN = /^1\.\.0$/
|
|
158
|
+
const TAP_ZERO_PLAN = /^1\.\.0$/gmu;
|
|
159
|
+
const TAP_TOP_LEVEL_RESULT = /^(?:not )?ok \d+ /gmu;
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Whether a filtered run's TAP says its filter selected no test. A run of
|
|
163
|
+
* several files prints, for each file the filter selected nothing in, an
|
|
164
|
+
* unindented `1..0` followed by that file's own top-level `ok` line, while a
|
|
165
|
+
* selected test prints its own top-level result with no zero plan in front of
|
|
166
|
+
* it. Measured 2026-09-26 on node v26.8.1 (AP1 of safe-to-hand-to-a-friend):
|
|
167
|
+
* reading a single `1..0` as "nothing selected" refused two proofs whose
|
|
168
|
+
* filter had selected a passing test in one of several files. Nothing was
|
|
169
|
+
* selected exactly when every top-level result is one of those file lines.
|
|
170
|
+
*
|
|
171
|
+
* @param {string} stdout
|
|
172
|
+
* @returns {boolean}
|
|
173
|
+
*/
|
|
174
|
+
function tapSelectedNothing(stdout) {
|
|
175
|
+
const zeroPlans = stdout.match(TAP_ZERO_PLAN)?.length ?? 0;
|
|
176
|
+
return zeroPlans > 0 && zeroPlans >= (stdout.match(TAP_TOP_LEVEL_RESULT)?.length ?? 0);
|
|
177
|
+
}
|
|
158
178
|
|
|
159
179
|
/**
|
|
160
180
|
* The environment a filtered proof runs under: the ambient environment with
|
|
@@ -252,7 +272,7 @@ async function proveCommand(id, proof, cwd, timeoutMs) {
|
|
|
252
272
|
child.on("close", (code, signal) => {
|
|
253
273
|
// Its own detail, not an ordinary command failure: the command succeeded,
|
|
254
274
|
// so what failed is that the declared filter selected nothing to prove.
|
|
255
|
-
if (code === 0 && signal === null && filters.length > 0 &&
|
|
275
|
+
if (code === 0 && signal === null && filters.length > 0 && tapSelectedNothing(stdout)) {
|
|
256
276
|
finish({
|
|
257
277
|
id,
|
|
258
278
|
kind: "command",
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* R18: the judge of a gated node is the first eligible entry of a static,
|
|
3
|
+
* ordered list rather than the single strongest cross-vendor candidate
|
|
4
|
+
* `composeAssignments` otherwise ranks. Split out of `runtime-discovery.mjs`
|
|
5
|
+
* and `lifecycle.mjs` -- both already near this repository's 800-line
|
|
6
|
+
* ceiling -- and kept out of `backoff.mjs`, whose `planRoute` is a pure
|
|
7
|
+
* decision this module cannot be: picking an entry reads the durable refusal
|
|
8
|
+
* and usage-window stores.
|
|
9
|
+
*/
|
|
10
|
+
import { effectiveProvider } from "../contract/provider.mjs";
|
|
11
|
+
import { availabilityKey, readRefusal } from "../run/availability.mjs";
|
|
12
|
+
import { readUsageWindows, usageAccountOf } from "../run/usage-windows.mjs";
|
|
13
|
+
import { getHarness } from "../harnesses/index.mjs";
|
|
14
|
+
|
|
15
|
+
/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
|
|
16
|
+
/** @typedef {{id: string, reason: string}} JudgeSkip */
|
|
17
|
+
/** @typedef {{chosen: string|null, skipped: JudgeSkip[]}} JudgeListPick */
|
|
18
|
+
/** @typedef {{list: string[], chosen: string|null, skipped: JudgeSkip[]}} JudgeListState */
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* A usage window at or above this spend is treated as unavailable for a judge
|
|
22
|
+
* pick, so a node's review does not race the account into a refusal the next
|
|
23
|
+
* call would hit anyway.
|
|
24
|
+
*/
|
|
25
|
+
export const JUDGE_USAGE_WINDOW_LIMIT_PERCENT = 90;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* The static judge list a node's assignment reads: the contract's own list
|
|
29
|
+
* wins over the machine default, exactly as `runtimeDefaults.worker` already
|
|
30
|
+
* wins over `config.worker` in `composeAssignments`.
|
|
31
|
+
*
|
|
32
|
+
* @param {{judges?: string[]}} contract
|
|
33
|
+
* @param {{judges?: string[]}|null|undefined} config
|
|
34
|
+
* @returns {string[]|undefined}
|
|
35
|
+
*/
|
|
36
|
+
export function resolveJudgeList(contract, config) {
|
|
37
|
+
return contract.judges ?? config?.judges ?? undefined;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Every provider a worker attempt on this node could actually run on: its own
|
|
42
|
+
* canonical provider, plus every hop of its declared fallback chain. A judge
|
|
43
|
+
* of any of these providers cannot independently review whichever one
|
|
44
|
+
* actually ran. Walked directly off `runtime.fallback` rather than through
|
|
45
|
+
* `failover.mjs`'s `synthesizedChain`, so this module does not import a
|
|
46
|
+
* runtime-discovery consumer and risk a cycle back through
|
|
47
|
+
* `run/availability.mjs`.
|
|
48
|
+
*
|
|
49
|
+
* @param {ValidatedContract} contract
|
|
50
|
+
* @param {string} workerId
|
|
51
|
+
* @returns {Set<string|undefined>}
|
|
52
|
+
*/
|
|
53
|
+
function reachableWorkerProviders(contract, workerId) {
|
|
54
|
+
const providers = new Set([effectiveProvider(contract.runtimes[workerId])]);
|
|
55
|
+
const seen = new Set([workerId]);
|
|
56
|
+
let current = workerId;
|
|
57
|
+
for (;;) {
|
|
58
|
+
const next = contract.runtimes[current]?.fallback;
|
|
59
|
+
if (!next || seen.has(next) || !contract.runtimes[next]) break;
|
|
60
|
+
seen.add(next);
|
|
61
|
+
providers.add(effectiveProvider(contract.runtimes[next]));
|
|
62
|
+
current = next;
|
|
63
|
+
}
|
|
64
|
+
return providers;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Is this runtime's account inside `JUDGE_USAGE_WINDOW_LIMIT_PERCENT` of a
|
|
69
|
+
* usage window closing? `null` means no window is known -- unknown must not
|
|
70
|
+
* read as over budget, exactly as an absent availability record must not read
|
|
71
|
+
* as exhausted (see `admitsAffinity` in `failover.mjs`).
|
|
72
|
+
*
|
|
73
|
+
* @param {import("../contract/index.mjs").ValidatedRuntime} runtime
|
|
74
|
+
* @param {number} now
|
|
75
|
+
* @returns {import("../harnesses/codex/usage-window.mjs").UsageWindow|null}
|
|
76
|
+
*/
|
|
77
|
+
function overspentWindow(runtime, now) {
|
|
78
|
+
const account = usageAccountOf(runtime);
|
|
79
|
+
if (!account) return null;
|
|
80
|
+
return readUsageWindows(account, now).find((window) => window.usedPercent > JUDGE_USAGE_WINDOW_LIMIT_PERCENT) ?? null;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* The first eligible entry of an ordered judge list, and why every earlier
|
|
85
|
+
* entry was skipped: already attempted this run, a provider the worker (or
|
|
86
|
+
* its fallback chain) already runs, a recorded refusal, or an account usage
|
|
87
|
+
* window over `JUDGE_USAGE_WINDOW_LIMIT_PERCENT`.
|
|
88
|
+
*
|
|
89
|
+
* @param {ValidatedContract} contract
|
|
90
|
+
* @param {string[]} list
|
|
91
|
+
* @param {string} workerId
|
|
92
|
+
* @param {{attempted?: Iterable<string>, now?: number}} [options]
|
|
93
|
+
* @returns {JudgeListPick}
|
|
94
|
+
*/
|
|
95
|
+
export function selectListJudge(contract, list, workerId, options = {}) {
|
|
96
|
+
const attempted = new Set(options.attempted ?? []);
|
|
97
|
+
const now = options.now ?? Date.now();
|
|
98
|
+
const workerProviders = reachableWorkerProviders(contract, workerId);
|
|
99
|
+
/** @type {JudgeSkip[]} */
|
|
100
|
+
const skipped = [];
|
|
101
|
+
for (const id of list) {
|
|
102
|
+
if (attempted.has(id)) {
|
|
103
|
+
skipped.push({ id, reason: "already attempted this run" });
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
const runtime = contract.runtimes[id];
|
|
107
|
+
if (!runtime) {
|
|
108
|
+
skipped.push({ id, reason: "not declared in contract.runtimes" });
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
const provider = effectiveProvider(runtime);
|
|
112
|
+
if (workerProviders.has(provider)) {
|
|
113
|
+
skipped.push({ id, reason: `same provider as the worker: ${provider}` });
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
const key = availabilityKey({ harness: runtime.harness, model: runtime.model, executable: getHarness(runtime.harness).executable(runtime) });
|
|
117
|
+
const refusal = readRefusal(key, now);
|
|
118
|
+
if (refusal) {
|
|
119
|
+
skipped.push({ id, reason: `recorded refusal: ${refusal.reason}` });
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
const window = overspentWindow(runtime, now);
|
|
123
|
+
if (window) {
|
|
124
|
+
skipped.push({ id, reason: `usage window ${window.window} at ${window.usedPercent}%, above ${JUDGE_USAGE_WINDOW_LIMIT_PERCENT}%` });
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
// An attempted entry after the pick is still recorded: `nextListJudge`
|
|
128
|
+
// rebuilds the attempted set from these entries alone, so dropping one
|
|
129
|
+
// here would let a later hop return to a judge that already refused.
|
|
130
|
+
const after = list.slice(list.indexOf(id) + 1).filter((later) => attempted.has(later));
|
|
131
|
+
return { chosen: id, skipped: [...skipped, ...after.map((later) => ({ id: later, reason: "already attempted this run" }))] };
|
|
132
|
+
}
|
|
133
|
+
return { chosen: null, skipped };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* The judge-list evidence a node's initial assignment records: the list it
|
|
138
|
+
* read, the entry it chose (or null when every entry was skipped), and why
|
|
139
|
+
* each earlier one was.
|
|
140
|
+
*
|
|
141
|
+
* @param {ValidatedContract} contract
|
|
142
|
+
* @param {string[]} list
|
|
143
|
+
* @param {string} workerId
|
|
144
|
+
* @param {number} [now]
|
|
145
|
+
* @returns {JudgeListState}
|
|
146
|
+
*/
|
|
147
|
+
export function initialJudgeListState(contract, list, workerId, now = Date.now()) {
|
|
148
|
+
const pick = selectListJudge(contract, list, workerId, { now });
|
|
149
|
+
return { list, chosen: pick.chosen, skipped: pick.skipped };
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* The next hop out of a list-driven judge that was just refused: the current
|
|
154
|
+
* choice joins the excluded set (it is the one that just failed), so the same
|
|
155
|
+
* eligibility pass never returns to it -- the hop-by-hop rule. Only an entry
|
|
156
|
+
* that actually ran and then failed belongs in that excluded set; a candidate
|
|
157
|
+
* that was skipped for cause (same provider, a recorded refusal, an overspent
|
|
158
|
+
* window) is re-checked fresh instead, so it keeps its real reason rather than
|
|
159
|
+
* being relabelled "already attempted" on every later hop. A prior hop's own
|
|
160
|
+
* "already attempted this run" entries name exactly the ids that were chosen
|
|
161
|
+
* and failed before this one, so folding those back in carries that set
|
|
162
|
+
* forward without a dedicated field.
|
|
163
|
+
*
|
|
164
|
+
* @param {ValidatedContract} contract
|
|
165
|
+
* @param {JudgeListState} judgeListState
|
|
166
|
+
* @param {string} workerId
|
|
167
|
+
* @param {number} [now]
|
|
168
|
+
* @returns {JudgeListState}
|
|
169
|
+
*/
|
|
170
|
+
export function nextListJudge(contract, judgeListState, workerId, now = Date.now()) {
|
|
171
|
+
const previouslyChosen = judgeListState.skipped
|
|
172
|
+
.filter((entry) => entry.reason === "already attempted this run")
|
|
173
|
+
.map((entry) => entry.id);
|
|
174
|
+
const attempted = new Set([...previouslyChosen, ...(judgeListState.chosen ? [judgeListState.chosen] : [])]);
|
|
175
|
+
// A fresh pass re-derives the whole prefix up to (and including) wherever
|
|
176
|
+
// it lands: every id the loop reaches before returning is pushed to
|
|
177
|
+
// `pick.skipped`, on the first hop as on the fifth. That is already a
|
|
178
|
+
// complete accounting of this hop, so carrying the previous hops' entries
|
|
179
|
+
// forward too would only repeat each of them once per later hop.
|
|
180
|
+
const pick = selectListJudge(contract, judgeListState.list, workerId, { attempted, now });
|
|
181
|
+
return { list: judgeListState.list, chosen: pick.chosen, skipped: pick.skipped };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Does a judge's next fallback runtime share the vendor of the worker it
|
|
186
|
+
* would be reviewing? Reachability depends on which worker runtime actually
|
|
187
|
+
* ran, which a static contract cannot know, so this is checked at the moment
|
|
188
|
+
* a fallback is about to be taken rather than at contract validation.
|
|
189
|
+
* A list-driven pick already excludes the worker's reachable providers
|
|
190
|
+
* (`selectListJudge`), so this only ever fires for the single declared
|
|
191
|
+
* `runtime.fallback` edge.
|
|
192
|
+
*
|
|
193
|
+
* @param {ValidatedContract} contract
|
|
194
|
+
* @param {string|null|undefined} workerRuntimeId
|
|
195
|
+
* @param {string} nextRuntimeId
|
|
196
|
+
* @returns {{code: string, message: string}|null}
|
|
197
|
+
*/
|
|
198
|
+
export function judgeFallbackVendorConflict(contract, workerRuntimeId, nextRuntimeId) {
|
|
199
|
+
const workerVendor = workerRuntimeId ? contract.runtimes[workerRuntimeId]?.vendor : undefined;
|
|
200
|
+
const judgeFallbackVendor = contract.runtimes[nextRuntimeId]?.vendor;
|
|
201
|
+
if (!workerVendor || !judgeFallbackVendor || workerVendor !== judgeFallbackVendor) return null;
|
|
202
|
+
return {
|
|
203
|
+
code: "judge_fallback_vendor_conflict",
|
|
204
|
+
message: `judge fallback runtime ${nextRuntimeId} shares vendor ${judgeFallbackVendor} with worker runtime ${workerRuntimeId}`,
|
|
205
|
+
};
|
|
206
|
+
}
|
package/src/engine/lifecycle.mjs
CHANGED
|
@@ -35,6 +35,7 @@ import {
|
|
|
35
35
|
import { exhaustedUntilOf } from "./runtime-discovery.mjs";
|
|
36
36
|
import { getHarness, normalizeProviderAvailability } from "../harnesses/index.mjs";
|
|
37
37
|
import { availabilityKey, recordRefusal, refusalOfAvailability } from "../run/availability.mjs";
|
|
38
|
+
import { judgeFallbackVendorConflict, nextListJudge } from "./judge-list.mjs";
|
|
38
39
|
|
|
39
40
|
import { acquire as acquireLock } from "../run/lock.mjs";
|
|
40
41
|
import { invalidResultRepair, parseDiscoveryResult } from "../contract/worker-result.mjs";
|
|
@@ -641,7 +642,15 @@ export function handleProviderExhaustion(contract, runDir, node, state, role, en
|
|
|
641
642
|
const runtime = contract.runtimes?.[current];
|
|
642
643
|
const refusal = runtime ? refusalOfAvailability(normalizeProviderAvailability(runtime, envelope)) : null;
|
|
643
644
|
if (runtime && refusal) recordRefusal(availabilityKey({ harness: runtime.harness, model: runtime.model, executable: getHarness(runtime.harness).executable(runtime) }), refusal);
|
|
644
|
-
|
|
645
|
+
// R18: a list-driven judge that is about to hop reads the durable refusal
|
|
646
|
+
// and usage-window stores here -- the impure orchestration layer, not
|
|
647
|
+
// `planRoute`'s pure decision -- so `current` joins the excluded set before
|
|
648
|
+
// the next eligible entry is chosen. A reset (wait, no hop) leaves the list
|
|
649
|
+
// untouched: it costs no runtime and must not spend a list entry either.
|
|
650
|
+
const judgeListState = role === "judge" && schedule.kind === "failover" && state.routing?.judgeList
|
|
651
|
+
? nextListJudge(contract, state.routing.judgeList, sourceWorkerRuntime(state) ?? state.routing?.assignments?.worker ?? current, now)
|
|
652
|
+
: undefined;
|
|
653
|
+
const plan = planRoute(contract, node, state, role, error, current, schedule, now, exhaustedUntil, judgeListState ? { judgeListState } : {});
|
|
645
654
|
const status = envelope.status === "failed" ? "failed" : "exhausted";
|
|
646
655
|
if (plan.blocked) {
|
|
647
656
|
// A caller that classified the failure itself also owns what happens when
|
|
@@ -659,17 +668,21 @@ export function handleProviderExhaustion(contract, runDir, node, state, role, en
|
|
|
659
668
|
phase: role,
|
|
660
669
|
result: state.result,
|
|
661
670
|
usage: state.usage,
|
|
662
|
-
routing: { ...(state.routing ?? {}), tierExhaustion },
|
|
671
|
+
routing: { ...(state.routing ?? {}), tierExhaustion, ...(judgeListState ? { judgeList: judgeListState } : {}) },
|
|
663
672
|
error: { ...plan.blocked, ...(exhaustedUntil ? { exhaustedUntil } : {}) },
|
|
664
673
|
}, lock);
|
|
665
674
|
void raiseNodeAttention(campaignPath, runDir, state, plan.blocked.code).catch(() => {});
|
|
666
675
|
} else {
|
|
667
|
-
//
|
|
676
|
+
// `judge_list_exhausted` reaches this branch (it is not
|
|
677
|
+
// `runtime_tier_exhausted`), and its evidence -- naming the judge that
|
|
678
|
+
// was just refused and why every remaining entry was ineligible -- is
|
|
679
|
+
// exactly this feature's evidence to keep; only `tierExhaustion` is not.
|
|
668
680
|
clearTierExhaustion(state);
|
|
669
681
|
transition(runDir, state, "exhausted", {
|
|
670
682
|
phase: role,
|
|
671
683
|
result: state.result,
|
|
672
684
|
usage: state.usage,
|
|
685
|
+
...(judgeListState ? { routing: { ...(state.routing ?? {}), judgeList: judgeListState } } : {}),
|
|
673
686
|
error: { ...plan.blocked, ...(exhaustedUntil ? { exhaustedUntil } : {}) },
|
|
674
687
|
}, lock);
|
|
675
688
|
}
|
|
@@ -681,21 +694,11 @@ export function handleProviderExhaustion(contract, runDir, node, state, role, en
|
|
|
681
694
|
// so it is refused here, and the node is parked for a human rather than
|
|
682
695
|
// quietly arbitrated by the vendor it is supposed to check.
|
|
683
696
|
if (role === "judge" && plan.nextRuntime !== current) {
|
|
684
|
-
const
|
|
685
|
-
|
|
686
|
-
const judgeFallbackVendor = contract.runtimes[plan.nextRuntime]?.vendor;
|
|
687
|
-
if (workerVendor && judgeFallbackVendor && workerVendor === judgeFallbackVendor) {
|
|
697
|
+
const conflict = judgeFallbackVendorConflict(contract, sourceWorkerRuntime(state), plan.nextRuntime);
|
|
698
|
+
if (conflict) {
|
|
688
699
|
if (precomputed) return false;
|
|
689
700
|
clearTierExhaustion(state);
|
|
690
|
-
transition(runDir, state, "blocked", {
|
|
691
|
-
phase: role,
|
|
692
|
-
result: state.result,
|
|
693
|
-
usage: state.usage,
|
|
694
|
-
error: {
|
|
695
|
-
code: "judge_fallback_vendor_conflict",
|
|
696
|
-
message: `judge fallback runtime ${plan.nextRuntime} shares vendor ${judgeFallbackVendor} with worker runtime ${workerRuntimeId}`,
|
|
697
|
-
},
|
|
698
|
-
}, lock);
|
|
701
|
+
transition(runDir, state, "blocked", { phase: role, result: state.result, usage: state.usage, error: conflict }, lock);
|
|
699
702
|
return true;
|
|
700
703
|
}
|
|
701
704
|
}
|
package/src/engine/resume.mjs
CHANGED
|
@@ -44,6 +44,7 @@ import { validateRunMetadata, OPERATOR_ANSWER_MAX_BYTES } from "../contract/snap
|
|
|
44
44
|
import { verificationFailureVerdict } from "./judge-gate.mjs";
|
|
45
45
|
import { verificationFailureWithScope } from "../contract/scope-findings.mjs";
|
|
46
46
|
import { applyRejection, applyVerificationFailure, raiseNodeAttention, settleDone } from "./settle.mjs";
|
|
47
|
+
import { isHumanStepPending } from "../contract/human-step.mjs";
|
|
47
48
|
|
|
48
49
|
/** @typedef {import("../repo/integrate.mjs").IntegrationResult} IntegrationResult */
|
|
49
50
|
/** @typedef {import("./lifecycle.mjs").Invocation} Invocation */
|
|
@@ -139,16 +140,37 @@ export async function resumeRun(runDirPath, options = {}) {
|
|
|
139
140
|
const answerNode = contract.nodes.find((node) => node.id === options.answer?.node);
|
|
140
141
|
if (!answerNode) throw new Error(`unknown node id: ${options.answer.node}`);
|
|
141
142
|
const answerState = states.get(options.answer.node);
|
|
142
|
-
if (!answerState || !isBlockedContextTerminal(answerState)) {
|
|
143
|
-
throw new Error(`node ${options.answer.node} is not blocked on missing context`);
|
|
144
|
-
}
|
|
145
143
|
const answerText = readOperatorAnswer(options.answer.path);
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
144
|
+
if (answerNode.humanStep) {
|
|
145
|
+
if (!answerState || !isHumanStepPending(answerState)) {
|
|
146
|
+
throw new Error(`node ${options.answer.node} is not a pending human step`);
|
|
147
|
+
}
|
|
148
|
+
recordExecutionOverride(runDir, answerState, /** @type {import("../contract/index.mjs").ExecutionOverride} */ (/** @type {unknown} */ ({
|
|
149
|
+
kind: "operator-answer",
|
|
150
|
+
reason: `operator confirmed the human step for node ${options.answer.node}`,
|
|
151
|
+
text: answerText,
|
|
152
|
+
})), lock);
|
|
153
|
+
// A human step never dispatches to a provider and leaves no attempt
|
|
154
|
+
// worktree for `settleDone` to seal and integrate -- the operator's
|
|
155
|
+
// confirmation is the whole of its work, so it settles done directly,
|
|
156
|
+
// here, before `planResumeRetry` (which knows nothing of `humanStep`)
|
|
157
|
+
// ever classifies it.
|
|
158
|
+
transition(runDir, answerState, "done", {
|
|
159
|
+
phase: "complete",
|
|
160
|
+
error: null,
|
|
161
|
+
result: { status: "done", summary: boundedSummary(answerText.trim() || answerNode.humanStep.step), verification: [], artifacts: [], missingContext: [] },
|
|
162
|
+
}, lock);
|
|
163
|
+
} else {
|
|
164
|
+
if (!answerState || !isBlockedContextTerminal(answerState)) {
|
|
165
|
+
throw new Error(`node ${options.answer.node} is not blocked on missing context`);
|
|
166
|
+
}
|
|
167
|
+
const answerOverride = /** @type {import("../contract/index.mjs").ExecutionOverride} */ (/** @type {unknown} */ ({
|
|
168
|
+
kind: "operator-answer",
|
|
169
|
+
reason: `operator answered missing context for node ${options.answer.node}`,
|
|
170
|
+
text: answerText,
|
|
171
|
+
}));
|
|
172
|
+
recordExecutionOverride(runDir, answerState, answerOverride, lock);
|
|
173
|
+
}
|
|
152
174
|
}
|
|
153
175
|
const plan = planResumeRetry(contract, states, { node: options.node, reconcile: options.reconcile, answer: options.answer?.node });
|
|
154
176
|
for (const item of plan.attention) {
|
|
@@ -164,6 +186,7 @@ export async function resumeRun(runDirPath, options = {}) {
|
|
|
164
186
|
if (!state) continue;
|
|
165
187
|
if (state.status === "done") continue;
|
|
166
188
|
if (isBlockedContextTerminal(state) && node.id !== options.answer?.node) continue;
|
|
189
|
+
if (isHumanStepPending(state) && node.id !== options.answer?.node) continue;
|
|
167
190
|
const action = plan.actions.get(node.id) ?? "recover";
|
|
168
191
|
// Adoption before retry: an unresolved blocking review is re-judged from
|
|
169
192
|
// the preserved worker result, never reset to a fresh worker attempt.
|
|
@@ -523,6 +546,20 @@ export async function resumeRun(runDirPath, options = {}) {
|
|
|
523
546
|
export function isBlockedContextTerminal(state) {
|
|
524
547
|
return state.status === "blocked" && state.error?.code === "context_missing";
|
|
525
548
|
}
|
|
549
|
+
/**
|
|
550
|
+
* The operator-answer result summary for a human-step node's `done`
|
|
551
|
+
* transition, bounded to the worker-result protocol's own summary ceiling
|
|
552
|
+
* (`contract/worker-result.mjs`'s `RESULT_LIMITS.summaryBytes`) since this
|
|
553
|
+
* writes the same `result.summary` field a provider's own answer would.
|
|
554
|
+
*
|
|
555
|
+
* @param {string} text
|
|
556
|
+
* @returns {string}
|
|
557
|
+
*/
|
|
558
|
+
function boundedSummary(text) {
|
|
559
|
+
const maxBytes = 4 * 1024;
|
|
560
|
+
if (Buffer.byteLength(text, "utf8") <= maxBytes) return text;
|
|
561
|
+
return `${Buffer.from(text, "utf8").subarray(0, maxBytes - 1).toString("utf8")}…`;
|
|
562
|
+
}
|
|
526
563
|
/**
|
|
527
564
|
* The routing write that opens the next tier-exhaustion generation, or null
|
|
528
565
|
* when this re-dispatch is not one: the generation counter bumps by exactly
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { normalizeProviderAvailability, probeRuntime } from "../harnesses/index.mjs";
|
|
2
|
+
import { effectiveProvider } from "../contract/provider.mjs";
|
|
3
|
+
import { SAME_VENDOR_REVIEW_MODE, sameVendorTierRefusal } from "../contract/judge-independence.mjs";
|
|
2
4
|
|
|
3
5
|
// Availability normalization belongs to the adapter registry, which is where
|
|
4
6
|
// each provider's own exhaustion, balance, and authentication wording is
|
|
@@ -8,7 +10,7 @@ export { exhaustedUntilOf, normalizeProviderAvailability } from "../harnesses/in
|
|
|
8
10
|
|
|
9
11
|
/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
|
|
10
12
|
/** @typedef {{harness?: string, model?: string, vendor: string, tier?: number|string, costRank?: number, [key: string]: unknown}} RuntimeLike */
|
|
11
|
-
/** @typedef {{runtimes: Record<string, RuntimeLike>, runtimeDefaults?: {worker?: string, judge?: string}, nodes?: {id: string, runtime?: string, gate: {enabled: boolean, runtime?: string}}[]}} RuntimeContract */
|
|
13
|
+
/** @typedef {{runtimes: Record<string, RuntimeLike>, runtimeDefaults?: {worker?: string, judge?: string}, judgeIndependence?: string, nodes?: {id: string, runtime?: string, gate: {enabled: boolean, runtime?: string}}[]}} RuntimeContract */
|
|
12
14
|
/**
|
|
13
15
|
* One runtime's catalogue record: what the harness de facto reported, and
|
|
14
16
|
* when. An unobservable datum is null -- never zero and never full allowance,
|
|
@@ -20,7 +22,20 @@ export { exhaustedUntilOf, normalizeProviderAvailability } from "../harnesses/in
|
|
|
20
22
|
/** @typedef {{harness: string, model: string, vendor: string, tier: number, costRank: number, config?: Record<string, unknown>}} DiscoveryRuntime */
|
|
21
23
|
/** @typedef {{id: string, runtime: RuntimeLike, order: number}} RuntimeCandidate */
|
|
22
24
|
/** @typedef {import("../host/config.mjs").UserConfig} UserConfig */
|
|
23
|
-
/** @typedef {{
|
|
25
|
+
/** @typedef {{id: string, gate: {enabled: boolean, runtime?: string}}} ComposeOptionsNode */
|
|
26
|
+
/**
|
|
27
|
+
* `listJudge`'s return governs precedence, not just a value: `undefined`
|
|
28
|
+
* means no list applies to this node (its gate is disabled), so
|
|
29
|
+
* `composeAssignments` falls through to its own candidates unchanged; once a
|
|
30
|
+
* list applies, its pick is final and nothing else is tried, whether that
|
|
31
|
+
* pick is a runtime id (`chosen`) or `null` for an exhausted list. The full
|
|
32
|
+
* pick, not just `chosen`, is returned so an exhausted list's thrown error can
|
|
33
|
+
* name every skipped entry and why -- an operator reading an aborted run
|
|
34
|
+
* cannot otherwise tell the list was even consulted.
|
|
35
|
+
* @typedef {{id: string, reason: string}} JudgeListSkip
|
|
36
|
+
* @typedef {{chosen: string|null, skipped: JudgeListSkip[]}} JudgeListPick
|
|
37
|
+
* @typedef {{config?: UserConfig|null, onWarning?: (message: string) => void, listJudge?: (node: ComposeOptionsNode, workerId: string, workerProvider: string|undefined) => JudgeListPick|undefined}} ComposeOptions
|
|
38
|
+
*/
|
|
24
39
|
|
|
25
40
|
/**
|
|
26
41
|
* Candidates used when a contract omits its runtime catalogue. The catalogue
|
|
@@ -124,6 +139,16 @@ export function availableCandidates(runtimes, availability = {}) {
|
|
|
124
139
|
* `options.onWarning`; config never overrides an explicit node, gate or
|
|
125
140
|
* runtime-default declaration, and the cross-vendor judge rule still applies.
|
|
126
141
|
*
|
|
142
|
+
* `options.listJudge` (R18) is asked for an omitted judge before the single
|
|
143
|
+
* `config.judge` preference and the strongest-candidate default: it is the
|
|
144
|
+
* caller's own ordered-list selection (`engine/judge-list.mjs`), kept out of
|
|
145
|
+
* this module so a list pick's refusal- and usage-window reads never import a
|
|
146
|
+
* `run/` consumer of this very function back into a cycle. Returning
|
|
147
|
+
* `undefined` -- no list governs this node -- falls through to the candidates
|
|
148
|
+
* below exactly as if it had not been asked. A governed node's pick is final
|
|
149
|
+
* instead: a string is used as-is, and `null` -- every entry was skipped --
|
|
150
|
+
* throws rather than silently falling through to a judge outside the list.
|
|
151
|
+
*
|
|
127
152
|
* @param {RuntimeContract} contract
|
|
128
153
|
* @param {Record<string, RuntimeAvailability>} availability
|
|
129
154
|
* @param {ComposeOptions} [options]
|
|
@@ -149,21 +174,57 @@ export function composeAssignments(contract, availability = {}, options = {}) {
|
|
|
149
174
|
const workerRuntime = contract.runtimes[worker];
|
|
150
175
|
const judgeOmitted = node.gate.runtime === undefined && contract.runtimeDefaults?.judge === undefined;
|
|
151
176
|
const preferredJudge = judgeOmitted ? candidateById(candidates, config?.judge) : undefined;
|
|
152
|
-
const
|
|
153
|
-
|
|
154
|
-
|
|
177
|
+
const workerProvider = effectiveProvider(workerRuntime);
|
|
178
|
+
const declaredJudge = node.gate.runtime ?? contract.runtimeDefaults?.judge;
|
|
179
|
+
// R18: once a list governs this node (`judgeListPick` is not
|
|
180
|
+
// `undefined`), its pick is final -- a string is used as-is, and `null`
|
|
181
|
+
// (every entry skipped) must not fall through to
|
|
182
|
+
// `preferredJudge`/`strongest` below, or the node would land on a judge
|
|
183
|
+
// outside the declared list and the skip evidence `listJudge` already
|
|
184
|
+
// recorded would go unused. The thrown error itself names every skipped
|
|
185
|
+
// entry and why: this is the only trace of the list an operator reading
|
|
186
|
+
// an aborted run has, since the caller's own record of `judgeListStates`
|
|
187
|
+
// is never reached when this constructor throws before returning.
|
|
188
|
+
const judgeListPick = declaredJudge === undefined && judgeOmitted ? options.listJudge?.(node, worker, workerProvider) : undefined;
|
|
189
|
+
if (judgeListPick && judgeListPick.chosen === null) {
|
|
190
|
+
const reasons = judgeListPick.skipped.map((entry) => `${entry.id}: ${entry.reason}`).join("; ");
|
|
191
|
+
throw new Error(`runtime_assignment_judge_unavailable: every entry of the judge list is unavailable for node ${node.id} and worker ${worker} (${reasons})`);
|
|
192
|
+
}
|
|
193
|
+
// R20: in same-vendor mode a judge of the worker's own provider is
|
|
194
|
+
// admissible when it is another model at an equal or higher declared tier;
|
|
195
|
+
// the strongest such candidate is the default only when no other vendor has one.
|
|
196
|
+
const sameVendorMode = contract.judgeIndependence === SAME_VENDOR_REVIEW_MODE;
|
|
197
|
+
const judge = declaredJudge
|
|
198
|
+
?? judgeListPick?.chosen
|
|
199
|
+
?? (preferredJudge && effectiveProvider(preferredJudge.runtime) !== workerProvider ? preferredJudge.id : undefined)
|
|
200
|
+
?? strongest(candidates, workerProvider)?.id
|
|
201
|
+
?? (sameVendorMode ? strongest(candidates.filter(({ id, runtime }) => id !== worker && isTierAdmissible(workerRuntime, runtime)), undefined)?.id : undefined);
|
|
155
202
|
if (node.gate.enabled && (!judge || !contract.runtimes[judge])) {
|
|
156
203
|
throw new Error(`runtime_assignment_judge_unavailable: no available cross-vendor judge for node ${node.id} and worker ${worker}`);
|
|
157
204
|
}
|
|
158
205
|
const judgeRuntime = judge ? contract.runtimes[judge] : undefined;
|
|
159
|
-
if (node.gate.enabled && workerRuntime && judgeRuntime && judgeRuntime
|
|
160
|
-
throw new Error(`runtime_assignment_judge_unavailable: no available cross-vendor judge for node ${node.id} and worker ${worker}`);
|
|
206
|
+
if (node.gate.enabled && workerRuntime && judgeRuntime && effectiveProvider(judgeRuntime) === workerProvider) {
|
|
207
|
+
if (!sameVendorMode) throw new Error(`runtime_assignment_judge_unavailable: no available cross-vendor judge for node ${node.id} and worker ${worker}`);
|
|
208
|
+
if (judge === worker) throw new Error(`runtime_assignment_judge_unavailable: same-vendor mode needs a judge other than worker ${worker} for node ${node.id}`);
|
|
209
|
+
const refusal = sameVendorTierRefusal(/** @type {{harness: string, model: string}} */ (workerRuntime), /** @type {{harness: string, model: string}} */ (judgeRuntime));
|
|
210
|
+
if (refusal) throw new Error(`runtime_assignment_judge_unavailable: ${refusal.message} for node ${node.id}`);
|
|
161
211
|
}
|
|
162
212
|
assignments[node.id] = { worker, judge: judge ?? worker };
|
|
163
213
|
}
|
|
164
214
|
return assignments;
|
|
165
215
|
}
|
|
166
216
|
|
|
217
|
+
/**
|
|
218
|
+
* @param {RuntimeLike} workerRuntime
|
|
219
|
+
* @param {RuntimeLike} judgeRuntime
|
|
220
|
+
* @returns {boolean}
|
|
221
|
+
*/
|
|
222
|
+
function isTierAdmissible(workerRuntime, judgeRuntime) {
|
|
223
|
+
if (typeof workerRuntime.harness !== "string" || typeof workerRuntime.model !== "string") return false;
|
|
224
|
+
if (typeof judgeRuntime.harness !== "string" || typeof judgeRuntime.model !== "string") return false;
|
|
225
|
+
return sameVendorTierRefusal({ harness: workerRuntime.harness, model: workerRuntime.model }, { harness: judgeRuntime.harness, model: judgeRuntime.model }) === null;
|
|
226
|
+
}
|
|
227
|
+
|
|
167
228
|
/**
|
|
168
229
|
* @param {RuntimeCandidate[]} candidates
|
|
169
230
|
* @param {string|undefined} id
|
|
@@ -189,12 +250,12 @@ export function nextSameTierRuntime(contract, stateRouting, role, current, attem
|
|
|
189
250
|
const currentRuntime = contract.runtimes[current];
|
|
190
251
|
if (!currentRuntime) return null;
|
|
191
252
|
const workerId = stateRouting.assignments?.worker;
|
|
192
|
-
const
|
|
253
|
+
const workerProvider = workerId && contract.runtimes[workerId] ? effectiveProvider(contract.runtimes[workerId]) : null;
|
|
193
254
|
const used = new Set(attempted);
|
|
194
255
|
return Object.entries(contract.runtimes)
|
|
195
256
|
.filter(([id, runtime]) => id !== current && !used.has(id) && sameTier(runtime, currentRuntime))
|
|
196
257
|
.filter(([id]) => isRuntimeAvailable(stateRouting.availability?.[id]))
|
|
197
|
-
.filter(([, runtime]) => role !== "judge" || runtime
|
|
258
|
+
.filter(([, runtime]) => role !== "judge" || effectiveProvider(runtime) !== workerProvider)
|
|
198
259
|
.sort((left, right) => runtimeOrder(left[1]) - runtimeOrder(right[1]))
|
|
199
260
|
.map(([id]) => id)
|
|
200
261
|
.at(0) ?? null;
|
|
@@ -232,11 +293,11 @@ export function cheapest(candidates) {
|
|
|
232
293
|
* `cheapest` for `setup`'s cross-vendor judge default.
|
|
233
294
|
*
|
|
234
295
|
* @param {RuntimeCandidate[]} candidates
|
|
235
|
-
* @param {string} vendor
|
|
296
|
+
* @param {string|undefined} vendor
|
|
236
297
|
* @returns {RuntimeCandidate|null}
|
|
237
298
|
*/
|
|
238
299
|
export function strongest(candidates, vendor) {
|
|
239
|
-
return [...candidates].filter(({ runtime }) => runtime
|
|
300
|
+
return [...candidates].filter(({ runtime }) => effectiveProvider(runtime) !== vendor)
|
|
240
301
|
.sort((left, right) => tierOrder(right.runtime) - tierOrder(left.runtime)
|
|
241
302
|
|| runtimeOrder(right.runtime) - runtimeOrder(left.runtime)
|
|
242
303
|
|| left.order - right.order).at(0) ?? null;
|