faberun 0.25.1 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/faberun/references/contract.md +30 -26
- package/src/campaign/campaign-brief-graph.mjs +218 -0
- package/src/campaign/campaign-brief.mjs +15 -167
- package/src/campaign/metrics-command.mjs +3 -1
- package/src/cli/brand.mjs +1 -0
- package/src/cli/launch.mjs +14 -1
- package/src/cli/plan.mjs +75 -3
- package/src/cli/setup.mjs +22 -5
- package/src/cli.mjs +35 -1
- package/src/contract/definition-of-done.mjs +48 -8
- package/src/contract/human-step.mjs +54 -0
- package/src/contract/index.mjs +25 -38
- package/src/contract/judge-independence.mjs +170 -0
- package/src/contract/judges.mjs +27 -0
- package/src/contract/provider.mjs +103 -0
- package/src/contract/runtime.mjs +19 -2
- package/src/contract/snapshot.mjs +31 -1
- package/src/engine/assignment.mjs +41 -5
- package/src/engine/backoff.mjs +30 -9
- package/src/engine/dispatch.mjs +19 -2
- package/src/engine/failover.mjs +10 -2
- package/src/engine/judge-gate.mjs +24 -4
- package/src/engine/judge-list.mjs +206 -0
- package/src/engine/lifecycle.mjs +19 -16
- package/src/engine/live-preflight.mjs +45 -7
- package/src/engine/resume.mjs +46 -9
- package/src/engine/run-identity.mjs +9 -6
- package/src/engine/runtime-discovery.mjs +72 -11
- package/src/engine/scheduler.mjs +27 -6
- package/src/harnesses/catalogue.mjs +9 -2
- package/src/harnesses/index.mjs +2 -2
- package/src/harnesses/model-tiers.mjs +39 -0
- package/src/harnesses/protocol.mjs +4 -1
- package/src/host/config.mjs +7 -2
- package/src/plan/contest.mjs +78 -0
- package/src/plan/freeze.mjs +118 -7
- package/src/plan/human-step.mjs +38 -0
- package/src/plan/pipeline.mjs +224 -313
- package/src/plan/preflight.mjs +5 -3
- package/src/plan/proof-check.mjs +161 -0
- package/src/plan/proof-run.mjs +6 -2
- package/src/plan/proof-scope.mjs +175 -0
- package/src/plan/resolve.mjs +182 -0
- package/src/plan/reviewer.mjs +82 -0
- package/src/plan/rounds.mjs +446 -0
- package/src/plan/routing.mjs +11 -4
- package/src/plan/template.mjs +36 -14
- package/src/repo/signal-block.mjs +10 -3
- package/src/repo/source-identity.mjs +45 -5
- package/src/repo/worktree-prune.mjs +72 -0
- package/src/report/campaign-brief-estimate.mjs +1 -1
- package/src/report/campaign-brief.mjs +4 -0
- package/src/report/metrics-report.mjs +34 -1
- package/src/report/render.mjs +31 -10
package/src/engine/failover.mjs
CHANGED
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
import { harnessCapabilities } from "../harnesses/index.mjs";
|
|
18
18
|
import { isRuntimeAvailable, nextSameTierRuntime } from "./runtime-discovery.mjs";
|
|
19
19
|
import { routeRuntime } from "../contract/runtime.mjs";
|
|
20
|
+
import { effectiveProvider } from "../contract/provider.mjs";
|
|
20
21
|
|
|
21
22
|
/** @typedef {import("../contract/index.mjs").ValidatedNode} ValidatedNode */
|
|
22
23
|
/** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
|
|
@@ -188,11 +189,17 @@ export function routingBackoffActive(state, phase) {
|
|
|
188
189
|
* survives both a routed hop and the revision boundary: `resetPhaseRouting`
|
|
189
190
|
* clears the routing override between revisions, but never the invocations.
|
|
190
191
|
*
|
|
192
|
+
* Exported so a same-provider-review mark (R20) can be taken from the worker
|
|
193
|
+
* invocation that actually ran instead of `state.runtime`, which `startJudge`
|
|
194
|
+
* overwrites with the judge's own runtime as soon as it dispatches -- a re-ask
|
|
195
|
+
* or a judge-failure retry then calls `startJudge` again with the judge still
|
|
196
|
+
* in `state.runtime`, so comparing against it compares the judge to itself.
|
|
197
|
+
*
|
|
191
198
|
* @param {NodeSnapshot} state
|
|
192
199
|
* @param {"worker"|"judge"} role
|
|
193
200
|
* @returns {string|undefined}
|
|
194
201
|
*/
|
|
195
|
-
function previousAttemptRuntimeId(state, role) {
|
|
202
|
+
export function previousAttemptRuntimeId(state, role) {
|
|
196
203
|
const found = [...(state.invocations ?? [])].reverse().find((invocation) => invocation.phase === role)?.runtimeId;
|
|
197
204
|
return typeof found === "string" ? found : undefined;
|
|
198
205
|
}
|
|
@@ -223,7 +230,8 @@ function admitsAffinity(contract, state, role, candidate) {
|
|
|
223
230
|
if (availability && !isRuntimeAvailable(availability)) return false;
|
|
224
231
|
if (role !== "judge") return true;
|
|
225
232
|
const workerId = previousAttemptRuntimeId(state, "worker") ?? state.routing?.assignments?.worker;
|
|
226
|
-
|
|
233
|
+
const workerRuntime = workerId ? contract.runtimes[workerId] : undefined;
|
|
234
|
+
return effectiveProvider(runtime) !== (workerRuntime ? effectiveProvider(workerRuntime) : null);
|
|
227
235
|
}
|
|
228
236
|
|
|
229
237
|
/**
|
|
@@ -142,8 +142,9 @@ function proveVerification(id, proof, recorded) {
|
|
|
142
142
|
* `--test-skip-pattern`) is judged on more than its exit code, because the
|
|
143
143
|
* runner exits 0 whether its filter selected anything or not. The proof runs
|
|
144
144
|
* with the TAP reporter selected through `NODE_OPTIONS` and is refused when the
|
|
145
|
-
* output
|
|
146
|
-
*
|
|
145
|
+
* output says the filter selected nothing (`tapSelectedNothing`). Measured
|
|
146
|
+
* 2026-09-21 on node v26.8.1, in a single-file run: a filter that matches
|
|
147
|
+
* emits `1..0` zero times, a filter that matches nothing
|
|
147
148
|
* emits it exactly once, a run with no filter emits it zero times, and an empty
|
|
148
149
|
* suite under a matching filter emits no nested zero plan. The default reporter
|
|
149
150
|
* cannot make the distinction -- both cases print identical counters, because
|
|
@@ -154,7 +155,26 @@ function proveVerification(id, proof, recorded) {
|
|
|
154
155
|
* never sees it.
|
|
155
156
|
*/
|
|
156
157
|
const TEST_FILTER_FLAGS = ["--test-name-pattern", "--test-skip-pattern"];
|
|
157
|
-
const TAP_ZERO_PLAN = /^1\.\.0$/
|
|
158
|
+
const TAP_ZERO_PLAN = /^1\.\.0$/gmu;
|
|
159
|
+
const TAP_TOP_LEVEL_RESULT = /^(?:not )?ok \d+ /gmu;
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Whether a filtered run's TAP says its filter selected no test. A run of
|
|
163
|
+
* several files prints, for each file the filter selected nothing in, an
|
|
164
|
+
* unindented `1..0` followed by that file's own top-level `ok` line, while a
|
|
165
|
+
* selected test prints its own top-level result with no zero plan in front of
|
|
166
|
+
* it. Measured 2026-09-26 on node v26.8.1 (AP1 of safe-to-hand-to-a-friend):
|
|
167
|
+
* reading a single `1..0` as "nothing selected" refused two proofs whose
|
|
168
|
+
* filter had selected a passing test in one of several files. Nothing was
|
|
169
|
+
* selected exactly when every top-level result is one of those file lines.
|
|
170
|
+
*
|
|
171
|
+
* @param {string} stdout
|
|
172
|
+
* @returns {boolean}
|
|
173
|
+
*/
|
|
174
|
+
function tapSelectedNothing(stdout) {
|
|
175
|
+
const zeroPlans = stdout.match(TAP_ZERO_PLAN)?.length ?? 0;
|
|
176
|
+
return zeroPlans > 0 && zeroPlans >= (stdout.match(TAP_TOP_LEVEL_RESULT)?.length ?? 0);
|
|
177
|
+
}
|
|
158
178
|
|
|
159
179
|
/**
|
|
160
180
|
* The environment a filtered proof runs under: the ambient environment with
|
|
@@ -252,7 +272,7 @@ async function proveCommand(id, proof, cwd, timeoutMs) {
|
|
|
252
272
|
child.on("close", (code, signal) => {
|
|
253
273
|
// Its own detail, not an ordinary command failure: the command succeeded,
|
|
254
274
|
// so what failed is that the declared filter selected nothing to prove.
|
|
255
|
-
if (code === 0 && signal === null && filters.length > 0 &&
|
|
275
|
+
if (code === 0 && signal === null && filters.length > 0 && tapSelectedNothing(stdout)) {
|
|
256
276
|
finish({
|
|
257
277
|
id,
|
|
258
278
|
kind: "command",
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* R18: the judge of a gated node is the first eligible entry of a static,
|
|
3
|
+
* ordered list rather than the single strongest cross-vendor candidate
|
|
4
|
+
* `composeAssignments` otherwise ranks. Split out of `runtime-discovery.mjs`
|
|
5
|
+
* and `lifecycle.mjs` -- both already near this repository's 800-line
|
|
6
|
+
* ceiling -- and kept out of `backoff.mjs`, whose `planRoute` is a pure
|
|
7
|
+
* decision this module cannot be: picking an entry reads the durable refusal
|
|
8
|
+
* and usage-window stores.
|
|
9
|
+
*/
|
|
10
|
+
import { effectiveProvider } from "../contract/provider.mjs";
|
|
11
|
+
import { availabilityKey, readRefusal } from "../run/availability.mjs";
|
|
12
|
+
import { readUsageWindows, usageAccountOf } from "../run/usage-windows.mjs";
|
|
13
|
+
import { getHarness } from "../harnesses/index.mjs";
|
|
14
|
+
|
|
15
|
+
/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
|
|
16
|
+
/** @typedef {{id: string, reason: string}} JudgeSkip */
|
|
17
|
+
/** @typedef {{chosen: string|null, skipped: JudgeSkip[]}} JudgeListPick */
|
|
18
|
+
/** @typedef {{list: string[], chosen: string|null, skipped: JudgeSkip[]}} JudgeListState */
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* A usage window at or above this spend is treated as unavailable for a judge
|
|
22
|
+
* pick, so a node's review does not race the account into a refusal the next
|
|
23
|
+
* call would hit anyway.
|
|
24
|
+
*/
|
|
25
|
+
export const JUDGE_USAGE_WINDOW_LIMIT_PERCENT = 90;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* The static judge list a node's assignment reads: the contract's own list
|
|
29
|
+
* wins over the machine default, exactly as `runtimeDefaults.worker` already
|
|
30
|
+
* wins over `config.worker` in `composeAssignments`.
|
|
31
|
+
*
|
|
32
|
+
* @param {{judges?: string[]}} contract
|
|
33
|
+
* @param {{judges?: string[]}|null|undefined} config
|
|
34
|
+
* @returns {string[]|undefined}
|
|
35
|
+
*/
|
|
36
|
+
export function resolveJudgeList(contract, config) {
|
|
37
|
+
return contract.judges ?? config?.judges ?? undefined;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Every provider a worker attempt on this node could actually run on: its own
|
|
42
|
+
* canonical provider, plus every hop of its declared fallback chain. A judge
|
|
43
|
+
* of any of these providers cannot independently review whichever one
|
|
44
|
+
* actually ran. Walked directly off `runtime.fallback` rather than through
|
|
45
|
+
* `failover.mjs`'s `synthesizedChain`, so this module does not import a
|
|
46
|
+
* runtime-discovery consumer and risk a cycle back through
|
|
47
|
+
* `run/availability.mjs`.
|
|
48
|
+
*
|
|
49
|
+
* @param {ValidatedContract} contract
|
|
50
|
+
* @param {string} workerId
|
|
51
|
+
* @returns {Set<string|undefined>}
|
|
52
|
+
*/
|
|
53
|
+
function reachableWorkerProviders(contract, workerId) {
|
|
54
|
+
const providers = new Set([effectiveProvider(contract.runtimes[workerId])]);
|
|
55
|
+
const seen = new Set([workerId]);
|
|
56
|
+
let current = workerId;
|
|
57
|
+
for (;;) {
|
|
58
|
+
const next = contract.runtimes[current]?.fallback;
|
|
59
|
+
if (!next || seen.has(next) || !contract.runtimes[next]) break;
|
|
60
|
+
seen.add(next);
|
|
61
|
+
providers.add(effectiveProvider(contract.runtimes[next]));
|
|
62
|
+
current = next;
|
|
63
|
+
}
|
|
64
|
+
return providers;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Is this runtime's account inside `JUDGE_USAGE_WINDOW_LIMIT_PERCENT` of a
|
|
69
|
+
* usage window closing? `null` means no window is known -- unknown must not
|
|
70
|
+
* read as over budget, exactly as an absent availability record must not read
|
|
71
|
+
* as exhausted (see `admitsAffinity` in `failover.mjs`).
|
|
72
|
+
*
|
|
73
|
+
* @param {import("../contract/index.mjs").ValidatedRuntime} runtime
|
|
74
|
+
* @param {number} now
|
|
75
|
+
* @returns {import("../harnesses/codex/usage-window.mjs").UsageWindow|null}
|
|
76
|
+
*/
|
|
77
|
+
function overspentWindow(runtime, now) {
|
|
78
|
+
const account = usageAccountOf(runtime);
|
|
79
|
+
if (!account) return null;
|
|
80
|
+
return readUsageWindows(account, now).find((window) => window.usedPercent > JUDGE_USAGE_WINDOW_LIMIT_PERCENT) ?? null;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* The first eligible entry of an ordered judge list, and why every earlier
|
|
85
|
+
* entry was skipped: already attempted this run, a provider the worker (or
|
|
86
|
+
* its fallback chain) already runs, a recorded refusal, or an account usage
|
|
87
|
+
* window over `JUDGE_USAGE_WINDOW_LIMIT_PERCENT`.
|
|
88
|
+
*
|
|
89
|
+
* @param {ValidatedContract} contract
|
|
90
|
+
* @param {string[]} list
|
|
91
|
+
* @param {string} workerId
|
|
92
|
+
* @param {{attempted?: Iterable<string>, now?: number}} [options]
|
|
93
|
+
* @returns {JudgeListPick}
|
|
94
|
+
*/
|
|
95
|
+
export function selectListJudge(contract, list, workerId, options = {}) {
|
|
96
|
+
const attempted = new Set(options.attempted ?? []);
|
|
97
|
+
const now = options.now ?? Date.now();
|
|
98
|
+
const workerProviders = reachableWorkerProviders(contract, workerId);
|
|
99
|
+
/** @type {JudgeSkip[]} */
|
|
100
|
+
const skipped = [];
|
|
101
|
+
for (const id of list) {
|
|
102
|
+
if (attempted.has(id)) {
|
|
103
|
+
skipped.push({ id, reason: "already attempted this run" });
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
const runtime = contract.runtimes[id];
|
|
107
|
+
if (!runtime) {
|
|
108
|
+
skipped.push({ id, reason: "not declared in contract.runtimes" });
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
const provider = effectiveProvider(runtime);
|
|
112
|
+
if (workerProviders.has(provider)) {
|
|
113
|
+
skipped.push({ id, reason: `same provider as the worker: ${provider}` });
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
const key = availabilityKey({ harness: runtime.harness, model: runtime.model, executable: getHarness(runtime.harness).executable(runtime) });
|
|
117
|
+
const refusal = readRefusal(key, now);
|
|
118
|
+
if (refusal) {
|
|
119
|
+
skipped.push({ id, reason: `recorded refusal: ${refusal.reason}` });
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
const window = overspentWindow(runtime, now);
|
|
123
|
+
if (window) {
|
|
124
|
+
skipped.push({ id, reason: `usage window ${window.window} at ${window.usedPercent}%, above ${JUDGE_USAGE_WINDOW_LIMIT_PERCENT}%` });
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
// An attempted entry after the pick is still recorded: `nextListJudge`
|
|
128
|
+
// rebuilds the attempted set from these entries alone, so dropping one
|
|
129
|
+
// here would let a later hop return to a judge that already refused.
|
|
130
|
+
const after = list.slice(list.indexOf(id) + 1).filter((later) => attempted.has(later));
|
|
131
|
+
return { chosen: id, skipped: [...skipped, ...after.map((later) => ({ id: later, reason: "already attempted this run" }))] };
|
|
132
|
+
}
|
|
133
|
+
return { chosen: null, skipped };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* The judge-list evidence a node's initial assignment records: the list it
|
|
138
|
+
* read, the entry it chose (or null when every entry was skipped), and why
|
|
139
|
+
* each earlier one was.
|
|
140
|
+
*
|
|
141
|
+
* @param {ValidatedContract} contract
|
|
142
|
+
* @param {string[]} list
|
|
143
|
+
* @param {string} workerId
|
|
144
|
+
* @param {number} [now]
|
|
145
|
+
* @returns {JudgeListState}
|
|
146
|
+
*/
|
|
147
|
+
export function initialJudgeListState(contract, list, workerId, now = Date.now()) {
|
|
148
|
+
const pick = selectListJudge(contract, list, workerId, { now });
|
|
149
|
+
return { list, chosen: pick.chosen, skipped: pick.skipped };
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* The next hop out of a list-driven judge that was just refused: the current
|
|
154
|
+
* choice joins the excluded set (it is the one that just failed), so the same
|
|
155
|
+
* eligibility pass never returns to it -- the hop-by-hop rule. Only an entry
|
|
156
|
+
* that actually ran and then failed belongs in that excluded set; a candidate
|
|
157
|
+
* that was skipped for cause (same provider, a recorded refusal, an overspent
|
|
158
|
+
* window) is re-checked fresh instead, so it keeps its real reason rather than
|
|
159
|
+
* being relabelled "already attempted" on every later hop. A prior hop's own
|
|
160
|
+
* "already attempted this run" entries name exactly the ids that were chosen
|
|
161
|
+
* and failed before this one, so folding those back in carries that set
|
|
162
|
+
* forward without a dedicated field.
|
|
163
|
+
*
|
|
164
|
+
* @param {ValidatedContract} contract
|
|
165
|
+
* @param {JudgeListState} judgeListState
|
|
166
|
+
* @param {string} workerId
|
|
167
|
+
* @param {number} [now]
|
|
168
|
+
* @returns {JudgeListState}
|
|
169
|
+
*/
|
|
170
|
+
export function nextListJudge(contract, judgeListState, workerId, now = Date.now()) {
|
|
171
|
+
const previouslyChosen = judgeListState.skipped
|
|
172
|
+
.filter((entry) => entry.reason === "already attempted this run")
|
|
173
|
+
.map((entry) => entry.id);
|
|
174
|
+
const attempted = new Set([...previouslyChosen, ...(judgeListState.chosen ? [judgeListState.chosen] : [])]);
|
|
175
|
+
// A fresh pass re-derives the whole prefix up to (and including) wherever
|
|
176
|
+
// it lands: every id the loop reaches before returning is pushed to
|
|
177
|
+
// `pick.skipped`, on the first hop as on the fifth. That is already a
|
|
178
|
+
// complete accounting of this hop, so carrying the previous hops' entries
|
|
179
|
+
// forward too would only repeat each of them once per later hop.
|
|
180
|
+
const pick = selectListJudge(contract, judgeListState.list, workerId, { attempted, now });
|
|
181
|
+
return { list: judgeListState.list, chosen: pick.chosen, skipped: pick.skipped };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Does a judge's next fallback runtime share the vendor of the worker it
|
|
186
|
+
* would be reviewing? Reachability depends on which worker runtime actually
|
|
187
|
+
* ran, which a static contract cannot know, so this is checked at the moment
|
|
188
|
+
* a fallback is about to be taken rather than at contract validation.
|
|
189
|
+
* A list-driven pick already excludes the worker's reachable providers
|
|
190
|
+
* (`selectListJudge`), so this only ever fires for the single declared
|
|
191
|
+
* `runtime.fallback` edge.
|
|
192
|
+
*
|
|
193
|
+
* @param {ValidatedContract} contract
|
|
194
|
+
* @param {string|null|undefined} workerRuntimeId
|
|
195
|
+
* @param {string} nextRuntimeId
|
|
196
|
+
* @returns {{code: string, message: string}|null}
|
|
197
|
+
*/
|
|
198
|
+
export function judgeFallbackVendorConflict(contract, workerRuntimeId, nextRuntimeId) {
|
|
199
|
+
const workerVendor = workerRuntimeId ? contract.runtimes[workerRuntimeId]?.vendor : undefined;
|
|
200
|
+
const judgeFallbackVendor = contract.runtimes[nextRuntimeId]?.vendor;
|
|
201
|
+
if (!workerVendor || !judgeFallbackVendor || workerVendor !== judgeFallbackVendor) return null;
|
|
202
|
+
return {
|
|
203
|
+
code: "judge_fallback_vendor_conflict",
|
|
204
|
+
message: `judge fallback runtime ${nextRuntimeId} shares vendor ${judgeFallbackVendor} with worker runtime ${workerRuntimeId}`,
|
|
205
|
+
};
|
|
206
|
+
}
|
package/src/engine/lifecycle.mjs
CHANGED
|
@@ -35,6 +35,7 @@ import {
|
|
|
35
35
|
import { exhaustedUntilOf } from "./runtime-discovery.mjs";
|
|
36
36
|
import { getHarness, normalizeProviderAvailability } from "../harnesses/index.mjs";
|
|
37
37
|
import { availabilityKey, recordRefusal, refusalOfAvailability } from "../run/availability.mjs";
|
|
38
|
+
import { judgeFallbackVendorConflict, nextListJudge } from "./judge-list.mjs";
|
|
38
39
|
|
|
39
40
|
import { acquire as acquireLock } from "../run/lock.mjs";
|
|
40
41
|
import { invalidResultRepair, parseDiscoveryResult } from "../contract/worker-result.mjs";
|
|
@@ -641,7 +642,15 @@ export function handleProviderExhaustion(contract, runDir, node, state, role, en
|
|
|
641
642
|
const runtime = contract.runtimes?.[current];
|
|
642
643
|
const refusal = runtime ? refusalOfAvailability(normalizeProviderAvailability(runtime, envelope)) : null;
|
|
643
644
|
if (runtime && refusal) recordRefusal(availabilityKey({ harness: runtime.harness, model: runtime.model, executable: getHarness(runtime.harness).executable(runtime) }), refusal);
|
|
644
|
-
|
|
645
|
+
// R18: a list-driven judge that is about to hop reads the durable refusal
|
|
646
|
+
// and usage-window stores here -- the impure orchestration layer, not
|
|
647
|
+
// `planRoute`'s pure decision -- so `current` joins the excluded set before
|
|
648
|
+
// the next eligible entry is chosen. A reset (wait, no hop) leaves the list
|
|
649
|
+
// untouched: it costs no runtime and must not spend a list entry either.
|
|
650
|
+
const judgeListState = role === "judge" && schedule.kind === "failover" && state.routing?.judgeList
|
|
651
|
+
? nextListJudge(contract, state.routing.judgeList, sourceWorkerRuntime(state) ?? state.routing?.assignments?.worker ?? current, now)
|
|
652
|
+
: undefined;
|
|
653
|
+
const plan = planRoute(contract, node, state, role, error, current, schedule, now, exhaustedUntil, judgeListState ? { judgeListState } : {});
|
|
645
654
|
const status = envelope.status === "failed" ? "failed" : "exhausted";
|
|
646
655
|
if (plan.blocked) {
|
|
647
656
|
// A caller that classified the failure itself also owns what happens when
|
|
@@ -659,17 +668,21 @@ export function handleProviderExhaustion(contract, runDir, node, state, role, en
|
|
|
659
668
|
phase: role,
|
|
660
669
|
result: state.result,
|
|
661
670
|
usage: state.usage,
|
|
662
|
-
routing: { ...(state.routing ?? {}), tierExhaustion },
|
|
671
|
+
routing: { ...(state.routing ?? {}), tierExhaustion, ...(judgeListState ? { judgeList: judgeListState } : {}) },
|
|
663
672
|
error: { ...plan.blocked, ...(exhaustedUntil ? { exhaustedUntil } : {}) },
|
|
664
673
|
}, lock);
|
|
665
674
|
void raiseNodeAttention(campaignPath, runDir, state, plan.blocked.code).catch(() => {});
|
|
666
675
|
} else {
|
|
667
|
-
//
|
|
676
|
+
// `judge_list_exhausted` reaches this branch (it is not
|
|
677
|
+
// `runtime_tier_exhausted`), and its evidence -- naming the judge that
|
|
678
|
+
// was just refused and why every remaining entry was ineligible -- is
|
|
679
|
+
// exactly this feature's evidence to keep; only `tierExhaustion` is not.
|
|
668
680
|
clearTierExhaustion(state);
|
|
669
681
|
transition(runDir, state, "exhausted", {
|
|
670
682
|
phase: role,
|
|
671
683
|
result: state.result,
|
|
672
684
|
usage: state.usage,
|
|
685
|
+
...(judgeListState ? { routing: { ...(state.routing ?? {}), judgeList: judgeListState } } : {}),
|
|
673
686
|
error: { ...plan.blocked, ...(exhaustedUntil ? { exhaustedUntil } : {}) },
|
|
674
687
|
}, lock);
|
|
675
688
|
}
|
|
@@ -681,21 +694,11 @@ export function handleProviderExhaustion(contract, runDir, node, state, role, en
|
|
|
681
694
|
// so it is refused here, and the node is parked for a human rather than
|
|
682
695
|
// quietly arbitrated by the vendor it is supposed to check.
|
|
683
696
|
if (role === "judge" && plan.nextRuntime !== current) {
|
|
684
|
-
const
|
|
685
|
-
|
|
686
|
-
const judgeFallbackVendor = contract.runtimes[plan.nextRuntime]?.vendor;
|
|
687
|
-
if (workerVendor && judgeFallbackVendor && workerVendor === judgeFallbackVendor) {
|
|
697
|
+
const conflict = judgeFallbackVendorConflict(contract, sourceWorkerRuntime(state), plan.nextRuntime);
|
|
698
|
+
if (conflict) {
|
|
688
699
|
if (precomputed) return false;
|
|
689
700
|
clearTierExhaustion(state);
|
|
690
|
-
transition(runDir, state, "blocked", {
|
|
691
|
-
phase: role,
|
|
692
|
-
result: state.result,
|
|
693
|
-
usage: state.usage,
|
|
694
|
-
error: {
|
|
695
|
-
code: "judge_fallback_vendor_conflict",
|
|
696
|
-
message: `judge fallback runtime ${plan.nextRuntime} shares vendor ${judgeFallbackVendor} with worker runtime ${workerRuntimeId}`,
|
|
697
|
-
},
|
|
698
|
-
}, lock);
|
|
701
|
+
transition(runDir, state, "blocked", { phase: role, result: state.result, usage: state.usage, error: conflict }, lock);
|
|
699
702
|
return true;
|
|
700
703
|
}
|
|
701
704
|
}
|
|
@@ -46,7 +46,7 @@ const LIVE_PREFLIGHT_OUTPUT_LIMIT_BYTES = 512 * 1024;
|
|
|
46
46
|
* whole is part of the answer.
|
|
47
47
|
*
|
|
48
48
|
* @param {string} contractPath
|
|
49
|
-
* @param {{static?: boolean, liveTimeoutSec?: number, persisted?: boolean}} [options]
|
|
49
|
+
* @param {{static?: boolean, liveTimeoutSec?: number, minimumTimeoutSec?: number, persisted?: boolean}} [options]
|
|
50
50
|
* @returns {Promise<ProbeResult[]>}
|
|
51
51
|
*/
|
|
52
52
|
export async function preflightContract(contractPath, options = {}) {
|
|
@@ -70,7 +70,7 @@ export async function preflightContract(contractPath, options = {}) {
|
|
|
70
70
|
* once, so a caller cannot accidentally ask a different question.
|
|
71
71
|
*
|
|
72
72
|
* @param {{runtime: RuntimeSnapshot, requiredCapabilitySets?: import("../harnesses/index.mjs").CapabilityRequirements[]}[]} entries
|
|
73
|
-
* @param {{static?: boolean, liveTimeoutSec?: number, cwd?: string}} [options]
|
|
73
|
+
* @param {{static?: boolean, liveTimeoutSec?: number, minimumTimeoutSec?: number, cwd?: string}} [options]
|
|
74
74
|
* @returns {Promise<ProbeResult[]>}
|
|
75
75
|
*/
|
|
76
76
|
export async function preflightRuntimes(entries, options = {}) {
|
|
@@ -80,7 +80,7 @@ export async function preflightRuntimes(entries, options = {}) {
|
|
|
80
80
|
));
|
|
81
81
|
if (options.static === true) return staticChecks;
|
|
82
82
|
|
|
83
|
-
const
|
|
83
|
+
const configuredTimeoutSec = configuredLivePreflightTimeout(options.liveTimeoutSec);
|
|
84
84
|
let liveRepo;
|
|
85
85
|
try {
|
|
86
86
|
liveRepo = createLivePreflightRepo();
|
|
@@ -95,7 +95,7 @@ export async function preflightRuntimes(entries, options = {}) {
|
|
|
95
95
|
}
|
|
96
96
|
try {
|
|
97
97
|
return await Promise.all(staticChecks.map(async (check, index) => {
|
|
98
|
-
const live = await livePreflight(runtimes[index], liveRepo,
|
|
98
|
+
const live = await livePreflight(runtimes[index], liveRepo, configuredTimeoutSec ?? Math.max(options.minimumTimeoutSec ?? 0, defaultLivePreflightTimeout(runtimes[index])));
|
|
99
99
|
const liveDetail = live.status === "done"
|
|
100
100
|
? `live done · usage ${formatUsage(live.usage)} · cost ${formatCost(live.costUsd)}`
|
|
101
101
|
: `live ${live.status} · ${live.error?.code ?? "provider_error"}: ${redactProviderText(live.error?.message ?? "generation failed")} · usage ${formatUsage(live.usage)} · cost ${formatCost(live.costUsd)}`;
|
|
@@ -116,16 +116,54 @@ export async function preflightRuntimes(entries, options = {}) {
|
|
|
116
116
|
rmSync(liveRepo, { recursive: true, force: true });
|
|
117
117
|
}
|
|
118
118
|
}
|
|
119
|
-
/**
|
|
120
|
-
|
|
119
|
+
/**
|
|
120
|
+
* An explicit budget (option, then environment) applies to every runtime.
|
|
121
|
+
*
|
|
122
|
+
* @param {number|undefined} configured
|
|
123
|
+
* @returns {number|null}
|
|
124
|
+
*/
|
|
125
|
+
function configuredLivePreflightTimeout(configured) {
|
|
121
126
|
const raw = configured ?? (process.env.FABERUN_PREFLIGHT_TIMEOUT_SEC === undefined
|
|
122
|
-
?
|
|
127
|
+
? null
|
|
123
128
|
: Number(process.env.FABERUN_PREFLIGHT_TIMEOUT_SEC));
|
|
129
|
+
if (raw === null) return null;
|
|
124
130
|
if (typeof raw !== "number" || !Number.isFinite(raw) || raw <= 0) {
|
|
125
131
|
throw new TypeError("preflight live timeout must be a positive number of seconds");
|
|
126
132
|
}
|
|
127
133
|
return raw;
|
|
128
134
|
}
|
|
135
|
+
/**
|
|
136
|
+
* Without an explicit budget, the wait follows the runtime's reasoning effort:
|
|
137
|
+
* a model at high effort thinks before answering even a one-word prompt.
|
|
138
|
+
* Measured 2026-09-26: Opus 5.5 at `xhigh` failed every `faberun plan`
|
|
139
|
+
* preflight at 15 s and passed once the budget was raised, so the planning
|
|
140
|
+
* pipeline was refusing a runtime that was healthy.
|
|
141
|
+
*
|
|
142
|
+
* @param {RuntimeSnapshot} runtime
|
|
143
|
+
* @returns {number}
|
|
144
|
+
*/
|
|
145
|
+
export function defaultLivePreflightTimeout(runtime) {
|
|
146
|
+
// Keep LIVE_PREFLIGHT_CEILING_SEC equal to the largest value returned here.
|
|
147
|
+
const reasoning = /** @type {{reasoning?: unknown}} */ (runtime).reasoning;
|
|
148
|
+
if (reasoning === "xhigh" || reasoning === "max") return 180;
|
|
149
|
+
if (reasoning === "high") return 60;
|
|
150
|
+
return 15;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* The longest a live preflight can take without an explicit budget: what a
|
|
154
|
+
* detached launcher must be willing to wait before the controller it spawned
|
|
155
|
+
* can report ready, because the dispatch gate asks before it does.
|
|
156
|
+
*/
|
|
157
|
+
export const LIVE_PREFLIGHT_CEILING_SEC = 180;
|
|
158
|
+
/**
|
|
159
|
+
* The longest one live preflight ask may take under the current environment.
|
|
160
|
+
*
|
|
161
|
+
* @returns {number}
|
|
162
|
+
*/
|
|
163
|
+
export function livePreflightCeilingSec() {
|
|
164
|
+
const override = Number(process.env.FABERUN_PREFLIGHT_TIMEOUT_SEC);
|
|
165
|
+
return process.env.FABERUN_PREFLIGHT_TIMEOUT_SEC !== undefined && Number.isFinite(override) && override > 0 ? override : LIVE_PREFLIGHT_CEILING_SEC;
|
|
166
|
+
}
|
|
129
167
|
/** @returns {string} */
|
|
130
168
|
function createLivePreflightRepo() {
|
|
131
169
|
const directory = mkdtempSync(join(tmpdir(), "faberun-preflight-"));
|
package/src/engine/resume.mjs
CHANGED
|
@@ -44,6 +44,7 @@ import { validateRunMetadata, OPERATOR_ANSWER_MAX_BYTES } from "../contract/snap
|
|
|
44
44
|
import { verificationFailureVerdict } from "./judge-gate.mjs";
|
|
45
45
|
import { verificationFailureWithScope } from "../contract/scope-findings.mjs";
|
|
46
46
|
import { applyRejection, applyVerificationFailure, raiseNodeAttention, settleDone } from "./settle.mjs";
|
|
47
|
+
import { isHumanStepPending } from "../contract/human-step.mjs";
|
|
47
48
|
|
|
48
49
|
/** @typedef {import("../repo/integrate.mjs").IntegrationResult} IntegrationResult */
|
|
49
50
|
/** @typedef {import("./lifecycle.mjs").Invocation} Invocation */
|
|
@@ -139,16 +140,37 @@ export async function resumeRun(runDirPath, options = {}) {
|
|
|
139
140
|
const answerNode = contract.nodes.find((node) => node.id === options.answer?.node);
|
|
140
141
|
if (!answerNode) throw new Error(`unknown node id: ${options.answer.node}`);
|
|
141
142
|
const answerState = states.get(options.answer.node);
|
|
142
|
-
if (!answerState || !isBlockedContextTerminal(answerState)) {
|
|
143
|
-
throw new Error(`node ${options.answer.node} is not blocked on missing context`);
|
|
144
|
-
}
|
|
145
143
|
const answerText = readOperatorAnswer(options.answer.path);
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
144
|
+
if (answerNode.humanStep) {
|
|
145
|
+
if (!answerState || !isHumanStepPending(answerState)) {
|
|
146
|
+
throw new Error(`node ${options.answer.node} is not a pending human step`);
|
|
147
|
+
}
|
|
148
|
+
recordExecutionOverride(runDir, answerState, /** @type {import("../contract/index.mjs").ExecutionOverride} */ (/** @type {unknown} */ ({
|
|
149
|
+
kind: "operator-answer",
|
|
150
|
+
reason: `operator confirmed the human step for node ${options.answer.node}`,
|
|
151
|
+
text: answerText,
|
|
152
|
+
})), lock);
|
|
153
|
+
// A human step never dispatches to a provider and leaves no attempt
|
|
154
|
+
// worktree for `settleDone` to seal and integrate -- the operator's
|
|
155
|
+
// confirmation is the whole of its work, so it settles done directly,
|
|
156
|
+
// here, before `planResumeRetry` (which knows nothing of `humanStep`)
|
|
157
|
+
// ever classifies it.
|
|
158
|
+
transition(runDir, answerState, "done", {
|
|
159
|
+
phase: "complete",
|
|
160
|
+
error: null,
|
|
161
|
+
result: { status: "done", summary: boundedSummary(answerText.trim() || answerNode.humanStep.step), verification: [], artifacts: [], missingContext: [] },
|
|
162
|
+
}, lock);
|
|
163
|
+
} else {
|
|
164
|
+
if (!answerState || !isBlockedContextTerminal(answerState)) {
|
|
165
|
+
throw new Error(`node ${options.answer.node} is not blocked on missing context`);
|
|
166
|
+
}
|
|
167
|
+
const answerOverride = /** @type {import("../contract/index.mjs").ExecutionOverride} */ (/** @type {unknown} */ ({
|
|
168
|
+
kind: "operator-answer",
|
|
169
|
+
reason: `operator answered missing context for node ${options.answer.node}`,
|
|
170
|
+
text: answerText,
|
|
171
|
+
}));
|
|
172
|
+
recordExecutionOverride(runDir, answerState, answerOverride, lock);
|
|
173
|
+
}
|
|
152
174
|
}
|
|
153
175
|
const plan = planResumeRetry(contract, states, { node: options.node, reconcile: options.reconcile, answer: options.answer?.node });
|
|
154
176
|
for (const item of plan.attention) {
|
|
@@ -164,6 +186,7 @@ export async function resumeRun(runDirPath, options = {}) {
|
|
|
164
186
|
if (!state) continue;
|
|
165
187
|
if (state.status === "done") continue;
|
|
166
188
|
if (isBlockedContextTerminal(state) && node.id !== options.answer?.node) continue;
|
|
189
|
+
if (isHumanStepPending(state) && node.id !== options.answer?.node) continue;
|
|
167
190
|
const action = plan.actions.get(node.id) ?? "recover";
|
|
168
191
|
// Adoption before retry: an unresolved blocking review is re-judged from
|
|
169
192
|
// the preserved worker result, never reset to a fresh worker attempt.
|
|
@@ -523,6 +546,20 @@ export async function resumeRun(runDirPath, options = {}) {
|
|
|
523
546
|
export function isBlockedContextTerminal(state) {
|
|
524
547
|
return state.status === "blocked" && state.error?.code === "context_missing";
|
|
525
548
|
}
|
|
549
|
+
/**
|
|
550
|
+
* The operator-answer result summary for a human-step node's `done`
|
|
551
|
+
* transition, bounded to the worker-result protocol's own summary ceiling
|
|
552
|
+
* (`contract/worker-result.mjs`'s `RESULT_LIMITS.summaryBytes`) since this
|
|
553
|
+
* writes the same `result.summary` field a provider's own answer would.
|
|
554
|
+
*
|
|
555
|
+
* @param {string} text
|
|
556
|
+
* @returns {string}
|
|
557
|
+
*/
|
|
558
|
+
function boundedSummary(text) {
|
|
559
|
+
const maxBytes = 4 * 1024;
|
|
560
|
+
if (Buffer.byteLength(text, "utf8") <= maxBytes) return text;
|
|
561
|
+
return `${Buffer.from(text, "utf8").subarray(0, maxBytes - 1).toString("utf8")}…`;
|
|
562
|
+
}
|
|
526
563
|
/**
|
|
527
564
|
* The routing write that opens the next tier-exhaustion generation, or null
|
|
528
565
|
* when this re-dispatch is not one: the generation counter bumps by exactly
|
|
@@ -507,11 +507,12 @@ export async function assertEnvironmentReady(contract, runDir, sourceIdentity) {
|
|
|
507
507
|
// worker and no judge, and can spend no availability.
|
|
508
508
|
if (blocking === null && launchMayDispatch(runDir)) {
|
|
509
509
|
// measured 2026-09-22: asking four routed runtimes in parallel took about
|
|
510
|
-
// 18s, so
|
|
511
|
-
//
|
|
512
|
-
//
|
|
510
|
+
// 18s, so no runtime is given less than 60s here; a runtime at high
|
|
511
|
+
// reasoning effort gets its longer default (defaultLivePreflightTimeout).
|
|
512
|
+
// FABERUN_PREFLIGHT_TIMEOUT_SEC stays the operator override for every
|
|
513
|
+
// runtime; preflightContract validates it, so only a valid number is lifted.
|
|
513
514
|
const override = Number(process.env.FABERUN_PREFLIGHT_TIMEOUT_SEC);
|
|
514
|
-
const timeoutSec = process.env.FABERUN_PREFLIGHT_TIMEOUT_SEC !== undefined && Number.isFinite(override) && override > 0 ? override :
|
|
515
|
+
const timeoutSec = process.env.FABERUN_PREFLIGHT_TIMEOUT_SEC !== undefined && Number.isFinite(override) && override > 0 ? override : undefined;
|
|
515
516
|
const probes = await livePreflightProbes(contract, runDir, sourceIdentity, timeoutSec);
|
|
516
517
|
const silent = probes.filter((probe) => liveSilenceCause(probe) !== null);
|
|
517
518
|
writeJsonAtomic(join(runDir, "env-preflight.json"), evidence(silent.length === 0, probes));
|
|
@@ -536,6 +537,8 @@ export async function assertEnvironmentReady(contract, runDir, sourceIdentity) {
|
|
|
536
537
|
throw Object.assign(new Error(`env_preflight_failed: ${blocking} · the run stays resumable: fix the environment and resume ${runDir}`), { code: "env_preflight_failed" });
|
|
537
538
|
}
|
|
538
539
|
|
|
540
|
+
/** No routed runtime is given less than this by the dispatch gate. */
|
|
541
|
+
const GATE_MINIMUM_PREFLIGHT_SEC = 60;
|
|
539
542
|
/**
|
|
540
543
|
* The live half of the gate for one launch. Every routed runtime either holds
|
|
541
544
|
* a verdict this machine recorded inside its freshness window -- reused, the
|
|
@@ -552,7 +555,7 @@ export async function assertEnvironmentReady(contract, runDir, sourceIdentity) {
|
|
|
552
555
|
* @param {ValidatedContract} contract
|
|
553
556
|
* @param {string} runDir
|
|
554
557
|
* @param {SourceIdentity|undefined} sourceIdentity
|
|
555
|
-
* @param {number} timeoutSec
|
|
558
|
+
* @param {number|undefined} timeoutSec
|
|
556
559
|
* @returns {Promise<ProbeResult[]>}
|
|
557
560
|
*/
|
|
558
561
|
async function livePreflightProbes(contract, runDir, sourceIdentity, timeoutSec) {
|
|
@@ -628,7 +631,7 @@ async function livePreflightProbes(contract, runDir, sourceIdentity, timeoutSec)
|
|
|
628
631
|
};
|
|
629
632
|
});
|
|
630
633
|
}
|
|
631
|
-
const probes = await preflightContract(join(runDir, "contract.json"), { liveTimeoutSec: timeoutSec, persisted: true });
|
|
634
|
+
const probes = await preflightContract(join(runDir, "contract.json"), { ...(timeoutSec === undefined ? {} : { liveTimeoutSec: timeoutSec }), minimumTimeoutSec: GATE_MINIMUM_PREFLIGHT_SEC, persisted: true });
|
|
632
635
|
// What this launch bought is durable from here on: every ask that reached a
|
|
633
636
|
// provider -- a refusal included, an answer being an answer -- is recorded
|
|
634
637
|
// under the provider's own identity. Silence and a command that never
|