faberun 0.25.0 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/package.json +1 -1
  2. package/skills/faberun/references/contract.md +34 -28
  3. package/src/campaign/campaign-brief-graph.mjs +218 -0
  4. package/src/campaign/campaign-brief.mjs +15 -167
  5. package/src/campaign/metrics-command.mjs +3 -1
  6. package/src/cli/brand.mjs +1 -0
  7. package/src/cli/plan.mjs +75 -3
  8. package/src/cli/setup.mjs +22 -5
  9. package/src/cli.mjs +35 -1
  10. package/src/contract/definition-of-done.mjs +48 -8
  11. package/src/contract/human-step.mjs +54 -0
  12. package/src/contract/index.mjs +25 -38
  13. package/src/contract/judge-independence.mjs +170 -0
  14. package/src/contract/judges.mjs +27 -0
  15. package/src/contract/provider.mjs +103 -0
  16. package/src/contract/runtime.mjs +19 -2
  17. package/src/contract/snapshot.mjs +31 -1
  18. package/src/contract/task-packet.mjs +4 -1
  19. package/src/contract/worker-result.mjs +35 -4
  20. package/src/engine/assignment.mjs +41 -5
  21. package/src/engine/backoff.mjs +30 -9
  22. package/src/engine/dispatch.mjs +19 -2
  23. package/src/engine/failover.mjs +10 -2
  24. package/src/engine/judge-gate.mjs +24 -4
  25. package/src/engine/judge-list.mjs +206 -0
  26. package/src/engine/lifecycle.mjs +29 -24
  27. package/src/engine/resume.mjs +49 -12
  28. package/src/engine/runtime-discovery.mjs +72 -11
  29. package/src/engine/scheduler.mjs +27 -6
  30. package/src/harnesses/catalogue.mjs +9 -2
  31. package/src/harnesses/index.mjs +2 -2
  32. package/src/harnesses/model-tiers.mjs +39 -0
  33. package/src/harnesses/protocol.mjs +4 -1
  34. package/src/host/config.mjs +7 -2
  35. package/src/plan/contest.mjs +78 -0
  36. package/src/plan/freeze.mjs +118 -7
  37. package/src/plan/human-step.mjs +38 -0
  38. package/src/plan/pipeline.mjs +224 -313
  39. package/src/plan/preflight.mjs +5 -3
  40. package/src/plan/proof-check.mjs +161 -0
  41. package/src/plan/proof-run.mjs +6 -2
  42. package/src/plan/proof-scope.mjs +175 -0
  43. package/src/plan/resolve.mjs +182 -0
  44. package/src/plan/reviewer.mjs +82 -0
  45. package/src/plan/rounds.mjs +446 -0
  46. package/src/plan/routing.mjs +11 -4
  47. package/src/plan/template.mjs +36 -14
  48. package/src/repo/signal-block.mjs +10 -3
  49. package/src/repo/source-identity.mjs +45 -5
  50. package/src/repo/worktree-prune.mjs +72 -0
  51. package/src/report/campaign-brief-estimate.mjs +1 -1
  52. package/src/report/campaign-brief.mjs +4 -0
  53. package/src/report/metrics-report.mjs +34 -1
  54. package/src/report/render.mjs +31 -10
@@ -362,9 +362,10 @@ export function upsertTierExhaustionCandidate(existing, role, runtimeId, exhaust
362
362
  * @param {Transition} schedule
363
363
  * @param {number} [now] epoch ms the backoff window is measured from
364
364
  * @param {string|null} [exhaustedUntil] the reset instant the exhausted candidate announced, if any
365
- * @returns {{blocked: RouteError|null, nextRuntime: string, ruleIndex: number|undefined, revision: number, hop: number, backoffSec: number, backoffUntil: string, composed: boolean, tierExhaustion: TierExhaustion|null}}
365
+ * @param {{judgeListState?: import("../contract/index.mjs").JudgeListState}} [options] R18: a judge-list hop already resolved by the impure caller (`engine/judge-list.mjs`'s `nextListJudge` reads the durable refusal and usage-window stores, which this decision module never touches); its `chosen` -- possibly null, once the list is exhausted -- replaces both the declared `runtime.fallback` edge and the dynamic-tier candidate a list-driven judge would otherwise get from `nextSynthesizedRuntime`/`nextSameTierRuntime`
366
+ * @returns {{blocked: RouteError|null, nextRuntime: string, ruleIndex: number|undefined, revision: number, hop: number, backoffSec: number, backoffUntil: string, composed: boolean, tierExhaustion: TierExhaustion|null, judgeList?: import("../contract/index.mjs").JudgeListState}}
366
367
  */
367
- export function planRoute(contract, node, state, role, error, current, schedule, now = Date.now(), exhaustedUntil = null) {
368
+ export function planRoute(contract, node, state, role, error, current, schedule, now = Date.now(), exhaustedUntil = null, options = {}) {
368
369
  const revision = state.revisions ?? 0;
369
370
  // Tier routing is scoped by generation, not revision: an invocation counts
370
371
  // as attempted here only when it ran in the current tier-exhaustion
@@ -394,17 +395,33 @@ export function planRoute(contract, node, state, role, error, current, schedule,
394
395
  // declared" — nextSynthesizedRuntime's own attempted-filter would otherwise
395
396
  // make that distinction unreachable.
396
397
  const declaredCandidate = synthesizedChain(contract, role, current)[0] ?? null;
397
- const fallback = nextSynthesizedRuntime(contract, role, current, attempted)
398
- ?? (dynamicComposed ? nextSameTierRuntime(contract, routing, role, current, attempted) : null);
398
+ // R18: once a list governs this hop (`options.judgeListState` is supplied),
399
+ // its precomputed pick *replaces* the declared `runtime.fallback` edge
400
+ // rather than following it -- the list is itself the ordering, and a list
401
+ // entry that happens to declare its own `fallback` must not hop there
402
+ // instead of the list's next eligible entry. Only when no list governs does
403
+ // the declared edge, then the dynamic-tier candidate, apply.
404
+ const judgeListState = options.judgeListState;
405
+ const listGoverned = judgeListState !== undefined;
406
+ const fallback = judgeListState !== undefined
407
+ ? judgeListState.chosen
408
+ : nextSynthesizedRuntime(contract, role, current, attempted) ?? (dynamicComposed ? nextSameTierRuntime(contract, routing, role, current, attempted) : null);
399
409
  const hop = nextHop(state, role, revision, schedule);
400
410
  const nextRuntime = schedule.kind === "reset" ? current : fallback ?? current;
411
+ // A list-governed hop is bounded by the list's own length (instructions
412
+ // R18 §4), never the one-hop cap that applies to a declared or dynamic-tier
413
+ // fallback, so it is exempted from the hop-cap check below exactly as a
414
+ // dynamic-tier fallback already is.
415
+ const noHopCap = dynamicComposed || listGoverned;
401
416
  const blocked = schedule.kind === "reset"
402
417
  ? null
403
418
  : fallback === null
404
- ? declaredCandidate !== null && attempted.has(declaredCandidate)
405
- ? { code: "provider_failover_cycle", message: `runtime ${declaredCandidate} was already attempted in ${role} revision ${revision}` }
406
- : { code: dynamicComposed ? "runtime_tier_exhausted" : error.code, message: dynamicComposed ? `no available runtime remains in tier ${String(contract.runtimes[current]?.tier ?? "unknown")} for ${role}` : error.message }
407
- : hop > 1 && !dynamicComposed
419
+ ? listGoverned
420
+ ? { code: "judge_list_exhausted", message: `no eligible entry remains in the judge list for ${role}` }
421
+ : declaredCandidate !== null && attempted.has(declaredCandidate)
422
+ ? { code: "provider_failover_cycle", message: `runtime ${declaredCandidate} was already attempted in ${role} revision ${revision}` }
423
+ : { code: dynamicComposed ? "runtime_tier_exhausted" : error.code, message: dynamicComposed ? `no available runtime remains in tier ${String(contract.runtimes[current]?.tier ?? "unknown")} for ${role}` : error.message }
424
+ : hop > 1 && !noHopCap
408
425
  ? { code: "provider_failover_hop_cap", message: "provider failover exceeded the one-hop cap" }
409
426
  : null;
410
427
  const backoffSec = schedule.kind === "reset"
@@ -417,7 +434,7 @@ export function planRoute(contract, node, state, role, error, current, schedule,
417
434
  const tierExhaustion = dynamicComposed
418
435
  ? upsertTierExhaustionCandidate(state.routing?.tierExhaustion, role, current, exhaustedUntil)
419
436
  : null;
420
- return { blocked, nextRuntime, ruleIndex: undefined, revision, hop, backoffSec, backoffUntil, composed: composedAssignment, tierExhaustion };
437
+ return { blocked, nextRuntime, ruleIndex: undefined, revision, hop, backoffSec, backoffUntil, composed: composedAssignment, tierExhaustion, judgeList: options.judgeListState };
421
438
  }
422
439
 
423
440
  /**
@@ -464,6 +481,10 @@ export function buildRouting(state, { role, error, current, plan, schedule, usag
464
481
  history: [...(state.routing?.history ?? []), { ...shared, runtime: current, status, errorCode }].slice(-MAX_ROUTING_HISTORY),
465
482
  currentOverride: override,
466
483
  ...(plan.tierExhaustion ? { tierExhaustion: plan.tierExhaustion } : {}),
484
+ // Unlike tierExhaustion, an untouched judgeList is not deleted above: a
485
+ // worker exhaustion round in the same call never resolved one, and the
486
+ // node's own list evidence must survive it unchanged.
487
+ ...(plan.judgeList !== undefined ? { judgeList: plan.judgeList } : {}),
467
488
  },
468
489
  override,
469
490
  errorCode,
@@ -37,7 +37,8 @@ import { mkdirSync, statSync } from "node:fs";
37
37
  import { READ_BYTE_LIMIT, READ_LINE_LIMIT, harnessCapabilities, normalizeProviderResult, writesWorkspace } from "../harnesses/index.mjs";
38
38
  import { writeJsonAtomic } from "../run/store.mjs";
39
39
  import { judgeReaskInstruction, reviewMode } from "../contract/review-modes.mjs";
40
- import { routeRuntimeForState, runtimeSnapshot } from "./failover.mjs";
40
+ import { isSameProviderReviewPair } from "../contract/judge-independence.mjs";
41
+ import { previousAttemptRuntimeId, routeRuntimeForState, runtimeSnapshot } from "./failover.mjs";
41
42
  import { fingerprintRuntime, forceFreshSession, phaseInvocationPlan } from "./phase-session.mjs";
42
43
 
43
44
  /** @typedef {import("./phase-session.mjs").SessionPolicy} SessionPolicy */
@@ -495,6 +496,22 @@ export async function startJudge(contract, node, state, runDir, running, workerR
495
496
  /** @type {import("../contract/index.mjs").VerificationState|null} */ (state.verification),
496
497
  );
497
498
  state.review = reviewMode(node.gate);
499
+ // The judge candidate this attempt would actually use, and the worker
500
+ // runtime read off the durable invocation record rather than
501
+ // `state.runtime`: on the first judge dispatch of an attempt `state.runtime`
502
+ // still holds the worker, but `startJudge` overwrites it with the judge's
503
+ // own runtime below, and a bounded re-ask (review.mjs) or a judge-failure
504
+ // retry (settle-judge.mjs) then calls `startJudge` again with the judge
505
+ // already in `state.runtime` -- comparing against that would compare the
506
+ // judge to itself and always mark the node. The invocation record survives
507
+ // every such re-dispatch, so the mark reflects the pairing that ran,
508
+ // fallback included -- not `node.sameProviderReview`, which only says the
509
+ // pairing was admitted under same-vendor mode at contract validation,
510
+ // primary or fallback alike.
511
+ const judgeCandidate = routeRuntimeForState(contract, node, state, "judge");
512
+ const workerRuntimeId = previousAttemptRuntimeId(state, "worker");
513
+ const workerRuntime = workerRuntimeId ? contract.runtimes[workerRuntimeId] : state.runtime;
514
+ state.sameProviderReview = isSameProviderReviewPair(contract, workerRuntime, judgeCandidate);
498
515
  if (verdict.verdict === "fail") return { kind: "rejected", verdict };
499
516
  // `skipWhen` is checked before the ordinary judgment rule so a green-and-small
500
517
  // change settles mechanically even when a Definition of Done item carries
@@ -502,7 +519,7 @@ export async function startJudge(contract, node, state, runDir, running, workerR
502
519
  // and a gate whose review mode is `none` is skipped there exactly as before.
503
520
  if (judgeSkippedByScope(node, state)) return { kind: "settle", gate: verdict };
504
521
  if (!judgeRequired(node)) return { kind: "settle", gate: verdict };
505
- const runtime = routeRuntimeForState(contract, node, state, "judge");
522
+ const runtime = judgeCandidate;
506
523
  const paths = logPaths(runDir, node.id, "judge", state.attempt);
507
524
  state.phase = "judge";
508
525
  state.runtime = runtime;
@@ -17,6 +17,7 @@
17
17
  import { harnessCapabilities } from "../harnesses/index.mjs";
18
18
  import { isRuntimeAvailable, nextSameTierRuntime } from "./runtime-discovery.mjs";
19
19
  import { routeRuntime } from "../contract/runtime.mjs";
20
+ import { effectiveProvider } from "../contract/provider.mjs";
20
21
 
21
22
  /** @typedef {import("../contract/index.mjs").ValidatedNode} ValidatedNode */
22
23
  /** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
@@ -188,11 +189,17 @@ export function routingBackoffActive(state, phase) {
188
189
  * survives both a routed hop and the revision boundary: `resetPhaseRouting`
189
190
  * clears the routing override between revisions, but never the invocations.
190
191
  *
192
+ * Exported so a same-provider-review mark (R20) can be taken from the worker
193
+ * invocation that actually ran instead of `state.runtime`, which `startJudge`
194
+ * overwrites with the judge's own runtime as soon as it dispatches -- a re-ask
195
+ * or a judge-failure retry then calls `startJudge` again with the judge still
196
+ * in `state.runtime`, so comparing against it compares the judge to itself.
197
+ *
191
198
  * @param {NodeSnapshot} state
192
199
  * @param {"worker"|"judge"} role
193
200
  * @returns {string|undefined}
194
201
  */
195
- function previousAttemptRuntimeId(state, role) {
202
+ export function previousAttemptRuntimeId(state, role) {
196
203
  const found = [...(state.invocations ?? [])].reverse().find((invocation) => invocation.phase === role)?.runtimeId;
197
204
  return typeof found === "string" ? found : undefined;
198
205
  }
@@ -223,7 +230,8 @@ function admitsAffinity(contract, state, role, candidate) {
223
230
  if (availability && !isRuntimeAvailable(availability)) return false;
224
231
  if (role !== "judge") return true;
225
232
  const workerId = previousAttemptRuntimeId(state, "worker") ?? state.routing?.assignments?.worker;
226
- return runtime.vendor !== (workerId ? contract.runtimes[workerId]?.vendor : null);
233
+ const workerRuntime = workerId ? contract.runtimes[workerId] : undefined;
234
+ return effectiveProvider(runtime) !== (workerRuntime ? effectiveProvider(workerRuntime) : null);
227
235
  }
228
236
 
229
237
  /**
@@ -142,8 +142,9 @@ function proveVerification(id, proof, recorded) {
142
142
  * `--test-skip-pattern`) is judged on more than its exit code, because the
143
143
  * runner exits 0 whether its filter selected anything or not. The proof runs
144
144
  * with the TAP reporter selected through `NODE_OPTIONS` and is refused when the
145
- * output carries TAP's zero-plan line. Measured 2026-09-21 on node v26.8.1: a
146
- * filter that matches emits `1..0` zero times, a filter that matches nothing
145
+ * output says the filter selected nothing (`tapSelectedNothing`). Measured
146
+ * 2026-09-21 on node v26.8.1, in a single-file run: a filter that matches
147
+ * emits `1..0` zero times, a filter that matches nothing
147
148
  * emits it exactly once, a run with no filter emits it zero times, and an empty
148
149
  * suite under a matching filter emits no nested zero plan. The default reporter
149
150
  * cannot make the distinction -- both cases print identical counters, because
@@ -154,7 +155,26 @@ function proveVerification(id, proof, recorded) {
154
155
  * never sees it.
155
156
  */
156
157
  const TEST_FILTER_FLAGS = ["--test-name-pattern", "--test-skip-pattern"];
157
- const TAP_ZERO_PLAN = /^1\.\.0$/mu;
158
+ const TAP_ZERO_PLAN = /^1\.\.0$/gmu;
159
+ const TAP_TOP_LEVEL_RESULT = /^(?:not )?ok \d+ /gmu;
160
+
161
+ /**
162
+ * Whether a filtered run's TAP says its filter selected no test. A run of
163
+ * several files prints, for each file the filter selected nothing in, an
164
+ * unindented `1..0` followed by that file's own top-level `ok` line, while a
165
+ * selected test prints its own top-level result with no zero plan in front of
166
+ * it. Measured 2026-09-26 on node v26.8.1 (AP1 of safe-to-hand-to-a-friend):
167
+ * reading a single `1..0` as "nothing selected" refused two proofs whose
168
+ * filter had selected a passing test in one of several files. Nothing was
169
+ * selected exactly when every top-level result is one of those file lines.
170
+ *
171
+ * @param {string} stdout
172
+ * @returns {boolean}
173
+ */
174
+ function tapSelectedNothing(stdout) {
175
+ const zeroPlans = stdout.match(TAP_ZERO_PLAN)?.length ?? 0;
176
+ return zeroPlans > 0 && zeroPlans >= (stdout.match(TAP_TOP_LEVEL_RESULT)?.length ?? 0);
177
+ }
158
178
 
159
179
  /**
160
180
  * The environment a filtered proof runs under: the ambient environment with
@@ -252,7 +272,7 @@ async function proveCommand(id, proof, cwd, timeoutMs) {
252
272
  child.on("close", (code, signal) => {
253
273
  // Its own detail, not an ordinary command failure: the command succeeded,
254
274
  // so what failed is that the declared filter selected nothing to prove.
255
- if (code === 0 && signal === null && filters.length > 0 && TAP_ZERO_PLAN.test(stdout)) {
275
+ if (code === 0 && signal === null && filters.length > 0 && tapSelectedNothing(stdout)) {
256
276
  finish({
257
277
  id,
258
278
  kind: "command",
@@ -0,0 +1,206 @@
1
+ /**
2
+ * R18: the judge of a gated node is the first eligible entry of a static,
3
+ * ordered list rather than the single strongest cross-vendor candidate
4
+ * `composeAssignments` otherwise ranks. Split out of `runtime-discovery.mjs`
5
+ * and `lifecycle.mjs` -- both already near this repository's 800-line
6
+ * ceiling -- and kept out of `backoff.mjs`, whose `planRoute` is a pure
7
+ * decision this module cannot be: picking an entry reads the durable refusal
8
+ * and usage-window stores.
9
+ */
10
+ import { effectiveProvider } from "../contract/provider.mjs";
11
+ import { availabilityKey, readRefusal } from "../run/availability.mjs";
12
+ import { readUsageWindows, usageAccountOf } from "../run/usage-windows.mjs";
13
+ import { getHarness } from "../harnesses/index.mjs";
14
+
15
+ /** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
16
+ /** @typedef {{id: string, reason: string}} JudgeSkip */
17
+ /** @typedef {{chosen: string|null, skipped: JudgeSkip[]}} JudgeListPick */
18
+ /** @typedef {{list: string[], chosen: string|null, skipped: JudgeSkip[]}} JudgeListState */
19
+
20
+ /**
21
+ * A usage window at or above this spend is treated as unavailable for a judge
22
+ * pick, so a node's review does not race the account into a refusal the next
23
+ * call would hit anyway.
24
+ */
25
+ export const JUDGE_USAGE_WINDOW_LIMIT_PERCENT = 90;
26
+
27
+ /**
28
+ * The static judge list a node's assignment reads: the contract's own list
29
+ * wins over the machine default, exactly as `runtimeDefaults.worker` already
30
+ * wins over `config.worker` in `composeAssignments`.
31
+ *
32
+ * @param {{judges?: string[]}} contract
33
+ * @param {{judges?: string[]}|null|undefined} config
34
+ * @returns {string[]|undefined}
35
+ */
36
+ export function resolveJudgeList(contract, config) {
37
+ return contract.judges ?? config?.judges ?? undefined;
38
+ }
39
+
40
+ /**
41
+ * Every provider a worker attempt on this node could actually run on: its own
42
+ * canonical provider, plus every hop of its declared fallback chain. A judge
43
+ * of any of these providers cannot independently review whichever one
44
+ * actually ran. Walked directly off `runtime.fallback` rather than through
45
+ * `failover.mjs`'s `synthesizedChain`, so this module does not import a
46
+ * runtime-discovery consumer and risk a cycle back through
47
+ * `run/availability.mjs`.
48
+ *
49
+ * @param {ValidatedContract} contract
50
+ * @param {string} workerId
51
+ * @returns {Set<string|undefined>}
52
+ */
53
+ function reachableWorkerProviders(contract, workerId) {
54
+ const providers = new Set([effectiveProvider(contract.runtimes[workerId])]);
55
+ const seen = new Set([workerId]);
56
+ let current = workerId;
57
+ for (;;) {
58
+ const next = contract.runtimes[current]?.fallback;
59
+ if (!next || seen.has(next) || !contract.runtimes[next]) break;
60
+ seen.add(next);
61
+ providers.add(effectiveProvider(contract.runtimes[next]));
62
+ current = next;
63
+ }
64
+ return providers;
65
+ }
66
+
67
+ /**
68
+ * Is this runtime's account inside `JUDGE_USAGE_WINDOW_LIMIT_PERCENT` of a
69
+ * usage window closing? `null` means no window is known -- unknown must not
70
+ * read as over budget, exactly as an absent availability record must not read
71
+ * as exhausted (see `admitsAffinity` in `failover.mjs`).
72
+ *
73
+ * @param {import("../contract/index.mjs").ValidatedRuntime} runtime
74
+ * @param {number} now
75
+ * @returns {import("../harnesses/codex/usage-window.mjs").UsageWindow|null}
76
+ */
77
+ function overspentWindow(runtime, now) {
78
+ const account = usageAccountOf(runtime);
79
+ if (!account) return null;
80
+ return readUsageWindows(account, now).find((window) => window.usedPercent > JUDGE_USAGE_WINDOW_LIMIT_PERCENT) ?? null;
81
+ }
82
+
83
+ /**
84
+ * The first eligible entry of an ordered judge list, and why every earlier
85
+ * entry was skipped: already attempted this run, a provider the worker (or
86
+ * its fallback chain) already runs, a recorded refusal, or an account usage
87
+ * window over `JUDGE_USAGE_WINDOW_LIMIT_PERCENT`.
88
+ *
89
+ * @param {ValidatedContract} contract
90
+ * @param {string[]} list
91
+ * @param {string} workerId
92
+ * @param {{attempted?: Iterable<string>, now?: number}} [options]
93
+ * @returns {JudgeListPick}
94
+ */
95
+ export function selectListJudge(contract, list, workerId, options = {}) {
96
+ const attempted = new Set(options.attempted ?? []);
97
+ const now = options.now ?? Date.now();
98
+ const workerProviders = reachableWorkerProviders(contract, workerId);
99
+ /** @type {JudgeSkip[]} */
100
+ const skipped = [];
101
+ for (const id of list) {
102
+ if (attempted.has(id)) {
103
+ skipped.push({ id, reason: "already attempted this run" });
104
+ continue;
105
+ }
106
+ const runtime = contract.runtimes[id];
107
+ if (!runtime) {
108
+ skipped.push({ id, reason: "not declared in contract.runtimes" });
109
+ continue;
110
+ }
111
+ const provider = effectiveProvider(runtime);
112
+ if (workerProviders.has(provider)) {
113
+ skipped.push({ id, reason: `same provider as the worker: ${provider}` });
114
+ continue;
115
+ }
116
+ const key = availabilityKey({ harness: runtime.harness, model: runtime.model, executable: getHarness(runtime.harness).executable(runtime) });
117
+ const refusal = readRefusal(key, now);
118
+ if (refusal) {
119
+ skipped.push({ id, reason: `recorded refusal: ${refusal.reason}` });
120
+ continue;
121
+ }
122
+ const window = overspentWindow(runtime, now);
123
+ if (window) {
124
+ skipped.push({ id, reason: `usage window ${window.window} at ${window.usedPercent}%, above ${JUDGE_USAGE_WINDOW_LIMIT_PERCENT}%` });
125
+ continue;
126
+ }
127
+ // An attempted entry after the pick is still recorded: `nextListJudge`
128
+ // rebuilds the attempted set from these entries alone, so dropping one
129
+ // here would let a later hop return to a judge that already refused.
130
+ const after = list.slice(list.indexOf(id) + 1).filter((later) => attempted.has(later));
131
+ return { chosen: id, skipped: [...skipped, ...after.map((later) => ({ id: later, reason: "already attempted this run" }))] };
132
+ }
133
+ return { chosen: null, skipped };
134
+ }
135
+
136
+ /**
137
+ * The judge-list evidence a node's initial assignment records: the list it
138
+ * read, the entry it chose (or null when every entry was skipped), and why
139
+ * each earlier one was.
140
+ *
141
+ * @param {ValidatedContract} contract
142
+ * @param {string[]} list
143
+ * @param {string} workerId
144
+ * @param {number} [now]
145
+ * @returns {JudgeListState}
146
+ */
147
+ export function initialJudgeListState(contract, list, workerId, now = Date.now()) {
148
+ const pick = selectListJudge(contract, list, workerId, { now });
149
+ return { list, chosen: pick.chosen, skipped: pick.skipped };
150
+ }
151
+
152
+ /**
153
+ * The next hop out of a list-driven judge that was just refused: the current
154
+ * choice joins the excluded set (it is the one that just failed), so the same
155
+ * eligibility pass never returns to it -- the hop-by-hop rule. Only an entry
156
+ * that actually ran and then failed belongs in that excluded set; a candidate
157
+ * that was skipped for cause (same provider, a recorded refusal, an overspent
158
+ * window) is re-checked fresh instead, so it keeps its real reason rather than
159
+ * being relabelled "already attempted" on every later hop. A prior hop's own
160
+ * "already attempted this run" entries name exactly the ids that were chosen
161
+ * and failed before this one, so folding those back in carries that set
162
+ * forward without a dedicated field.
163
+ *
164
+ * @param {ValidatedContract} contract
165
+ * @param {JudgeListState} judgeListState
166
+ * @param {string} workerId
167
+ * @param {number} [now]
168
+ * @returns {JudgeListState}
169
+ */
170
+ export function nextListJudge(contract, judgeListState, workerId, now = Date.now()) {
171
+ const previouslyChosen = judgeListState.skipped
172
+ .filter((entry) => entry.reason === "already attempted this run")
173
+ .map((entry) => entry.id);
174
+ const attempted = new Set([...previouslyChosen, ...(judgeListState.chosen ? [judgeListState.chosen] : [])]);
175
+ // A fresh pass re-derives the whole prefix up to (and including) wherever
176
+ // it lands: every id the loop reaches before returning is pushed to
177
+ // `pick.skipped`, on the first hop as on the fifth. That is already a
178
+ // complete accounting of this hop, so carrying the previous hops' entries
179
+ // forward too would only repeat each of them once per later hop.
180
+ const pick = selectListJudge(contract, judgeListState.list, workerId, { attempted, now });
181
+ return { list: judgeListState.list, chosen: pick.chosen, skipped: pick.skipped };
182
+ }
183
+
184
+ /**
185
+ * Does a judge's next fallback runtime share the vendor of the worker it
186
+ * would be reviewing? Reachability depends on which worker runtime actually
187
+ * ran, which a static contract cannot know, so this is checked at the moment
188
+ * a fallback is about to be taken rather than at contract validation.
189
+ * A list-driven pick already excludes the worker's reachable providers
190
+ * (`selectListJudge`), so this only ever fires for the single declared
191
+ * `runtime.fallback` edge.
192
+ *
193
+ * @param {ValidatedContract} contract
194
+ * @param {string|null|undefined} workerRuntimeId
195
+ * @param {string} nextRuntimeId
196
+ * @returns {{code: string, message: string}|null}
197
+ */
198
+ export function judgeFallbackVendorConflict(contract, workerRuntimeId, nextRuntimeId) {
199
+ const workerVendor = workerRuntimeId ? contract.runtimes[workerRuntimeId]?.vendor : undefined;
200
+ const judgeFallbackVendor = contract.runtimes[nextRuntimeId]?.vendor;
201
+ if (!workerVendor || !judgeFallbackVendor || workerVendor !== judgeFallbackVendor) return null;
202
+ return {
203
+ code: "judge_fallback_vendor_conflict",
204
+ message: `judge fallback runtime ${nextRuntimeId} shares vendor ${judgeFallbackVendor} with worker runtime ${workerRuntimeId}`,
205
+ };
206
+ }
@@ -35,9 +35,10 @@ import {
35
35
  import { exhaustedUntilOf } from "./runtime-discovery.mjs";
36
36
  import { getHarness, normalizeProviderAvailability } from "../harnesses/index.mjs";
37
37
  import { availabilityKey, recordRefusal, refusalOfAvailability } from "../run/availability.mjs";
38
+ import { judgeFallbackVendorConflict, nextListJudge } from "./judge-list.mjs";
38
39
 
39
40
  import { acquire as acquireLock } from "../run/lock.mjs";
40
- import { parseDiscoveryResult } from "../contract/worker-result.mjs";
41
+ import { invalidResultRepair, parseDiscoveryResult } from "../contract/worker-result.mjs";
41
42
  import { boundedUtf8, errorCode, errorMessage, excerpt } from "../util.mjs";
42
43
  import { readJson, writeJsonAtomic } from "../run/store.mjs";
43
44
  import { invocationAlive } from "./process.mjs";
@@ -443,7 +444,7 @@ export async function finalizeClosedJobs(contract, runDir, states, closed, lock,
443
444
  // path fails it today.
444
445
  if (workerResultError) {
445
446
  if (envelope.status === "done") {
446
- await applyInvalidWorkerResult(contract, job.node, state, runDir, running, lock, errorMessage(workerResultError), states, campaignPath);
447
+ await applyInvalidWorkerResult(contract, job.node, state, runDir, running, lock, workerResultError, states, campaignPath);
447
448
  continue;
448
449
  }
449
450
  adoptedWorkerResult = null;
@@ -549,7 +550,7 @@ export async function finalizeClosedJobs(contract, runDir, states, closed, lock,
549
550
  }, lock);
550
551
  continue;
551
552
  }
552
- await applyInvalidWorkerResult(contract, job.node, state, runDir, running, lock, errorMessage(error), states, campaignPath);
553
+ await applyInvalidWorkerResult(contract, job.node, state, runDir, running, lock, error, states, campaignPath);
553
554
  continue;
554
555
  }
555
556
  // The artifact demand follows the documented discovery contract, not the
@@ -562,7 +563,7 @@ export async function finalizeClosedJobs(contract, runDir, states, closed, lock,
562
563
  try {
563
564
  parseDiscoveryResult(workerResult, attemptWorkspace(state) ?? contract.cwd);
564
565
  } catch (error) {
565
- await applyInvalidWorkerResult(contract, job.node, state, runDir, running, lock, errorMessage(error), states, campaignPath);
566
+ await applyInvalidWorkerResult(contract, job.node, state, runDir, running, lock, error, states, campaignPath);
566
567
  continue;
567
568
  }
568
569
  }
@@ -641,7 +642,15 @@ export function handleProviderExhaustion(contract, runDir, node, state, role, en
641
642
  const runtime = contract.runtimes?.[current];
642
643
  const refusal = runtime ? refusalOfAvailability(normalizeProviderAvailability(runtime, envelope)) : null;
643
644
  if (runtime && refusal) recordRefusal(availabilityKey({ harness: runtime.harness, model: runtime.model, executable: getHarness(runtime.harness).executable(runtime) }), refusal);
644
- const plan = planRoute(contract, node, state, role, error, current, schedule, now, exhaustedUntil);
645
+ // R18: a list-driven judge that is about to hop reads the durable refusal
646
+ // and usage-window stores here -- the impure orchestration layer, not
647
+ // `planRoute`'s pure decision -- so `current` joins the excluded set before
648
+ // the next eligible entry is chosen. A reset (wait, no hop) leaves the list
649
+ // untouched: it costs no runtime and must not spend a list entry either.
650
+ const judgeListState = role === "judge" && schedule.kind === "failover" && state.routing?.judgeList
651
+ ? nextListJudge(contract, state.routing.judgeList, sourceWorkerRuntime(state) ?? state.routing?.assignments?.worker ?? current, now)
652
+ : undefined;
653
+ const plan = planRoute(contract, node, state, role, error, current, schedule, now, exhaustedUntil, judgeListState ? { judgeListState } : {});
645
654
  const status = envelope.status === "failed" ? "failed" : "exhausted";
646
655
  if (plan.blocked) {
647
656
  // A caller that classified the failure itself also owns what happens when
@@ -659,17 +668,21 @@ export function handleProviderExhaustion(contract, runDir, node, state, role, en
659
668
  phase: role,
660
669
  result: state.result,
661
670
  usage: state.usage,
662
- routing: { ...(state.routing ?? {}), tierExhaustion },
671
+ routing: { ...(state.routing ?? {}), tierExhaustion, ...(judgeListState ? { judgeList: judgeListState } : {}) },
663
672
  error: { ...plan.blocked, ...(exhaustedUntil ? { exhaustedUntil } : {}) },
664
673
  }, lock);
665
674
  void raiseNodeAttention(campaignPath, runDir, state, plan.blocked.code).catch(() => {});
666
675
  } else {
667
- // Any other block reason is not this feature's evidence to keep.
676
+ // `judge_list_exhausted` reaches this branch (it is not
677
+ // `runtime_tier_exhausted`), and its evidence -- naming the judge that
678
+ // was just refused and why every remaining entry was ineligible -- is
679
+ // exactly this feature's evidence to keep; only `tierExhaustion` is not.
668
680
  clearTierExhaustion(state);
669
681
  transition(runDir, state, "exhausted", {
670
682
  phase: role,
671
683
  result: state.result,
672
684
  usage: state.usage,
685
+ ...(judgeListState ? { routing: { ...(state.routing ?? {}), judgeList: judgeListState } } : {}),
673
686
  error: { ...plan.blocked, ...(exhaustedUntil ? { exhaustedUntil } : {}) },
674
687
  }, lock);
675
688
  }
@@ -681,21 +694,11 @@ export function handleProviderExhaustion(contract, runDir, node, state, role, en
681
694
  // so it is refused here, and the node is parked for a human rather than
682
695
  // quietly arbitrated by the vendor it is supposed to check.
683
696
  if (role === "judge" && plan.nextRuntime !== current) {
684
- const workerRuntimeId = sourceWorkerRuntime(state);
685
- const workerVendor = workerRuntimeId ? contract.runtimes[workerRuntimeId]?.vendor : undefined;
686
- const judgeFallbackVendor = contract.runtimes[plan.nextRuntime]?.vendor;
687
- if (workerVendor && judgeFallbackVendor && workerVendor === judgeFallbackVendor) {
697
+ const conflict = judgeFallbackVendorConflict(contract, sourceWorkerRuntime(state), plan.nextRuntime);
698
+ if (conflict) {
688
699
  if (precomputed) return false;
689
700
  clearTierExhaustion(state);
690
- transition(runDir, state, "blocked", {
691
- phase: role,
692
- result: state.result,
693
- usage: state.usage,
694
- error: {
695
- code: "judge_fallback_vendor_conflict",
696
- message: `judge fallback runtime ${plan.nextRuntime} shares vendor ${judgeFallbackVendor} with worker runtime ${workerRuntimeId}`,
697
- },
698
- }, lock);
701
+ transition(runDir, state, "blocked", { phase: role, result: state.result, usage: state.usage, error: conflict }, lock);
699
702
  return true;
700
703
  }
701
704
  }
@@ -737,19 +740,21 @@ function applyRoute(contract, runDir, state, lock, { role, error, current, plan,
737
740
  * told to return. A second one is evidence about the provider rather than the
738
741
  * packet, so the node records a protocol_failure and takes its failover edge —
739
742
  * and when no edge remains, it blocks and raises attention rather than filing
740
- * a quiet exhaustion nobody reads.
743
+ * a quiet exhaustion nobody reads. The repair names what was wrong
744
+ * (invalidResultRepair), so a broken byte ceiling is not told about fences.
741
745
  *
742
- * @param {ValidatedContract} contract @param {ValidatedNode} node @param {NodeSnapshot} state @param {string} runDir @param {Map<string, Job>|null} running @param {LockHandle} lock @param {string} message @param {Map<string, NodeSnapshot>} states @param {string} campaignPath
746
+ * @param {ValidatedContract} contract @param {ValidatedNode} node @param {NodeSnapshot} state @param {string} runDir @param {Map<string, Job>|null} running @param {LockHandle} lock @param {unknown} cause @param {Map<string, NodeSnapshot>} states @param {string} campaignPath
743
747
  */
744
- export async function applyInvalidWorkerResult(contract, node, state, runDir, running, lock, message, states, campaignPath) {
748
+ export async function applyInvalidWorkerResult(contract, node, state, runDir, running, lock, cause, states, campaignPath) {
745
749
  clearTierExhaustion(state);
750
+ const message = errorMessage(cause);
746
751
  const verdict = /** @type {JudgeVerdict} */ ({
747
752
  verdict: "fail",
748
753
  maxSeverity: "critical",
749
754
  summary: "worker result did not match the structured result protocol",
750
755
  findings: [{
751
756
  severity: "critical",
752
- description: "the entire final message must be exactly the required JSON object: no markdown fences, no prose before or after it. Return it as the only content of the final message.",
757
+ description: invalidResultRepair(cause),
753
758
  evidence: boundedUtf8(message, 4 * 1024),
754
759
  }],
755
760
  });
@@ -44,6 +44,7 @@ import { validateRunMetadata, OPERATOR_ANSWER_MAX_BYTES } from "../contract/snap
44
44
  import { verificationFailureVerdict } from "./judge-gate.mjs";
45
45
  import { verificationFailureWithScope } from "../contract/scope-findings.mjs";
46
46
  import { applyRejection, applyVerificationFailure, raiseNodeAttention, settleDone } from "./settle.mjs";
47
+ import { isHumanStepPending } from "../contract/human-step.mjs";
47
48
 
48
49
  /** @typedef {import("../repo/integrate.mjs").IntegrationResult} IntegrationResult */
49
50
  /** @typedef {import("./lifecycle.mjs").Invocation} Invocation */
@@ -139,16 +140,37 @@ export async function resumeRun(runDirPath, options = {}) {
139
140
  const answerNode = contract.nodes.find((node) => node.id === options.answer?.node);
140
141
  if (!answerNode) throw new Error(`unknown node id: ${options.answer.node}`);
141
142
  const answerState = states.get(options.answer.node);
142
- if (!answerState || !isBlockedContextTerminal(answerState)) {
143
- throw new Error(`node ${options.answer.node} is not blocked on missing context`);
144
- }
145
143
  const answerText = readOperatorAnswer(options.answer.path);
146
- const answerOverride = /** @type {import("../contract/index.mjs").ExecutionOverride} */ (/** @type {unknown} */ ({
147
- kind: "operator-answer",
148
- reason: `operator answered missing context for node ${options.answer.node}`,
149
- text: answerText,
150
- }));
151
- recordExecutionOverride(runDir, answerState, answerOverride, lock);
144
+ if (answerNode.humanStep) {
145
+ if (!answerState || !isHumanStepPending(answerState)) {
146
+ throw new Error(`node ${options.answer.node} is not a pending human step`);
147
+ }
148
+ recordExecutionOverride(runDir, answerState, /** @type {import("../contract/index.mjs").ExecutionOverride} */ (/** @type {unknown} */ ({
149
+ kind: "operator-answer",
150
+ reason: `operator confirmed the human step for node ${options.answer.node}`,
151
+ text: answerText,
152
+ })), lock);
153
+ // A human step never dispatches to a provider and leaves no attempt
154
+ // worktree for `settleDone` to seal and integrate -- the operator's
155
+ // confirmation is the whole of its work, so it settles done directly,
156
+ // here, before `planResumeRetry` (which knows nothing of `humanStep`)
157
+ // ever classifies it.
158
+ transition(runDir, answerState, "done", {
159
+ phase: "complete",
160
+ error: null,
161
+ result: { status: "done", summary: boundedSummary(answerText.trim() || answerNode.humanStep.step), verification: [], artifacts: [], missingContext: [] },
162
+ }, lock);
163
+ } else {
164
+ if (!answerState || !isBlockedContextTerminal(answerState)) {
165
+ throw new Error(`node ${options.answer.node} is not blocked on missing context`);
166
+ }
167
+ const answerOverride = /** @type {import("../contract/index.mjs").ExecutionOverride} */ (/** @type {unknown} */ ({
168
+ kind: "operator-answer",
169
+ reason: `operator answered missing context for node ${options.answer.node}`,
170
+ text: answerText,
171
+ }));
172
+ recordExecutionOverride(runDir, answerState, answerOverride, lock);
173
+ }
152
174
  }
153
175
  const plan = planResumeRetry(contract, states, { node: options.node, reconcile: options.reconcile, answer: options.answer?.node });
154
176
  for (const item of plan.attention) {
@@ -164,6 +186,7 @@ export async function resumeRun(runDirPath, options = {}) {
164
186
  if (!state) continue;
165
187
  if (state.status === "done") continue;
166
188
  if (isBlockedContextTerminal(state) && node.id !== options.answer?.node) continue;
189
+ if (isHumanStepPending(state) && node.id !== options.answer?.node) continue;
167
190
  const action = plan.actions.get(node.id) ?? "recover";
168
191
  // Adoption before retry: an unresolved blocking review is re-judged from
169
192
  // the preserved worker result, never reset to a fresh worker attempt.
@@ -284,7 +307,7 @@ export async function resumeRun(runDirPath, options = {}) {
284
307
  ? recoverWorkerResult(runDir, state, contract, node)
285
308
  : canonicalWorkerResultText(runDir, node.id) ?? recovery.result;
286
309
  } catch (error) {
287
- await applyInvalidWorkerResult(contract, node, state, runDir, null, lock, errorMessage(error), states, campaign.path);
310
+ await applyInvalidWorkerResult(contract, node, state, runDir, null, lock, error, states, campaign.path);
288
311
  continue;
289
312
  }
290
313
  if (recovery.phase === "worker" && workerResult !== null && workerResult !== undefined) {
@@ -301,7 +324,7 @@ export async function resumeRun(runDirPath, options = {}) {
301
324
  try {
302
325
  parsedWorkerResult = parseWorkerResult(String(extractJson(workerResult) ?? workerResult));
303
326
  } catch (error) {
304
- await applyInvalidWorkerResult(contract, node, state, runDir, null, lock, errorMessage(error), states, campaign.path);
327
+ await applyInvalidWorkerResult(contract, node, state, runDir, null, lock, error, states, campaign.path);
305
328
  continue;
306
329
  }
307
330
  // Same discriminator as lifecycle.mjs: the artifact is owed only by
@@ -311,7 +334,7 @@ export async function resumeRun(runDirPath, options = {}) {
311
334
  try {
312
335
  parseDiscoveryResult(parsedWorkerResult, attemptWorkspace(state) ?? contract.cwd);
313
336
  } catch (error) {
314
- await applyInvalidWorkerResult(contract, node, state, runDir, null, lock, errorMessage(error), states, campaign.path);
337
+ await applyInvalidWorkerResult(contract, node, state, runDir, null, lock, error, states, campaign.path);
315
338
  continue;
316
339
  }
317
340
  }
@@ -523,6 +546,20 @@ export async function resumeRun(runDirPath, options = {}) {
523
546
  export function isBlockedContextTerminal(state) {
524
547
  return state.status === "blocked" && state.error?.code === "context_missing";
525
548
  }
549
+ /**
550
+ * The operator-answer result summary for a human-step node's `done`
551
+ * transition, bounded to the worker-result protocol's own summary ceiling
552
+ * (`contract/worker-result.mjs`'s `RESULT_LIMITS.summaryBytes`) since this
553
+ * writes the same `result.summary` field a provider's own answer would.
554
+ *
555
+ * @param {string} text
556
+ * @returns {string}
557
+ */
558
+ function boundedSummary(text) {
559
+ const maxBytes = 4 * 1024;
560
+ if (Buffer.byteLength(text, "utf8") <= maxBytes) return text;
561
+ return `${Buffer.from(text, "utf8").subarray(0, maxBytes - 1).toString("utf8")}…`;
562
+ }
526
563
  /**
527
564
  * The routing write that opens the next tier-exhaustion generation, or null
528
565
  * when this re-dispatch is not one: the generation counter bumps by exactly