faberun 0.14.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "faberun",
3
- "version": "0.14.0",
3
+ "version": "0.16.0",
4
4
  "description": "Faberun is a development orchestration system that turns intent into verified software: harness- and model-agnostic, it keeps the intent, coordinates the work, verifies the result and decides what happens next.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -109,18 +109,29 @@ the contract is frozen with a digest, and the phone's middle ground is a note.
109
109
 
110
110
  ## Notify
111
111
 
112
- On `node.terminal`, `run.terminal` and `attention` the controller renders a
113
- one-line message from counters and identifiers only (node id, run id, state,
114
- attempt, error code, done/total never model text), calls the executable named
115
- by `FABERUN_NOTIFY_BIN` with that event as JSON on stdin, and appends a
116
- timestamped receipt (`delivered`, `failed`, `no_transport`) to
112
+ On `node.terminal`, `run.terminal` and `attention` the controller renders one
113
+ message from persisted state, in the operator's own language (detected from
114
+ the campaign goal, journal notes and node objectives; English otherwise):
115
+ line one is the outcome (`✅ <node> · done in 8m · $0.09`, `🏁 run 15 · <name>
116
+ · 2/2 done`, or `👀 <node> needs you · <error>`), then asked / done / proof
117
+ for a node, what every node delivered for a run, or why / asked / do for
118
+ attention, then a progress bar, a rule and the `🐦 faberun` signature with
119
+ campaign percent, cost and elapsed; an `⬆️` line names a newer release when
120
+ the cached update check has one. ≤2 KiB. It delivers that
121
+ same text to every bound transport at once, appending one receipt
122
+ (`delivered`, `failed`, `no_transport`, one entry per transport) to
117
123
  `<run-dir>/notify.jsonl`. Delivery is lossy: **exactly one attempt**, no retry,
118
- no backoff. Unset, nothing is spawned and the receipt is `no_transport`.
119
- `FABERUN_NOTIFY_BIN=os-macos` selects the bundled `osascript` adapter
120
- (`canWake: false`); any other value is an executable path. A resume never
121
- re-sends a notification already recorded for the same node, attempt and outcome.
122
- No transport is a default: `doctor`, `preflight` and the foreground launch warn
123
- when the variable is empty, and `--wake` reports no adapter can wake a session.
124
+ no backoff. `FABERUN_NOTIFY_BIN` names an executable called with the event as
125
+ JSON on stdin (`os-macos` selects the bundled `osascript` adapter); it pushes to
126
+ a person, `canWake: false`. `FABERUN_NOTIFY_SESSION=auto` wakes the harness
127
+ session the controller was launched from, `canWake: true`: the Claude Code
128
+ inbox socket and the Codex thread the environment names. A seat window sets it;
129
+ nothing else does, so a test suite never wakes a session. **On an inbound
130
+ `🐦 faberun` message**: `✅` or `🏁` with nothing waiting, answer in one line
131
+ and keep waiting; `👀`, act on the `do` command it names. A
132
+ resume never re-sends a notification already recorded for the same node, attempt
133
+ and outcome. No transport is a default: `doctor`, `preflight` and the foreground
134
+ launch warn when both variables are empty, and `--wake` names what will wake.
124
135
  Campaign-level lines are queued in `.runs/inbox.jsonl`, the managed block's
125
136
  append-only record — one object per line `{schemaVersion, eventId, at, type,
126
137
  campaignId, runId, nodeId, status, errorCode, dedupeKey, summary}`, deduped on
package/src/cli/plan.mjs CHANGED
@@ -11,6 +11,8 @@ import { classifyRunProgress } from "../campaign/chain.mjs";
11
11
  import { runProgress } from "../engine/supervise.mjs";
12
12
  import { DISCOVERY_RUNTIME_DEFINITIONS } from "../engine/runtime-discovery.mjs";
13
13
  import { validateRuntime } from "../contract/runtime.mjs";
14
+ import { validateFinalVerification, validateSharedVerification } from "../contract/final-verification.mjs";
15
+ import { colorLevel, statusToken } from "./brand.mjs";
14
16
  import { delay } from "../util.mjs";
15
17
  import { runPlanningPipeline } from "../plan/pipeline.mjs";
16
18
  import { runDirectory } from "../run/paths.mjs";
@@ -79,9 +81,40 @@ export function loadRuntimesCatalogue(path) {
79
81
  return runtimes;
80
82
  }
81
83
 
84
+ /**
85
+ * A `--verification <path>` catalogue: a JSON object carrying either or both
86
+ * of the contract's own suite keys, `sharedVerification` and
87
+ * `finalVerification`, each validated with the same validator
88
+ * `validateContract` applies. A key that is not a contract suite is refused
89
+ * rather than ignored: a typo'd key would freeze a contract that looks
90
+ * ratcheted and is not, which is the failure mode this flag exists to close.
91
+ *
92
+ * @param {string} path
93
+ * @returns {{sharedVerification?: import("../contract/index.mjs").VerificationCommand[], finalVerification?: import("../contract/index.mjs").VerificationCommand[]}}
94
+ */
95
+ export function loadVerificationSuites(path) {
96
+ const resolved = resolve(path);
97
+ /** @type {unknown} */
98
+ let raw;
99
+ try {
100
+ raw = JSON.parse(readFileSync(resolved, "utf8"));
101
+ } catch (error) {
102
+ throw new Error(`--verification ${path} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
103
+ }
104
+ if (!raw || typeof raw !== "object" || Array.isArray(raw)) throw new Error(`--verification ${path} must be a JSON object`);
105
+ const record = /** @type {Record<string, unknown>} */ (raw);
106
+ for (const key of Object.keys(record)) {
107
+ if (key !== "sharedVerification" && key !== "finalVerification") throw new Error(`--verification ${path} must carry only sharedVerification and finalVerification: ${key}`);
108
+ }
109
+ return {
110
+ ...(record.sharedVerification === undefined ? {} : { sharedVerification: validateSharedVerification(record.sharedVerification, "contract.sharedVerification") }),
111
+ ...(record.finalVerification === undefined ? {} : { finalVerification: validateFinalVerification(record.finalVerification, "contract.finalVerification") }),
112
+ };
113
+ }
114
+
82
115
  /**
83
116
  * @param {string} target
84
- * @param {{campaign?: string, phase?: string, "review-rounds"?: string, "approve-below"?: string, "runtime-defaults"?: string, runtimes?: string, detach?: boolean, json?: boolean}} values
117
+ * @param {{campaign?: string, phase?: string, "review-rounds"?: string, "approve-below"?: string, "runtime-defaults"?: string, runtimes?: string, verification?: string, detach?: boolean, json?: boolean}} values
85
118
  * @returns {Promise<void>}
86
119
  */
87
120
  export async function planCli(target, values) {
@@ -95,12 +128,16 @@ export async function planCli(target, values) {
95
128
  const runtimes = typeof values.runtimes === "string" && values.runtimes
96
129
  ? loadRuntimesCatalogue(values.runtimes)
97
130
  : DISCOVERY_RUNTIME_DEFINITIONS;
131
+ const verification = typeof values.verification === "string" && values.verification
132
+ ? loadVerificationSuites(values.verification)
133
+ : {};
98
134
 
99
135
  if (values.detach === true) {
100
136
  const argv = ["plan", specPath, "--campaign", campaignId, "--phase", phase, "--review-rounds", String(reviewRounds)];
101
137
  if (approveBelow !== undefined) argv.push("--approve-below", approveBelow);
102
138
  if (values["runtime-defaults"] !== undefined) argv.push("--runtime-defaults", values["runtime-defaults"]);
103
139
  if (typeof values.runtimes === "string" && values.runtimes) argv.push("--runtimes", resolve(values.runtimes));
140
+ if (typeof values.verification === "string" && values.verification) argv.push("--verification", resolve(values.verification));
104
141
  const child = detachArgv(argv);
105
142
  if (child.pid === undefined) throw new Error("detached plan has no pid");
106
143
  process.stdout.write(`[plan] detached · pid ${child.pid} · ${specPath}\n`);
@@ -115,6 +152,7 @@ export async function planCli(target, values) {
115
152
  approveBelow,
116
153
  runtimeDefaults,
117
154
  runtimes,
155
+ verification,
118
156
  launch: async (contractPath, contract) => {
119
157
  const child = detachSelf("run", contractPath);
120
158
  if (child.pid === undefined) throw new Error("detached planning run has no pid");
@@ -139,5 +177,6 @@ export async function planCli(target, values) {
139
177
  process.exitCode = 1;
140
178
  return;
141
179
  }
180
+ for (const warning of result.warnings) process.stdout.write(`${statusToken("warn", colorLevel(process.env, process.stdout.isTTY))} ${warning}\n`);
142
181
  process.stdout.write(`[plan] ${campaignId} phase ${phase} frozen · approved ${result.approved} · ${result.contractPath}\n`);
143
182
  }
package/src/cli.mjs CHANGED
@@ -122,6 +122,7 @@ export const COMMAND_OPTIONS = {
122
122
  "approve-below": { type: "string" },
123
123
  "runtime-defaults": { type: "string" },
124
124
  runtimes: { type: "string" },
125
+ verification: { type: "string" },
125
126
  detach: { type: "boolean" },
126
127
  json: { type: "boolean" },
127
128
  },
@@ -1,13 +1,15 @@
1
1
  /**
2
2
  * A declared runtime: its fields, the harness names it may name, whether its
3
- * permission mode can execute a command, and how a role resolves to one.
3
+ * permission mode can execute a command, and how a role resolves to one -- the
4
+ * latter including the strategies a routing rule may name, which are authored
5
+ * protocol surface a reader can reject by name just like a harness name.
4
6
  *
5
7
  * Split out because both the contract validator and the snapshot validator need
6
8
  * it -- a persisted `runtime` on a node snapshot is the shape the contract
7
9
  * declared -- and the snapshot validator should not import the contract
8
10
  * validator to reach it.
9
11
  */
10
- import { assertObject, nonNegativeNumber, positiveInteger, positiveNumber, rejectUnknown, requireId, requireString, requireStringArray } from "./assert.mjs";
12
+ import { assertObject, nonNegativeNumber, positiveInteger, positiveNumber, rejectUnknown, requireId, requireString, requireStringArray, requireTimestamp } from "./assert.mjs";
11
13
  import { composeAssignments } from "../engine/runtime-discovery.mjs";
12
14
  import { harnessCapabilities, resolvePermissionExecution, resolveVendor, validateCapabilityRequirements } from "../harnesses/index.mjs";
13
15
  import { stableJson } from "../util.mjs";
@@ -26,6 +28,19 @@ const RUNTIME_FIELDS = new Set([
26
28
  ]);
27
29
  const RUNTIME_HARNESSES = new Set(["claude", "codex", "agy", "dsh", "zcode", "exec-jsonl", "replay"]);
28
30
 
31
+ /**
32
+ * The strategies a routing rule may name: how a rule consumes its `prefer`
33
+ * list. `priority` takes the first admissible candidate; `cost` the lowest
34
+ * declared `costRank`; `reset-proximity` the least observed `remaining`
35
+ * allowance -- the window nearest its reset is spent first; `attempt-affinity`
36
+ * the previous attempt's runtime for the same node. A strategy whose datum a
37
+ * given runtime does not expose is inert for that runtime -- never a failure.
38
+ * An assignment records `declared` instead of any of these when an operator's
39
+ * runtime instruction prevailed over the table and every strategy.
40
+ * @typedef {"priority" | "cost" | "reset-proximity" | "attempt-affinity"} RoutingStrategy
41
+ */
42
+ export const ROUTING_STRATEGIES = Object.freeze(new Set(["priority", "cost", "reset-proximity", "attempt-affinity"]));
43
+
29
44
  /**
30
45
  * Harness-specific stall thresholds where the contract's single default is
31
46
  * wrong for every turn the harness runs. `zcode` has no streaming flag: its
@@ -54,6 +69,14 @@ const CAPABILITY_FIELDS = new Set([
54
69
  */
55
70
  export function routeRuntime(contract, node, role = "worker", event = {}) {
56
71
  if (role !== "worker" && role !== "judge") throw new TypeError("route role must be worker or judge");
72
+ // Catalogue records entering a routing decision are validated here, the one
73
+ // boundary every runtime-routing reader shares; the copies persisted on node
74
+ // snapshots are validated where they are written.
75
+ if (event.availability) {
76
+ for (const [id, availability] of Object.entries(event.availability)) {
77
+ validateRuntimeAvailability(availability, `routing availability ${id}`);
78
+ }
79
+ }
57
80
  const initialRuntimeId = role === "judge"
58
81
  ? node.gate.runtime ?? contract.runtimeDefaults?.judge
59
82
  : node.runtime ?? contract.runtimeDefaults?.worker;
@@ -149,6 +172,31 @@ function validatePricing(value, label) {
149
172
  if (rate < 0) throw new TypeError(`${label}.${key} must not be negative`);
150
173
  }
151
174
  }
175
+ /** The fields of one runtime-catalogue record (`RuntimeAvailability`). */
176
+ const AVAILABILITY_FIELDS = new Set(["available", "exhaustedUntil", "reason", "observedAt", "window", "remaining"]);
177
+
178
+ /**
179
+ * One runtime-catalogue record: what the harness de facto reported, and when.
180
+ * The three classified fields are required; the observables may be absent (a
181
+ * record that predates the field) or null (the harness exposes nothing -- never
182
+ * zero and never full allowance), but a present one must be typed: a record is
183
+ * persisted or read into a routing decision only through validators that can
184
+ * say what each datum means.
185
+ *
186
+ * @param {unknown} value
187
+ * @param {string} label
188
+ */
189
+ export function validateRuntimeAvailability(value, label) {
190
+ assertObject(value, label);
191
+ rejectUnknown(value, AVAILABILITY_FIELDS, label);
192
+ if (typeof value.available !== "boolean") throw new TypeError(`${label}.available must be boolean`);
193
+ if (value.exhaustedUntil !== undefined && value.exhaustedUntil !== null) requireTimestamp(value.exhaustedUntil, `${label}.exhaustedUntil`);
194
+ requireString(value.reason, `${label}.reason`);
195
+ if (value.observedAt !== undefined && value.observedAt !== null) requireTimestamp(value.observedAt, `${label}.observedAt`);
196
+ if (value.window !== undefined && value.window !== null) requireString(value.window, `${label}.window`);
197
+ if (value.remaining !== undefined && value.remaining !== null) nonNegativeNumber(value.remaining, `${label}.remaining`);
198
+ }
199
+
152
200
  /**
153
201
  * @param {Record<string, ValidatedRuntime>} runtimes
154
202
  * @param {string} runtimeId
@@ -21,12 +21,28 @@ export const VERIFICATION_LIMITS = Object.freeze({
21
21
  snapshotPathBytes: 1024,
22
22
  });
23
23
 
24
+ /**
25
+ * The declared risk tiers and the fraction of sampled mutants a suite must kill
26
+ * for a `mutation` verification entry to pass. The tier is what the entry
27
+ * declares; the fraction is not re-picked per entry, so two nodes at the same
28
+ * tier sit the same bar. The wall-clock budget that bounds how many mutants run
29
+ * at all is a measurement, not policy, and lives with the runner that enforces
30
+ * it (`MUTATION_TIME_BUDGET_MS`, `src/engine/mutation.mjs`).
31
+ */
32
+ export const MUTATION_TIERS = Object.freeze({
33
+ high: 1,
34
+ medium: 0.75,
35
+ low: 0.5,
36
+ });
37
+
38
+ /** @typedef {keyof typeof MUTATION_TIERS} MutationTier */
39
+
24
40
  /** @typedef {"active"|"closed"|"failed"|"crashed"|"canceled"} VerificationAttemptStatus */
25
41
 
26
42
  /**
27
43
  * One declared deterministic check: an argv command run by the controller.
28
44
  *
29
- * @typedef {{argv: string[], cwd?: string, timeoutSec?: number, repeat?: number, env?: string[], mutation?: {threshold: number}}} VerificationCommand
45
+ * @typedef {{argv: string[], cwd?: string, timeoutSec?: number, repeat?: number, env?: string[], mutation?: {tier: MutationTier}}} VerificationCommand
30
46
  */
31
47
 
32
48
  /**
@@ -126,20 +142,20 @@ function validateVerificationCommand(command, label = "verification command") {
126
142
  const envBytes = env.reduce((sum, name) => sum + Buffer.byteLength(/** @type {string} */ (name), "utf8"), 0);
127
143
  if (envBytes > VERIFICATION_LIMITS.maxEnvBytes) throw new TypeError(`${label}.env exceeds aggregate byte limit`);
128
144
  // Mutation testing is opt-in per entry: it re-runs the same argv against
129
- // deliberately broken copies of the node's written files. `threshold` is the
130
- // fraction of mutants the suite must kill, so 0 accepts any suite and 1
131
- // demands every sampled mutant fail it.
132
- /** @type {{threshold: number}|undefined} */
145
+ // deliberately broken copies of the node's written files. The entry declares
146
+ // its risk tier; `MUTATION_TIERS` fixes the kill fraction each tier demands,
147
+ // so the bar is a property of the tier, not of the author's caution.
148
+ /** @type {{tier: MutationTier}|undefined} */
133
149
  let mutation;
134
150
  if (record.mutation !== undefined) {
135
151
  const rawMutation = record.mutation;
136
- if (!rawMutation || typeof rawMutation !== "object" || Array.isArray(rawMutation)) throw new TypeError(`${label}.mutation must be an object with a threshold between 0 and 1`);
152
+ if (!rawMutation || typeof rawMutation !== "object" || Array.isArray(rawMutation)) throw new TypeError(`${label}.mutation must be an object with a declared risk tier`);
137
153
  const mutationRecord = /** @type {Record<string, unknown>} */ (rawMutation);
138
- for (const key of Object.keys(mutationRecord)) if (key !== "threshold") throw new TypeError(`${label}.mutation has unexpected field ${key}`);
139
- if (typeof mutationRecord.threshold !== "number" || !Number.isFinite(mutationRecord.threshold) || mutationRecord.threshold < 0 || mutationRecord.threshold > 1) {
140
- throw new TypeError(`${label}.mutation.threshold must be a number between 0 and 1`);
154
+ for (const key of Object.keys(mutationRecord)) if (key !== "tier") throw new TypeError(`${label}.mutation has unexpected field ${key}`);
155
+ if (typeof mutationRecord.tier !== "string" || !Object.hasOwn(MUTATION_TIERS, mutationRecord.tier)) {
156
+ throw new TypeError(`${label}.mutation.tier must be one of ${Object.keys(MUTATION_TIERS).join(", ")}`);
141
157
  }
142
- mutation = { threshold: mutationRecord.threshold };
158
+ mutation = { tier: /** @type {MutationTier} */ (mutationRecord.tier) };
143
159
  }
144
160
  /** @type {VerificationCommand} */
145
161
  const normalized = { argv: [.../** @type {string[]} */ (record.argv)], timeoutSec, repeat, env: [.../** @type {string[]} */ (env)] };
@@ -20,9 +20,17 @@ import { transition } from "./state.mjs";
20
20
  /**
21
21
  * Resolve role assignments once at run creation. Discovery is used only for
22
22
  * omitted roles; the resulting pair is persisted so resume is deterministic.
23
+ * Alongside it, `decisions` records, for every assignment, the strategy that
24
+ * was applied and the reason for the choice: `declared` with the declaring
25
+ * field when the contract named the runtime -- an operator instruction
26
+ * prevails over every strategy -- or the discovery ranking that composed an
27
+ * omitted role. The record lives at this boundary, not on the persisted
28
+ * assignment entries, because the snapshot's routing.allowlist still carries
29
+ * the classified fields only (the same declared lag as the catalogue
30
+ * observables); it widens when a reader needs the record durably.
23
31
  *
24
32
  * @param {ValidatedContract} contract
25
- * @returns {Promise<{assignments: Record<string, {worker: string, judge: string, composedWorker: boolean, composedJudge: boolean}>, availability: Record<string, import("./runtime-discovery.mjs").RuntimeAvailability>}>}
33
+ * @returns {Promise<{assignments: Record<string, {worker: string, judge: string, composedWorker: boolean, composedJudge: boolean}>, decisions: Record<string, {worker: {strategy: string|null, reason: string}, judge: {strategy: string|null, reason: string}}>, availability: Record<string, import("./runtime-discovery.mjs").RuntimeAvailability>}>}
26
34
  */
27
35
  export async function runtimeAssignments(contract) {
28
36
  const needsComposition = contract.nodes.some((node) =>
@@ -31,15 +39,34 @@ export async function runtimeAssignments(contract) {
31
39
  const availability = needsComposition ? await discoverRuntimes(contract.runtimes, { cwd: contract.cwd }) : {};
32
40
  const config = readUserConfig(process.env);
33
41
  const assignments = composeAssignments(contract, availability, { config });
42
+ /** @type {Record<string, {worker: {strategy: string|null, reason: string}, judge: {strategy: string|null, reason: string}}>} */
43
+ const decisions = {};
34
44
  return {
35
45
  assignments: Object.fromEntries(Object.entries(assignments).map(([nodeId, assignment]) => {
36
46
  const node = contract.nodes.find((candidate) => candidate.id === nodeId);
37
- return [nodeId, {
38
- ...assignment,
39
- composedWorker: node?.runtime === undefined && contract.runtimeDefaults?.worker === undefined,
40
- composedJudge: Boolean(node?.gate.enabled && node.gate.runtime === undefined && contract.runtimeDefaults?.judge === undefined),
41
- }];
47
+ const composedWorker = node?.runtime === undefined && contract.runtimeDefaults?.worker === undefined;
48
+ const composedJudge = Boolean(node?.gate.enabled && node.gate.runtime === undefined && contract.runtimeDefaults?.judge === undefined);
49
+ // A judge the gate never asks for is no choice at all: no strategy
50
+ // decided it, so none is recorded.
51
+ const judgeSource = node?.gate.enabled && node.gate.runtime !== undefined
52
+ ? "gate runtime"
53
+ : contract.runtimeDefaults?.judge !== undefined ? "runtimeDefaults.judge" : null;
54
+ const workerSource = node?.runtime !== undefined
55
+ ? "node runtime"
56
+ : contract.runtimeDefaults?.worker !== undefined ? "runtimeDefaults.worker" : null;
57
+ decisions[nodeId] = {
58
+ worker: {
59
+ strategy: composedWorker ? "cost" : workerSource !== null ? "declared" : null,
60
+ reason: composedWorker ? "discovery: cheapest available runtime" : /** @type {string} */ (workerSource),
61
+ },
62
+ judge: {
63
+ strategy: composedJudge ? "priority" : judgeSource !== null ? "declared" : null,
64
+ reason: composedJudge ? "discovery: strongest available runtime" : /** @type {string} */ (judgeSource ?? "no judge required"),
65
+ },
66
+ };
67
+ return [nodeId, { ...assignment, composedWorker, composedJudge }];
42
68
  })),
69
+ decisions,
43
70
  availability,
44
71
  };
45
72
  }
@@ -476,6 +476,12 @@ const MAX_ROUTING_HISTORY = 64;
476
476
  /**
477
477
  * The override reason recorded on the node, in the operator's words.
478
478
  *
479
+ * The reason also names the attempt-affinity outcome, because the override is
480
+ * the role's working assignment and the record of why it names the runtime it
481
+ * names: a reset holds affinity (the previous attempt's runtime stays warm
482
+ * for the retry), an edge yields it (the previous runtime is the one that
483
+ * just failed, and the reason quotes its code).
484
+ *
479
485
  * @param {Transition} schedule
480
486
  * @param {"worker"|"judge"} role
481
487
  * @param {string} current
@@ -484,10 +490,10 @@ const MAX_ROUTING_HISTORY = 64;
484
490
  */
485
491
  function routeReason(schedule, role, current, error) {
486
492
  if (schedule.kind === "reset" && schedule.reason === "quota_reset") {
487
- return `${role} provider ${current} quota resets at ${schedule.at}: ${error.message}`;
493
+ return `${role} provider ${current} quota resets at ${schedule.at}: ${error.message}; attempt-affinity held: ${current} keeps the node's context for the retry`;
488
494
  }
489
- if (schedule.kind === "reset") return `${role} provider ${current} hit a transient network failure, retrying at ${schedule.at}: ${error.message}`;
490
- if (schedule.reason === "network_backoff") return `${role} provider ${current} kept failing on the network: ${error.message}`;
491
- if (schedule.reason === "protocol_failure") return `${role} provider ${current} could not hold the result protocol: ${error.message}`;
492
- return `${role} provider ${current} exhausted: ${error.message}`;
495
+ if (schedule.kind === "reset") return `${role} provider ${current} hit a transient network failure, retrying at ${schedule.at}: ${error.message}; attempt-affinity held: ${current} keeps the node's context for the retry`;
496
+ if (schedule.reason === "network_backoff") return `${role} provider ${current} kept failing on the network: ${error.message}; attempt-affinity yielded: ${current} reported ${error.code}`;
497
+ if (schedule.reason === "protocol_failure") return `${role} provider ${current} could not hold the result protocol: ${error.message}; attempt-affinity yielded: ${current} reported ${error.code}`;
498
+ return `${role} provider ${current} exhausted: ${error.message}; attempt-affinity yielded: ${current} reported ${error.code}`;
493
499
  }
@@ -17,7 +17,7 @@ import { terminateInvocation } from "./process.mjs";
17
17
  import { join, resolve } from "node:path";
18
18
  import { readFileSync } from "node:fs";
19
19
  import { readRunNodes } from "./scheduler.mjs";
20
- import { deleteRef, releaseAttemptWorktree, runRefName } from "../repo/worktree.mjs";
20
+ import { createPreservedRef, deleteRef, releaseAttemptWorktree, runRefName } from "../repo/worktree.mjs";
21
21
  import { syncAgentSignal } from "../repo/signal.mjs";
22
22
  import { transition, writeNode } from "./state.mjs";
23
23
  import { validateContract } from "../contract/index.mjs";
@@ -26,10 +26,11 @@ import { writeJsonAtomic } from "../run/store.mjs";
26
26
  /** @typedef {import("./process.mjs").InvocationProbe} InvocationProbe */
27
27
  /** @typedef {import("../cli.mjs").LockHandle} LockHandle */
28
28
  /** @typedef {import("../run/lock.mjs").LockRecord} LockRecord */
29
+ /** @typedef {{preservedRefs: string[], released: string[]}} CancelResult */
29
30
 
30
31
  /**
31
32
  * @param {string} runDirPath
32
- * @returns {Promise<boolean>}
33
+ * @returns {Promise<CancelResult>}
33
34
  */
34
35
  export async function cancelRun(runDirPath) {
35
36
  const runDir = resolve(runDirPath);
@@ -108,20 +109,33 @@ export async function cancelRun(runDirPath) {
108
109
  throw error;
109
110
  }
110
111
  if (!await waitForTerminal(runDir, 1_000)) throw new Error("cancel could not confirm a terminal run state");
112
+ // Preserved refs come first, before anything is released: a cancel that
113
+ // dies part-way must leave more work reachable, never less. If a creation
114
+ // fails here, nothing below has run and every integrated commit is as
115
+ // reachable as cancel found it. A node whose integratedHead is null was
116
+ // never integrated and gets none.
117
+ const preservedRefs = states
118
+ .filter((state) => state.integratedHead)
119
+ .map((state) => createPreservedRef(contract.cwd, contract.id, state.id, state.integratedHead));
111
120
  // The run directory is evidence a campaign ledger may still want, so it
112
121
  // stays; the run ref and every node's attempt branch are just git names
113
122
  // the next launch of this same contract id needs back, and cancel is the
114
123
  // operator saying this run is over. Releasing a name is not destroying a
115
- // record -- each state's `worktree.branch`/`commit` fields, and the sha
116
- // this ref pointed at, remain in the persisted snapshot regardless.
124
+ // record: the sha remains in the persisted snapshot, and the commit it
125
+ // names stays reachable through the preserved ref created above.
117
126
  // Idempotent both ways: `removeWorktree` and `deleteRef` already tolerate
118
127
  // an artefact a previous cancel (or the run itself) already released.
128
+ const released = [];
119
129
  for (const state of states) {
120
- if (state.worktree?.branch) releaseAttemptWorktree(contract.cwd, state.worktree.path, state.worktree.branch);
130
+ if (!state.worktree?.branch) continue;
131
+ releaseAttemptWorktree(contract.cwd, state.worktree.path, state.worktree.branch);
132
+ released.push(`refs/heads/${state.worktree.branch}`);
133
+ if (state.worktree.path) released.push(state.worktree.path);
121
134
  }
122
135
  deleteRef(contract.cwd, runRefName(contract.id));
136
+ released.push(runRefName(contract.id));
123
137
  syncAgentSignal(join(runDir, ".."));
124
- return true;
138
+ return { preservedRefs, released };
125
139
  } finally {
126
140
  controllerLock.release();
127
141
  }
@@ -8,9 +8,14 @@
8
8
  * runtime a role started on, and (if declared) the runtime its `fallback`
9
9
  * names. contract.mjs validates the field is never a self-loop; because a
10
10
  * hop is bounded at one, a multi-runtime cycle is structurally impossible.
11
+ *
12
+ * Per-attempt resolution (`routeRuntimeForState`) lives here too, because
13
+ * attempt affinity is read against the same failover facts: the runtime the
14
+ * previous attempt ran is preferred until the hop, the catalogue, or the
15
+ * judge's vendor rule disqualifies it.
11
16
  */
12
17
  import { harnessCapabilities } from "../harnesses/index.mjs";
13
- import { nextSameTierRuntime } from "./runtime-discovery.mjs";
18
+ import { isRuntimeAvailable, nextSameTierRuntime } from "./runtime-discovery.mjs";
14
19
  import { routeRuntime } from "../contract/runtime.mjs";
15
20
 
16
21
  /** @typedef {import("../contract/index.mjs").ValidatedNode} ValidatedNode */
@@ -170,6 +175,50 @@ export function routingBackoffActive(state, phase) {
170
175
  return Boolean(override?.role === phase && override.backoffUntil && Date.parse(override.backoffUntil) > Date.now());
171
176
  }
172
177
 
178
+ /**
179
+ * The runtime id this role's previous attempt on this node ran on, read off
180
+ * the durable invocation record. The invocations are the one record that
181
+ * survives both a routed hop and the revision boundary: `resetPhaseRouting`
182
+ * clears the routing override between revisions, but never the invocations.
183
+ *
184
+ * @param {NodeSnapshot} state
185
+ * @param {"worker"|"judge"} role
186
+ * @returns {string|undefined}
187
+ */
188
+ function previousAttemptRuntimeId(state, role) {
189
+ const found = [...(state.invocations ?? [])].reverse().find((invocation) => invocation.phase === role)?.runtimeId;
190
+ return typeof found === "string" ? found : undefined;
191
+ }
192
+
193
+ /**
194
+ * May attempt affinity keep this role on `candidate` -- the runtime its
195
+ * previous attempt on this node ran? Affinity yields to a known exhaustion
196
+ * only: a catalogue record that `isRuntimeAvailable` -- the one home of the
197
+ * exhaustion and staleness rule -- refuses to admit. An absent record is not
198
+ * that: the snapshot's catalogue copy is the classified fields at best and
199
+ * often empty outright, and the strongest evidence of health here is the
200
+ * attempt that just ran on this runtime, so absence of a record must not read
201
+ * as exhaustion any more than R12 lets it read as rested. A judge candidate
202
+ * stays bound by the cross-vendor rule its assignment was composed under,
203
+ * compared against the worker that actually ran the node, exactly as
204
+ * `nextSameTierRuntime` compares its own candidates.
205
+ *
206
+ * @param {ValidatedContract} contract
207
+ * @param {NodeSnapshot} state
208
+ * @param {"worker"|"judge"} role
209
+ * @param {string} candidate
210
+ * @returns {boolean}
211
+ */
212
+ function admitsAffinity(contract, state, role, candidate) {
213
+ const runtime = contract.runtimes[candidate];
214
+ if (!runtime) return false;
215
+ const availability = state.routing?.availability?.[candidate];
216
+ if (availability && !isRuntimeAvailable(availability)) return false;
217
+ if (role !== "judge") return true;
218
+ const workerId = previousAttemptRuntimeId(state, "worker") ?? state.routing?.assignments?.worker;
219
+ return runtime.vendor !== (workerId ? contract.runtimes[workerId]?.vendor : null);
220
+ }
221
+
173
222
  /**
174
223
  * @param {ValidatedContract} contract
175
224
  * @param {ValidatedNode} node
@@ -183,6 +232,18 @@ export function routeRuntimeForState(contract, node, state, role) {
183
232
  const runtime = contract.runtimes[override.runtime];
184
233
  return { id: override.runtime, ...runtime, capabilities: harnessCapabilities(runtime) };
185
234
  }
235
+ // Attempt affinity: successive attempts and revisions of this node prefer
236
+ // the runtime the previous attempt ran, because it is the one holding the
237
+ // node's context. It ranks ahead of the frozen assignment -- the assignment
238
+ // decided the first attempt, the attempt that ran since decides the next --
239
+ // and below the role-matched override above, which is itself already an
240
+ // affinity outcome: a reset hold on the warm runtime, or the failover edge
241
+ // affinity yielded to.
242
+ const previous = previousAttemptRuntimeId(state, role);
243
+ if (previous !== undefined && admitsAffinity(contract, state, role, previous)) {
244
+ const runtime = contract.runtimes[previous];
245
+ return { id: previous, ...runtime, capabilities: harnessCapabilities(runtime) };
246
+ }
186
247
  const assigned = state.routing?.assignments?.[role];
187
248
  if (assigned && contract.runtimes[assigned]) {
188
249
  const runtime = contract.runtimes[assigned];
@@ -14,8 +14,10 @@ import { stat } from "node:fs/promises";
14
14
  import { isAbsolute, resolve } from "node:path";
15
15
  import { reviewMode, UNCITED_REJECTION_REASON } from "../contract/review-modes.mjs";
16
16
  import { JUDGE_LIMITS } from "../contract/judge-envelope.mjs";
17
+ import { sharedVerificationCommands } from "../contract/final-verification.mjs";
17
18
 
18
19
  /** @typedef {import("../contract/definition-of-done.mjs").DefinitionOfDoneItem} DefinitionOfDoneItem */
20
+ /** @typedef {import("../contract/verification.mjs").VerificationCommand} VerificationCommand */
19
21
  /** @typedef {import("../contract/definition-of-done.mjs").DefinitionOfDoneProof} DefinitionOfDoneProof */
20
22
  /** @typedef {import("../contract/index.mjs").ExecutionOverride} ExecutionOverride */
21
23
  /** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
@@ -349,6 +351,48 @@ function declaredWriteCoverage(state) {
349
351
  directoryRoots.some((root) => path === root || path.startsWith(`${root}/`));
350
352
  }
351
353
 
354
+ /**
355
+ * Whether a path named by verification output belongs to the contract's own
356
+ * `sharedVerification` suite -- the repository ratchets the operator appends to
357
+ * every node -- rather than to the node's own work. An argv entry names either
358
+ * the file the runner was given or a directory it walked.
359
+ *
360
+ * Deliberately not caught: a `sharedVerification` that reaches the ratchet
361
+ * without naming it in argv (`npm test`, or any script that picks the files
362
+ * itself). No token matches, the file is not recognized as a ratchet, and the
363
+ * contract-defect advice applies to it as it did before. Widening the match to
364
+ * guess what a script runs would be worse than the gap it closes.
365
+ *
366
+ * @param {{sharedVerification?: VerificationCommand[]}} contract
367
+ * @returns {(path: string) => boolean}
368
+ */
369
+ function sharedVerificationCoverage(contract) {
370
+ const argv = sharedVerificationCommands(contract).flatMap((command) => command.argv);
371
+ return (path) => argv.some((token) => token === path || path.startsWith(`${token}/`));
372
+ }
373
+
374
+ /**
375
+ * The operator-facing description for a failing ratchet: a check the contract
376
+ * itself declared, that runs on every node, and that this node's change broke.
377
+ *
378
+ * It says nothing about the write scope on purpose. Telling the operator to
379
+ * hand the node the ratchet licenses the next worker to edit the rule it just
380
+ * violated -- observed 2026-09-21, when a node that pushed
381
+ * `src/report/render.mjs` to 801 lines was advised to add the 800-line ratchet
382
+ * to its `writeFiles`. The remedy is already inside the node's scope: its own
383
+ * code.
384
+ *
385
+ * @param {string[]} paths
386
+ * @returns {string}
387
+ */
388
+ function ratchetFailureDescription(paths) {
389
+ const files = paths.join(", ");
390
+ return boundedText(
391
+ `deterministic verification failed in ${files}, which the contract runs on every node as sharedVerification: this node's change broke a repository-wide rule. The remedy is in the node's own code -- bring the change back within the rule the check enforces. The check itself is not the node's to change, and relaxing it is not a fix`,
392
+ JUDGE_LIMITS.descriptionBytes,
393
+ );
394
+ }
395
+
352
396
  /**
353
397
  * The operator-facing description for a failure that named a test outside the
354
398
  * declared write scope. The defect is not in the worker's code: the worker is
@@ -372,24 +416,38 @@ function undeclaredTestDescription(paths) {
372
416
  * The deterministic controller-verification failure verdict, kept next to the
373
417
  * Definition of Done gate so every deterministic failure settles identically.
374
418
  *
419
+ * The contract is a parameter because an undeclared test file has two opposite
420
+ * remedies and only the contract tells them apart: a ratchet it declared in
421
+ * `sharedVerification` is the node's code to fix, while any other withheld test
422
+ * is the contract's scope to widen.
423
+ *
424
+ * @param {{sharedVerification?: VerificationCommand[]}} contract
375
425
  * @param {{verification?: {commands?: Array<{argv: string[], passed?: boolean, attempts?: Array<{stdout?: string, stderr?: string, exitCode?: number|null, timedOut?: boolean}>}>, error?: unknown}|null, scope?: {boundary?: {files?: string[], roots?: string[], fileRoots?: string[]}}|null}} state
376
426
  * @returns {import("./prompts.mjs").JudgeVerdict}
377
427
  */
378
- export function verificationFailureVerdict(state) {
428
+ export function verificationFailureVerdict(contract, state) {
379
429
  const failedCommands = (state.verification?.commands ?? []).filter((command) => !command.passed);
380
430
  const evidence = failedCommands.length
381
431
  ? failedCommands.map((command) => `${command.argv.join(" ")}: ${(command.attempts ?? []).map((attempt) => `exit=${attempt.exitCode ?? "-"}${attempt.timedOut ? " timeout" : ""}`).join(", ")}`).join("; ")
382
432
  : state.verification?.error ?? "verification controller failed to execute a command";
383
- const undeclared = namedTestFiles(failedCommands).filter((path) => !declaredWriteCoverage(state)(path));
433
+ const declared = declaredWriteCoverage(state);
434
+ const undeclared = namedTestFiles(failedCommands).filter((path) => !declared(path));
435
+ const isRatchet = sharedVerificationCoverage(contract);
436
+ const ratchets = undeclared.filter(isRatchet);
437
+ const withheld = undeclared.filter((path) => !isRatchet(path));
438
+ const descriptions = [
439
+ ...(ratchets.length ? [ratchetFailureDescription(ratchets)] : []),
440
+ ...(withheld.length ? [undeclaredTestDescription(withheld)] : []),
441
+ ];
384
442
  return {
385
443
  verdict: "fail",
386
444
  maxSeverity: "critical",
387
445
  summary: "deterministic verification failed",
388
- findings: [{
446
+ findings: (descriptions.length ? descriptions : ["deterministic verification failed"]).map((description) => ({
389
447
  severity: "critical",
390
- description: undeclared.length ? undeclaredTestDescription(undeclared) : "deterministic verification failed",
448
+ description,
391
449
  evidence: boundedText(evidence),
392
- }],
450
+ })),
393
451
  };
394
452
  }
395
453