faberun 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/plan.mjs +40 -1
- package/src/cli.mjs +1 -0
- package/src/contract/runtime.mjs +50 -2
- package/src/contract/verification.mjs +26 -10
- package/src/engine/assignment.mjs +33 -6
- package/src/engine/backoff.mjs +11 -5
- package/src/engine/failover.mjs +62 -1
- package/src/engine/judge-gate.mjs +63 -5
- package/src/engine/lifecycle.mjs +17 -4
- package/src/engine/mutation.mjs +47 -7
- package/src/engine/resume.mjs +1 -1
- package/src/engine/runtime-discovery.mjs +44 -7
- package/src/engine/scheduler.mjs +19 -23
- package/src/engine/scope.mjs +108 -6
- package/src/engine/settle.mjs +19 -8
- package/src/plan/pipeline.mjs +203 -22
- package/src/plan/routing.mjs +151 -33
- package/src/plan/sizing.mjs +33 -0
- package/src/report/final.mjs +2 -1
- package/src/report/packet-repetition.mjs +121 -0
- package/src/report/render.mjs +3 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "faberun",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.15.0",
|
|
4
4
|
"description": "Faberun is a development orchestration system that turns intent into verified software: harness- and model-agnostic, it keeps the intent, coordinates the work, verifies the result and decides what happens next.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/src/cli/plan.mjs
CHANGED
|
@@ -11,6 +11,8 @@ import { classifyRunProgress } from "../campaign/chain.mjs";
|
|
|
11
11
|
import { runProgress } from "../engine/supervise.mjs";
|
|
12
12
|
import { DISCOVERY_RUNTIME_DEFINITIONS } from "../engine/runtime-discovery.mjs";
|
|
13
13
|
import { validateRuntime } from "../contract/runtime.mjs";
|
|
14
|
+
import { validateFinalVerification, validateSharedVerification } from "../contract/final-verification.mjs";
|
|
15
|
+
import { colorLevel, statusToken } from "./brand.mjs";
|
|
14
16
|
import { delay } from "../util.mjs";
|
|
15
17
|
import { runPlanningPipeline } from "../plan/pipeline.mjs";
|
|
16
18
|
import { runDirectory } from "../run/paths.mjs";
|
|
@@ -79,9 +81,40 @@ export function loadRuntimesCatalogue(path) {
|
|
|
79
81
|
return runtimes;
|
|
80
82
|
}
|
|
81
83
|
|
|
84
|
+
/**
|
|
85
|
+
* A `--verification <path>` catalogue: a JSON object carrying either or both
|
|
86
|
+
* of the contract's own suite keys, `sharedVerification` and
|
|
87
|
+
* `finalVerification`, each validated with the same validator
|
|
88
|
+
* `validateContract` applies. A key that is not a contract suite is refused
|
|
89
|
+
* rather than ignored: a typo'd key would freeze a contract that looks
|
|
90
|
+
* ratcheted and is not, which is the failure mode this flag exists to close.
|
|
91
|
+
*
|
|
92
|
+
* @param {string} path
|
|
93
|
+
* @returns {{sharedVerification?: import("../contract/index.mjs").VerificationCommand[], finalVerification?: import("../contract/index.mjs").VerificationCommand[]}}
|
|
94
|
+
*/
|
|
95
|
+
export function loadVerificationSuites(path) {
|
|
96
|
+
const resolved = resolve(path);
|
|
97
|
+
/** @type {unknown} */
|
|
98
|
+
let raw;
|
|
99
|
+
try {
|
|
100
|
+
raw = JSON.parse(readFileSync(resolved, "utf8"));
|
|
101
|
+
} catch (error) {
|
|
102
|
+
throw new Error(`--verification ${path} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
|
|
103
|
+
}
|
|
104
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) throw new Error(`--verification ${path} must be a JSON object`);
|
|
105
|
+
const record = /** @type {Record<string, unknown>} */ (raw);
|
|
106
|
+
for (const key of Object.keys(record)) {
|
|
107
|
+
if (key !== "sharedVerification" && key !== "finalVerification") throw new Error(`--verification ${path} must carry only sharedVerification and finalVerification: ${key}`);
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
...(record.sharedVerification === undefined ? {} : { sharedVerification: validateSharedVerification(record.sharedVerification, "contract.sharedVerification") }),
|
|
111
|
+
...(record.finalVerification === undefined ? {} : { finalVerification: validateFinalVerification(record.finalVerification, "contract.finalVerification") }),
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
82
115
|
/**
|
|
83
116
|
* @param {string} target
|
|
84
|
-
* @param {{campaign?: string, phase?: string, "review-rounds"?: string, "approve-below"?: string, "runtime-defaults"?: string, runtimes?: string, detach?: boolean, json?: boolean}} values
|
|
117
|
+
* @param {{campaign?: string, phase?: string, "review-rounds"?: string, "approve-below"?: string, "runtime-defaults"?: string, runtimes?: string, verification?: string, detach?: boolean, json?: boolean}} values
|
|
85
118
|
* @returns {Promise<void>}
|
|
86
119
|
*/
|
|
87
120
|
export async function planCli(target, values) {
|
|
@@ -95,12 +128,16 @@ export async function planCli(target, values) {
|
|
|
95
128
|
const runtimes = typeof values.runtimes === "string" && values.runtimes
|
|
96
129
|
? loadRuntimesCatalogue(values.runtimes)
|
|
97
130
|
: DISCOVERY_RUNTIME_DEFINITIONS;
|
|
131
|
+
const verification = typeof values.verification === "string" && values.verification
|
|
132
|
+
? loadVerificationSuites(values.verification)
|
|
133
|
+
: {};
|
|
98
134
|
|
|
99
135
|
if (values.detach === true) {
|
|
100
136
|
const argv = ["plan", specPath, "--campaign", campaignId, "--phase", phase, "--review-rounds", String(reviewRounds)];
|
|
101
137
|
if (approveBelow !== undefined) argv.push("--approve-below", approveBelow);
|
|
102
138
|
if (values["runtime-defaults"] !== undefined) argv.push("--runtime-defaults", values["runtime-defaults"]);
|
|
103
139
|
if (typeof values.runtimes === "string" && values.runtimes) argv.push("--runtimes", resolve(values.runtimes));
|
|
140
|
+
if (typeof values.verification === "string" && values.verification) argv.push("--verification", resolve(values.verification));
|
|
104
141
|
const child = detachArgv(argv);
|
|
105
142
|
if (child.pid === undefined) throw new Error("detached plan has no pid");
|
|
106
143
|
process.stdout.write(`[plan] detached · pid ${child.pid} · ${specPath}\n`);
|
|
@@ -115,6 +152,7 @@ export async function planCli(target, values) {
|
|
|
115
152
|
approveBelow,
|
|
116
153
|
runtimeDefaults,
|
|
117
154
|
runtimes,
|
|
155
|
+
verification,
|
|
118
156
|
launch: async (contractPath, contract) => {
|
|
119
157
|
const child = detachSelf("run", contractPath);
|
|
120
158
|
if (child.pid === undefined) throw new Error("detached planning run has no pid");
|
|
@@ -139,5 +177,6 @@ export async function planCli(target, values) {
|
|
|
139
177
|
process.exitCode = 1;
|
|
140
178
|
return;
|
|
141
179
|
}
|
|
180
|
+
for (const warning of result.warnings) process.stdout.write(`${statusToken("warn", colorLevel(process.env, process.stdout.isTTY))} ${warning}\n`);
|
|
142
181
|
process.stdout.write(`[plan] ${campaignId} phase ${phase} frozen · approved ${result.approved} · ${result.contractPath}\n`);
|
|
143
182
|
}
|
package/src/cli.mjs
CHANGED
|
@@ -122,6 +122,7 @@ export const COMMAND_OPTIONS = {
|
|
|
122
122
|
"approve-below": { type: "string" },
|
|
123
123
|
"runtime-defaults": { type: "string" },
|
|
124
124
|
runtimes: { type: "string" },
|
|
125
|
+
verification: { type: "string" },
|
|
125
126
|
detach: { type: "boolean" },
|
|
126
127
|
json: { type: "boolean" },
|
|
127
128
|
},
|
package/src/contract/runtime.mjs
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* A declared runtime: its fields, the harness names it may name, whether its
|
|
3
|
-
* permission mode can execute a command, and how a role resolves to one
|
|
3
|
+
* permission mode can execute a command, and how a role resolves to one -- the
|
|
4
|
+
* latter including the strategies a routing rule may name, which are authored
|
|
5
|
+
* protocol surface a reader can reject by name just like a harness name.
|
|
4
6
|
*
|
|
5
7
|
* Split out because both the contract validator and the snapshot validator need
|
|
6
8
|
* it -- a persisted `runtime` on a node snapshot is the shape the contract
|
|
7
9
|
* declared -- and the snapshot validator should not import the contract
|
|
8
10
|
* validator to reach it.
|
|
9
11
|
*/
|
|
10
|
-
import { assertObject, nonNegativeNumber, positiveInteger, positiveNumber, rejectUnknown, requireId, requireString, requireStringArray } from "./assert.mjs";
|
|
12
|
+
import { assertObject, nonNegativeNumber, positiveInteger, positiveNumber, rejectUnknown, requireId, requireString, requireStringArray, requireTimestamp } from "./assert.mjs";
|
|
11
13
|
import { composeAssignments } from "../engine/runtime-discovery.mjs";
|
|
12
14
|
import { harnessCapabilities, resolvePermissionExecution, resolveVendor, validateCapabilityRequirements } from "../harnesses/index.mjs";
|
|
13
15
|
import { stableJson } from "../util.mjs";
|
|
@@ -26,6 +28,19 @@ const RUNTIME_FIELDS = new Set([
|
|
|
26
28
|
]);
|
|
27
29
|
const RUNTIME_HARNESSES = new Set(["claude", "codex", "agy", "dsh", "zcode", "exec-jsonl", "replay"]);
|
|
28
30
|
|
|
31
|
+
/**
|
|
32
|
+
* The strategies a routing rule may name: how a rule consumes its `prefer`
|
|
33
|
+
* list. `priority` takes the first admissible candidate; `cost` the lowest
|
|
34
|
+
* declared `costRank`; `reset-proximity` the least observed `remaining`
|
|
35
|
+
* allowance -- the window nearest its reset is spent first; `attempt-affinity`
|
|
36
|
+
* the previous attempt's runtime for the same node. A strategy whose datum a
|
|
37
|
+
* given runtime does not expose is inert for that runtime -- never a failure.
|
|
38
|
+
* An assignment records `declared` instead of any of these when an operator's
|
|
39
|
+
* runtime instruction prevailed over the table and every strategy.
|
|
40
|
+
* @typedef {"priority" | "cost" | "reset-proximity" | "attempt-affinity"} RoutingStrategy
|
|
41
|
+
*/
|
|
42
|
+
export const ROUTING_STRATEGIES = Object.freeze(new Set(["priority", "cost", "reset-proximity", "attempt-affinity"]));
|
|
43
|
+
|
|
29
44
|
/**
|
|
30
45
|
* Harness-specific stall thresholds where the contract's single default is
|
|
31
46
|
* wrong for every turn the harness runs. `zcode` has no streaming flag: its
|
|
@@ -54,6 +69,14 @@ const CAPABILITY_FIELDS = new Set([
|
|
|
54
69
|
*/
|
|
55
70
|
export function routeRuntime(contract, node, role = "worker", event = {}) {
|
|
56
71
|
if (role !== "worker" && role !== "judge") throw new TypeError("route role must be worker or judge");
|
|
72
|
+
// Catalogue records entering a routing decision are validated here, the one
|
|
73
|
+
// boundary every runtime-routing reader shares; the copies persisted on node
|
|
74
|
+
// snapshots are validated where they are written.
|
|
75
|
+
if (event.availability) {
|
|
76
|
+
for (const [id, availability] of Object.entries(event.availability)) {
|
|
77
|
+
validateRuntimeAvailability(availability, `routing availability ${id}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
57
80
|
const initialRuntimeId = role === "judge"
|
|
58
81
|
? node.gate.runtime ?? contract.runtimeDefaults?.judge
|
|
59
82
|
: node.runtime ?? contract.runtimeDefaults?.worker;
|
|
@@ -149,6 +172,31 @@ function validatePricing(value, label) {
|
|
|
149
172
|
if (rate < 0) throw new TypeError(`${label}.${key} must not be negative`);
|
|
150
173
|
}
|
|
151
174
|
}
|
|
175
|
+
/** The fields of one runtime-catalogue record (`RuntimeAvailability`). */
|
|
176
|
+
const AVAILABILITY_FIELDS = new Set(["available", "exhaustedUntil", "reason", "observedAt", "window", "remaining"]);
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* One runtime-catalogue record: what the harness de facto reported, and when.
|
|
180
|
+
* The three classified fields are required; the observables may be absent (a
|
|
181
|
+
* record that predates the field) or null (the harness exposes nothing -- never
|
|
182
|
+
* zero and never full allowance), but a present one must be typed: a record is
|
|
183
|
+
* persisted or read into a routing decision only through validators that can
|
|
184
|
+
* say what each datum means.
|
|
185
|
+
*
|
|
186
|
+
* @param {unknown} value
|
|
187
|
+
* @param {string} label
|
|
188
|
+
*/
|
|
189
|
+
export function validateRuntimeAvailability(value, label) {
|
|
190
|
+
assertObject(value, label);
|
|
191
|
+
rejectUnknown(value, AVAILABILITY_FIELDS, label);
|
|
192
|
+
if (typeof value.available !== "boolean") throw new TypeError(`${label}.available must be boolean`);
|
|
193
|
+
if (value.exhaustedUntil !== undefined && value.exhaustedUntil !== null) requireTimestamp(value.exhaustedUntil, `${label}.exhaustedUntil`);
|
|
194
|
+
requireString(value.reason, `${label}.reason`);
|
|
195
|
+
if (value.observedAt !== undefined && value.observedAt !== null) requireTimestamp(value.observedAt, `${label}.observedAt`);
|
|
196
|
+
if (value.window !== undefined && value.window !== null) requireString(value.window, `${label}.window`);
|
|
197
|
+
if (value.remaining !== undefined && value.remaining !== null) nonNegativeNumber(value.remaining, `${label}.remaining`);
|
|
198
|
+
}
|
|
199
|
+
|
|
152
200
|
/**
|
|
153
201
|
* @param {Record<string, ValidatedRuntime>} runtimes
|
|
154
202
|
* @param {string} runtimeId
|
|
@@ -21,12 +21,28 @@ export const VERIFICATION_LIMITS = Object.freeze({
|
|
|
21
21
|
snapshotPathBytes: 1024,
|
|
22
22
|
});
|
|
23
23
|
|
|
24
|
+
/**
|
|
25
|
+
* The declared risk tiers and the fraction of sampled mutants a suite must kill
|
|
26
|
+
* for a `mutation` verification entry to pass. The tier is what the entry
|
|
27
|
+
* declares; the fraction is not re-picked per entry, so two nodes at the same
|
|
28
|
+
* tier sit the same bar. The wall-clock budget that bounds how many mutants run
|
|
29
|
+
* at all is a measurement, not policy, and lives with the runner that enforces
|
|
30
|
+
* it (`MUTATION_TIME_BUDGET_MS`, `src/engine/mutation.mjs`).
|
|
31
|
+
*/
|
|
32
|
+
export const MUTATION_TIERS = Object.freeze({
|
|
33
|
+
high: 1,
|
|
34
|
+
medium: 0.75,
|
|
35
|
+
low: 0.5,
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
/** @typedef {keyof typeof MUTATION_TIERS} MutationTier */
|
|
39
|
+
|
|
24
40
|
/** @typedef {"active"|"closed"|"failed"|"crashed"|"canceled"} VerificationAttemptStatus */
|
|
25
41
|
|
|
26
42
|
/**
|
|
27
43
|
* One declared deterministic check: an argv command run by the controller.
|
|
28
44
|
*
|
|
29
|
-
* @typedef {{argv: string[], cwd?: string, timeoutSec?: number, repeat?: number, env?: string[], mutation?: {
|
|
45
|
+
* @typedef {{argv: string[], cwd?: string, timeoutSec?: number, repeat?: number, env?: string[], mutation?: {tier: MutationTier}}} VerificationCommand
|
|
30
46
|
*/
|
|
31
47
|
|
|
32
48
|
/**
|
|
@@ -126,20 +142,20 @@ function validateVerificationCommand(command, label = "verification command") {
|
|
|
126
142
|
const envBytes = env.reduce((sum, name) => sum + Buffer.byteLength(/** @type {string} */ (name), "utf8"), 0);
|
|
127
143
|
if (envBytes > VERIFICATION_LIMITS.maxEnvBytes) throw new TypeError(`${label}.env exceeds aggregate byte limit`);
|
|
128
144
|
// Mutation testing is opt-in per entry: it re-runs the same argv against
|
|
129
|
-
// deliberately broken copies of the node's written files.
|
|
130
|
-
//
|
|
131
|
-
//
|
|
132
|
-
/** @type {{
|
|
145
|
+
// deliberately broken copies of the node's written files. The entry declares
|
|
146
|
+
// its risk tier; `MUTATION_TIERS` fixes the kill fraction each tier demands,
|
|
147
|
+
// so the bar is a property of the tier, not of the author's caution.
|
|
148
|
+
/** @type {{tier: MutationTier}|undefined} */
|
|
133
149
|
let mutation;
|
|
134
150
|
if (record.mutation !== undefined) {
|
|
135
151
|
const rawMutation = record.mutation;
|
|
136
|
-
if (!rawMutation || typeof rawMutation !== "object" || Array.isArray(rawMutation)) throw new TypeError(`${label}.mutation must be an object with a
|
|
152
|
+
if (!rawMutation || typeof rawMutation !== "object" || Array.isArray(rawMutation)) throw new TypeError(`${label}.mutation must be an object with a declared risk tier`);
|
|
137
153
|
const mutationRecord = /** @type {Record<string, unknown>} */ (rawMutation);
|
|
138
|
-
for (const key of Object.keys(mutationRecord)) if (key !== "
|
|
139
|
-
if (typeof mutationRecord.
|
|
140
|
-
throw new TypeError(`${label}.mutation.
|
|
154
|
+
for (const key of Object.keys(mutationRecord)) if (key !== "tier") throw new TypeError(`${label}.mutation has unexpected field ${key}`);
|
|
155
|
+
if (typeof mutationRecord.tier !== "string" || !Object.hasOwn(MUTATION_TIERS, mutationRecord.tier)) {
|
|
156
|
+
throw new TypeError(`${label}.mutation.tier must be one of ${Object.keys(MUTATION_TIERS).join(", ")}`);
|
|
141
157
|
}
|
|
142
|
-
mutation = {
|
|
158
|
+
mutation = { tier: /** @type {MutationTier} */ (mutationRecord.tier) };
|
|
143
159
|
}
|
|
144
160
|
/** @type {VerificationCommand} */
|
|
145
161
|
const normalized = { argv: [.../** @type {string[]} */ (record.argv)], timeoutSec, repeat, env: [.../** @type {string[]} */ (env)] };
|
|
@@ -20,9 +20,17 @@ import { transition } from "./state.mjs";
|
|
|
20
20
|
/**
|
|
21
21
|
* Resolve role assignments once at run creation. Discovery is used only for
|
|
22
22
|
* omitted roles; the resulting pair is persisted so resume is deterministic.
|
|
23
|
+
* Alongside it, `decisions` records, for every assignment, the strategy that
|
|
24
|
+
* was applied and the reason for the choice: `declared` with the declaring
|
|
25
|
+
* field when the contract named the runtime -- an operator instruction
|
|
26
|
+
* prevails over every strategy -- or the discovery ranking that composed an
|
|
27
|
+
* omitted role. The record lives at this boundary, not on the persisted
|
|
28
|
+
* assignment entries, because the snapshot's routing.allowlist still carries
|
|
29
|
+
* the classified fields only (the same declared lag as the catalogue
|
|
30
|
+
* observables); it widens when a reader needs the record durably.
|
|
23
31
|
*
|
|
24
32
|
* @param {ValidatedContract} contract
|
|
25
|
-
* @returns {Promise<{assignments: Record<string, {worker: string, judge: string, composedWorker: boolean, composedJudge: boolean}>, availability: Record<string, import("./runtime-discovery.mjs").RuntimeAvailability>}>}
|
|
33
|
+
* @returns {Promise<{assignments: Record<string, {worker: string, judge: string, composedWorker: boolean, composedJudge: boolean}>, decisions: Record<string, {worker: {strategy: string|null, reason: string}, judge: {strategy: string|null, reason: string}}>, availability: Record<string, import("./runtime-discovery.mjs").RuntimeAvailability>}>}
|
|
26
34
|
*/
|
|
27
35
|
export async function runtimeAssignments(contract) {
|
|
28
36
|
const needsComposition = contract.nodes.some((node) =>
|
|
@@ -31,15 +39,34 @@ export async function runtimeAssignments(contract) {
|
|
|
31
39
|
const availability = needsComposition ? await discoverRuntimes(contract.runtimes, { cwd: contract.cwd }) : {};
|
|
32
40
|
const config = readUserConfig(process.env);
|
|
33
41
|
const assignments = composeAssignments(contract, availability, { config });
|
|
42
|
+
/** @type {Record<string, {worker: {strategy: string|null, reason: string}, judge: {strategy: string|null, reason: string}}>} */
|
|
43
|
+
const decisions = {};
|
|
34
44
|
return {
|
|
35
45
|
assignments: Object.fromEntries(Object.entries(assignments).map(([nodeId, assignment]) => {
|
|
36
46
|
const node = contract.nodes.find((candidate) => candidate.id === nodeId);
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
47
|
+
const composedWorker = node?.runtime === undefined && contract.runtimeDefaults?.worker === undefined;
|
|
48
|
+
const composedJudge = Boolean(node?.gate.enabled && node.gate.runtime === undefined && contract.runtimeDefaults?.judge === undefined);
|
|
49
|
+
// A judge the gate never asks for is no choice at all: no strategy
|
|
50
|
+
// decided it, so none is recorded.
|
|
51
|
+
const judgeSource = node?.gate.enabled && node.gate.runtime !== undefined
|
|
52
|
+
? "gate runtime"
|
|
53
|
+
: contract.runtimeDefaults?.judge !== undefined ? "runtimeDefaults.judge" : null;
|
|
54
|
+
const workerSource = node?.runtime !== undefined
|
|
55
|
+
? "node runtime"
|
|
56
|
+
: contract.runtimeDefaults?.worker !== undefined ? "runtimeDefaults.worker" : null;
|
|
57
|
+
decisions[nodeId] = {
|
|
58
|
+
worker: {
|
|
59
|
+
strategy: composedWorker ? "cost" : workerSource !== null ? "declared" : null,
|
|
60
|
+
reason: composedWorker ? "discovery: cheapest available runtime" : /** @type {string} */ (workerSource),
|
|
61
|
+
},
|
|
62
|
+
judge: {
|
|
63
|
+
strategy: composedJudge ? "priority" : judgeSource !== null ? "declared" : null,
|
|
64
|
+
reason: composedJudge ? "discovery: strongest available runtime" : /** @type {string} */ (judgeSource ?? "no judge required"),
|
|
65
|
+
},
|
|
66
|
+
};
|
|
67
|
+
return [nodeId, { ...assignment, composedWorker, composedJudge }];
|
|
42
68
|
})),
|
|
69
|
+
decisions,
|
|
43
70
|
availability,
|
|
44
71
|
};
|
|
45
72
|
}
|
package/src/engine/backoff.mjs
CHANGED
|
@@ -476,6 +476,12 @@ const MAX_ROUTING_HISTORY = 64;
|
|
|
476
476
|
/**
|
|
477
477
|
* The override reason recorded on the node, in the operator's words.
|
|
478
478
|
*
|
|
479
|
+
* The reason also names the attempt-affinity outcome, because the override is
|
|
480
|
+
* the role's working assignment and the record of why it names the runtime it
|
|
481
|
+
* names: a reset holds affinity (the previous attempt's runtime stays warm
|
|
482
|
+
* for the retry), an edge yields it (the previous runtime is the one that
|
|
483
|
+
* just failed, and the reason quotes its code).
|
|
484
|
+
*
|
|
479
485
|
* @param {Transition} schedule
|
|
480
486
|
* @param {"worker"|"judge"} role
|
|
481
487
|
* @param {string} current
|
|
@@ -484,10 +490,10 @@ const MAX_ROUTING_HISTORY = 64;
|
|
|
484
490
|
*/
|
|
485
491
|
function routeReason(schedule, role, current, error) {
|
|
486
492
|
if (schedule.kind === "reset" && schedule.reason === "quota_reset") {
|
|
487
|
-
return `${role} provider ${current} quota resets at ${schedule.at}: ${error.message}`;
|
|
493
|
+
return `${role} provider ${current} quota resets at ${schedule.at}: ${error.message}; attempt-affinity held: ${current} keeps the node's context for the retry`;
|
|
488
494
|
}
|
|
489
|
-
if (schedule.kind === "reset") return `${role} provider ${current} hit a transient network failure, retrying at ${schedule.at}: ${error.message}`;
|
|
490
|
-
if (schedule.reason === "network_backoff") return `${role} provider ${current} kept failing on the network: ${error.message}`;
|
|
491
|
-
if (schedule.reason === "protocol_failure") return `${role} provider ${current} could not hold the result protocol: ${error.message}`;
|
|
492
|
-
return `${role} provider ${current} exhausted: ${error.message}`;
|
|
495
|
+
if (schedule.kind === "reset") return `${role} provider ${current} hit a transient network failure, retrying at ${schedule.at}: ${error.message}; attempt-affinity held: ${current} keeps the node's context for the retry`;
|
|
496
|
+
if (schedule.reason === "network_backoff") return `${role} provider ${current} kept failing on the network: ${error.message}; attempt-affinity yielded: ${current} reported ${error.code}`;
|
|
497
|
+
if (schedule.reason === "protocol_failure") return `${role} provider ${current} could not hold the result protocol: ${error.message}; attempt-affinity yielded: ${current} reported ${error.code}`;
|
|
498
|
+
return `${role} provider ${current} exhausted: ${error.message}; attempt-affinity yielded: ${current} reported ${error.code}`;
|
|
493
499
|
}
|
package/src/engine/failover.mjs
CHANGED
|
@@ -8,9 +8,14 @@
|
|
|
8
8
|
* runtime a role started on, and (if declared) the runtime its `fallback`
|
|
9
9
|
* names. contract.mjs validates the field is never a self-loop; because a
|
|
10
10
|
* hop is bounded at one, a multi-runtime cycle is structurally impossible.
|
|
11
|
+
*
|
|
12
|
+
* Per-attempt resolution (`routeRuntimeForState`) lives here too, because
|
|
13
|
+
* attempt affinity is read against the same failover facts: the runtime the
|
|
14
|
+
* previous attempt ran is preferred until the hop, the catalogue, or the
|
|
15
|
+
* judge's vendor rule disqualifies it.
|
|
11
16
|
*/
|
|
12
17
|
import { harnessCapabilities } from "../harnesses/index.mjs";
|
|
13
|
-
import { nextSameTierRuntime } from "./runtime-discovery.mjs";
|
|
18
|
+
import { isRuntimeAvailable, nextSameTierRuntime } from "./runtime-discovery.mjs";
|
|
14
19
|
import { routeRuntime } from "../contract/runtime.mjs";
|
|
15
20
|
|
|
16
21
|
/** @typedef {import("../contract/index.mjs").ValidatedNode} ValidatedNode */
|
|
@@ -170,6 +175,50 @@ export function routingBackoffActive(state, phase) {
|
|
|
170
175
|
return Boolean(override?.role === phase && override.backoffUntil && Date.parse(override.backoffUntil) > Date.now());
|
|
171
176
|
}
|
|
172
177
|
|
|
178
|
+
/**
|
|
179
|
+
* The runtime id this role's previous attempt on this node ran on, read off
|
|
180
|
+
* the durable invocation record. The invocations are the one record that
|
|
181
|
+
* survives both a routed hop and the revision boundary: `resetPhaseRouting`
|
|
182
|
+
* clears the routing override between revisions, but never the invocations.
|
|
183
|
+
*
|
|
184
|
+
* @param {NodeSnapshot} state
|
|
185
|
+
* @param {"worker"|"judge"} role
|
|
186
|
+
* @returns {string|undefined}
|
|
187
|
+
*/
|
|
188
|
+
function previousAttemptRuntimeId(state, role) {
|
|
189
|
+
const found = [...(state.invocations ?? [])].reverse().find((invocation) => invocation.phase === role)?.runtimeId;
|
|
190
|
+
return typeof found === "string" ? found : undefined;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* May attempt affinity keep this role on `candidate` -- the runtime its
|
|
195
|
+
* previous attempt on this node ran? Affinity yields to a known exhaustion
|
|
196
|
+
* only: a catalogue record that `isRuntimeAvailable` -- the one home of the
|
|
197
|
+
* exhaustion and staleness rule -- refuses to admit. An absent record is not
|
|
198
|
+
* that: the snapshot's catalogue copy is the classified fields at best and
|
|
199
|
+
* often empty outright, and the strongest evidence of health here is the
|
|
200
|
+
* attempt that just ran on this runtime, so absence of a record must not read
|
|
201
|
+
* as exhaustion any more than R12 lets it read as rested. A judge candidate
|
|
202
|
+
* stays bound by the cross-vendor rule its assignment was composed under,
|
|
203
|
+
* compared against the worker that actually ran the node, exactly as
|
|
204
|
+
* `nextSameTierRuntime` compares its own candidates.
|
|
205
|
+
*
|
|
206
|
+
* @param {ValidatedContract} contract
|
|
207
|
+
* @param {NodeSnapshot} state
|
|
208
|
+
* @param {"worker"|"judge"} role
|
|
209
|
+
* @param {string} candidate
|
|
210
|
+
* @returns {boolean}
|
|
211
|
+
*/
|
|
212
|
+
function admitsAffinity(contract, state, role, candidate) {
|
|
213
|
+
const runtime = contract.runtimes[candidate];
|
|
214
|
+
if (!runtime) return false;
|
|
215
|
+
const availability = state.routing?.availability?.[candidate];
|
|
216
|
+
if (availability && !isRuntimeAvailable(availability)) return false;
|
|
217
|
+
if (role !== "judge") return true;
|
|
218
|
+
const workerId = previousAttemptRuntimeId(state, "worker") ?? state.routing?.assignments?.worker;
|
|
219
|
+
return runtime.vendor !== (workerId ? contract.runtimes[workerId]?.vendor : null);
|
|
220
|
+
}
|
|
221
|
+
|
|
173
222
|
/**
|
|
174
223
|
* @param {ValidatedContract} contract
|
|
175
224
|
* @param {ValidatedNode} node
|
|
@@ -183,6 +232,18 @@ export function routeRuntimeForState(contract, node, state, role) {
|
|
|
183
232
|
const runtime = contract.runtimes[override.runtime];
|
|
184
233
|
return { id: override.runtime, ...runtime, capabilities: harnessCapabilities(runtime) };
|
|
185
234
|
}
|
|
235
|
+
// Attempt affinity: successive attempts and revisions of this node prefer
|
|
236
|
+
// the runtime the previous attempt ran, because it is the one holding the
|
|
237
|
+
// node's context. It ranks ahead of the frozen assignment -- the assignment
|
|
238
|
+
// decided the first attempt, the attempt that ran since decides the next --
|
|
239
|
+
// and below the role-matched override above, which is itself already an
|
|
240
|
+
// affinity outcome: a reset hold on the warm runtime, or the failover edge
|
|
241
|
+
// affinity yielded to.
|
|
242
|
+
const previous = previousAttemptRuntimeId(state, role);
|
|
243
|
+
if (previous !== undefined && admitsAffinity(contract, state, role, previous)) {
|
|
244
|
+
const runtime = contract.runtimes[previous];
|
|
245
|
+
return { id: previous, ...runtime, capabilities: harnessCapabilities(runtime) };
|
|
246
|
+
}
|
|
186
247
|
const assigned = state.routing?.assignments?.[role];
|
|
187
248
|
if (assigned && contract.runtimes[assigned]) {
|
|
188
249
|
const runtime = contract.runtimes[assigned];
|
|
@@ -14,8 +14,10 @@ import { stat } from "node:fs/promises";
|
|
|
14
14
|
import { isAbsolute, resolve } from "node:path";
|
|
15
15
|
import { reviewMode, UNCITED_REJECTION_REASON } from "../contract/review-modes.mjs";
|
|
16
16
|
import { JUDGE_LIMITS } from "../contract/judge-envelope.mjs";
|
|
17
|
+
import { sharedVerificationCommands } from "../contract/final-verification.mjs";
|
|
17
18
|
|
|
18
19
|
/** @typedef {import("../contract/definition-of-done.mjs").DefinitionOfDoneItem} DefinitionOfDoneItem */
|
|
20
|
+
/** @typedef {import("../contract/verification.mjs").VerificationCommand} VerificationCommand */
|
|
19
21
|
/** @typedef {import("../contract/definition-of-done.mjs").DefinitionOfDoneProof} DefinitionOfDoneProof */
|
|
20
22
|
/** @typedef {import("../contract/index.mjs").ExecutionOverride} ExecutionOverride */
|
|
21
23
|
/** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
|
|
@@ -349,6 +351,48 @@ function declaredWriteCoverage(state) {
|
|
|
349
351
|
directoryRoots.some((root) => path === root || path.startsWith(`${root}/`));
|
|
350
352
|
}
|
|
351
353
|
|
|
354
|
+
/**
|
|
355
|
+
* Whether a path named by verification output belongs to the contract's own
|
|
356
|
+
* `sharedVerification` suite -- the repository ratchets the operator appends to
|
|
357
|
+
* every node -- rather than to the node's own work. An argv entry names either
|
|
358
|
+
* the file the runner was given or a directory it walked.
|
|
359
|
+
*
|
|
360
|
+
* Deliberately not caught: a `sharedVerification` that reaches the ratchet
|
|
361
|
+
* without naming it in argv (`npm test`, or any script that picks the files
|
|
362
|
+
* itself). No token matches, the file is not recognized as a ratchet, and the
|
|
363
|
+
* contract-defect advice applies to it as it did before. Widening the match to
|
|
364
|
+
* guess what a script runs would be worse than the gap it closes.
|
|
365
|
+
*
|
|
366
|
+
* @param {{sharedVerification?: VerificationCommand[]}} contract
|
|
367
|
+
* @returns {(path: string) => boolean}
|
|
368
|
+
*/
|
|
369
|
+
function sharedVerificationCoverage(contract) {
|
|
370
|
+
const argv = sharedVerificationCommands(contract).flatMap((command) => command.argv);
|
|
371
|
+
return (path) => argv.some((token) => token === path || path.startsWith(`${token}/`));
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/**
|
|
375
|
+
* The operator-facing description for a failing ratchet: a check the contract
|
|
376
|
+
* itself declared, that runs on every node, and that this node's change broke.
|
|
377
|
+
*
|
|
378
|
+
* It says nothing about the write scope on purpose. Telling the operator to
|
|
379
|
+
* hand the node the ratchet licenses the next worker to edit the rule it just
|
|
380
|
+
* violated -- observed 2026-09-21, when a node that pushed
|
|
381
|
+
* `src/report/render.mjs` to 801 lines was advised to add the 800-line ratchet
|
|
382
|
+
* to its `writeFiles`. The remedy is already inside the node's scope: its own
|
|
383
|
+
* code.
|
|
384
|
+
*
|
|
385
|
+
* @param {string[]} paths
|
|
386
|
+
* @returns {string}
|
|
387
|
+
*/
|
|
388
|
+
function ratchetFailureDescription(paths) {
|
|
389
|
+
const files = paths.join(", ");
|
|
390
|
+
return boundedText(
|
|
391
|
+
`deterministic verification failed in ${files}, which the contract runs on every node as sharedVerification: this node's change broke a repository-wide rule. The remedy is in the node's own code -- bring the change back within the rule the check enforces. The check itself is not the node's to change, and relaxing it is not a fix`,
|
|
392
|
+
JUDGE_LIMITS.descriptionBytes,
|
|
393
|
+
);
|
|
394
|
+
}
|
|
395
|
+
|
|
352
396
|
/**
|
|
353
397
|
* The operator-facing description for a failure that named a test outside the
|
|
354
398
|
* declared write scope. The defect is not in the worker's code: the worker is
|
|
@@ -372,24 +416,38 @@ function undeclaredTestDescription(paths) {
|
|
|
372
416
|
* The deterministic controller-verification failure verdict, kept next to the
|
|
373
417
|
* Definition of Done gate so every deterministic failure settles identically.
|
|
374
418
|
*
|
|
419
|
+
* The contract is a parameter because an undeclared test file has two opposite
|
|
420
|
+
* remedies and only the contract tells them apart: a ratchet it declared in
|
|
421
|
+
* `sharedVerification` is the node's code to fix, while any other withheld test
|
|
422
|
+
* is the contract's scope to widen.
|
|
423
|
+
*
|
|
424
|
+
* @param {{sharedVerification?: VerificationCommand[]}} contract
|
|
375
425
|
* @param {{verification?: {commands?: Array<{argv: string[], passed?: boolean, attempts?: Array<{stdout?: string, stderr?: string, exitCode?: number|null, timedOut?: boolean}>}>, error?: unknown}|null, scope?: {boundary?: {files?: string[], roots?: string[], fileRoots?: string[]}}|null}} state
|
|
376
426
|
* @returns {import("./prompts.mjs").JudgeVerdict}
|
|
377
427
|
*/
|
|
378
|
-
export function verificationFailureVerdict(state) {
|
|
428
|
+
export function verificationFailureVerdict(contract, state) {
|
|
379
429
|
const failedCommands = (state.verification?.commands ?? []).filter((command) => !command.passed);
|
|
380
430
|
const evidence = failedCommands.length
|
|
381
431
|
? failedCommands.map((command) => `${command.argv.join(" ")}: ${(command.attempts ?? []).map((attempt) => `exit=${attempt.exitCode ?? "-"}${attempt.timedOut ? " timeout" : ""}`).join(", ")}`).join("; ")
|
|
382
432
|
: state.verification?.error ?? "verification controller failed to execute a command";
|
|
383
|
-
const
|
|
433
|
+
const declared = declaredWriteCoverage(state);
|
|
434
|
+
const undeclared = namedTestFiles(failedCommands).filter((path) => !declared(path));
|
|
435
|
+
const isRatchet = sharedVerificationCoverage(contract);
|
|
436
|
+
const ratchets = undeclared.filter(isRatchet);
|
|
437
|
+
const withheld = undeclared.filter((path) => !isRatchet(path));
|
|
438
|
+
const descriptions = [
|
|
439
|
+
...(ratchets.length ? [ratchetFailureDescription(ratchets)] : []),
|
|
440
|
+
...(withheld.length ? [undeclaredTestDescription(withheld)] : []),
|
|
441
|
+
];
|
|
384
442
|
return {
|
|
385
443
|
verdict: "fail",
|
|
386
444
|
maxSeverity: "critical",
|
|
387
445
|
summary: "deterministic verification failed",
|
|
388
|
-
findings: [{
|
|
446
|
+
findings: (descriptions.length ? descriptions : ["deterministic verification failed"]).map((description) => ({
|
|
389
447
|
severity: "critical",
|
|
390
|
-
description
|
|
448
|
+
description,
|
|
391
449
|
evidence: boundedText(evidence),
|
|
392
|
-
}
|
|
450
|
+
})),
|
|
393
451
|
};
|
|
394
452
|
}
|
|
395
453
|
|
package/src/engine/lifecycle.mjs
CHANGED
|
@@ -280,22 +280,35 @@ export function clearTierExhaustion(state) {
|
|
|
280
280
|
}
|
|
281
281
|
|
|
282
282
|
/**
|
|
283
|
+
* Settle what a closed invocation produced, and start whatever the outcome
|
|
284
|
+
* earns next.
|
|
285
|
+
*
|
|
286
|
+
* The two maps are two different things, and conflating them is what let a run
|
|
287
|
+
* exceed its own `maxParallel` (measured 2026-09-21, run
|
|
288
|
+
* state-location-and-routing-economics-13): `closed` is the job this call owns
|
|
289
|
+
* -- the scheduler hands one node's job at a time so two settlements never
|
|
290
|
+
* interleave -- while `running` is the run's live dispatch authority, the very
|
|
291
|
+
* map the scheduler's `maxParallel - running.size` counts. Anything started
|
|
292
|
+
* here lands in `running` and is accounted from that instant; nothing started
|
|
293
|
+
* here may be settled by this call.
|
|
294
|
+
*
|
|
283
295
|
* @param {ValidatedContract} contract
|
|
284
296
|
* @param {string} runDir
|
|
285
297
|
* @param {Map<string, NodeSnapshot>} states
|
|
286
|
-
* @param {Map<string, Job>}
|
|
298
|
+
* @param {Map<string, Job>} closed
|
|
287
299
|
* @param {LockHandle} lock
|
|
288
300
|
* @param {string} campaignPath
|
|
301
|
+
* @param {Map<string, Job>} running
|
|
289
302
|
* @returns {Promise<void>}
|
|
290
303
|
*/
|
|
291
|
-
export async function finalizeClosedJobs(contract, runDir, states,
|
|
304
|
+
export async function finalizeClosedJobs(contract, runDir, states, closed, lock, campaignPath, running) {
|
|
292
305
|
// Advisory spend lines are checked every tick, before outcome handling: a
|
|
293
306
|
// crossing must be visible while the spend is happening, not only when the
|
|
294
307
|
// run is already over. The check never stops or transitions a node.
|
|
295
308
|
await emitNodeAdvisories(contract, runDir, states);
|
|
296
|
-
for (const [nodeId, job] of
|
|
309
|
+
for (const [nodeId, job] of closed) {
|
|
297
310
|
if (!job.closed || invocationAlive(job.invocation)) continue;
|
|
298
|
-
|
|
311
|
+
closed.delete(nodeId);
|
|
299
312
|
const state = states.get(nodeId);
|
|
300
313
|
if (!state) continue;
|
|
301
314
|
// Usage is extracted and persisted BEFORE any outcome-specific handling:
|