faberun 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/campaign/index.mjs +74 -2
- package/src/campaign/record.mjs +28 -0
- package/src/cli/plan.mjs +40 -1
- package/src/cli.mjs +1 -0
- package/src/contract/index.mjs +15 -4
- package/src/contract/runtime.mjs +50 -2
- package/src/contract/snapshot.mjs +21 -2
- package/src/contract/verification.mjs +26 -10
- package/src/engine/assignment.mjs +33 -6
- package/src/engine/backoff.mjs +11 -5
- package/src/engine/failover.mjs +62 -1
- package/src/engine/judge-gate.mjs +63 -5
- package/src/engine/lifecycle.mjs +17 -4
- package/src/engine/mutation.mjs +47 -7
- package/src/engine/resume.mjs +1 -1
- package/src/engine/runtime-discovery.mjs +44 -7
- package/src/engine/scheduler.mjs +19 -23
- package/src/engine/scope.mjs +108 -6
- package/src/engine/settle.mjs +22 -8
- package/src/engine/state.mjs +1 -0
- package/src/plan/freeze.mjs +29 -0
- package/src/plan/pipeline.mjs +203 -22
- package/src/plan/routing.mjs +151 -33
- package/src/plan/sizing.mjs +45 -0
- package/src/report/final.mjs +2 -1
- package/src/report/packet-repetition.mjs +121 -0
- package/src/report/render.mjs +3 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "faberun",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.15.0",
|
|
4
4
|
"description": "Faberun is a development orchestration system that turns intent into verified software: harness- and model-agnostic, it keeps the intent, coordinates the work, verifies the result and decides what happens next.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/src/campaign/index.mjs
CHANGED
|
@@ -21,7 +21,9 @@ import { handoffFromState, materializeHandoff } from "./handoff.mjs";
|
|
|
21
21
|
/** @typedef {{path: string, digest: string}} CampaignContract */
|
|
22
22
|
/** @typedef {{runId: string, contractPath?: string, branch: string, sha: string, previousSha: string|null, at: string}} PromotionRecord */
|
|
23
23
|
/** @typedef {{code: string, message: string, at: string, contractPath?: string, contractId?: string, runId?: string, node?: string|null, status?: string|null, resume?: string}} CampaignAttention */
|
|
24
|
-
/** @typedef {{
|
|
24
|
+
/** @typedef {{runId: string, node: string, passed: boolean|null, verdict: string|null}} RequirementNodeEvidence */
|
|
25
|
+
/** @typedef {{requirementId: string, status: "covered"|"open", nodes: RequirementNodeEvidence[]}} RequirementClosure */
|
|
26
|
+
/** @typedef {{id: string, goal: string, status: "active"|"closed", linkedRunIds: string[], contracts: CampaignContract[], landBranch: string, promotions: PromotionRecord[], attention?: CampaignAttention, requirements?: RequirementClosure[], createdAt: string, updatedAt: string, closedAt?: string}} Campaign */
|
|
25
27
|
/** @typedef {{type: string, eventId: string, at: string, sessionId?: string, text?: string, tool?: string, transcript?: string|null, transcriptUnavailable?: boolean, format?: string|null, cursor?: string|null, decisionId?: string, supersedes?: string, runId?: string, questionId?: string, campaignId?: string, nodeId?: string|null, phase?: string, checkpointsDone?: number, checkpointsTotal?: number, runtime?: string|null, state?: string, lastProgressAt?: string, attention?: string|null}} JournalEntry */
|
|
26
28
|
/** @typedef {{updatedAt: string|null, decisions: Record<string, JournalEntry>, questions: Record<string, JournalEntry>, constraints: JournalEntry[], intents: JournalEntry[], outcomes: JournalEntry[], sessions: JournalEntry[], next: JournalEntry|null, evicted: Record<string, number>}} Projection */
|
|
27
29
|
/** @typedef {{cursor: number, byte: number, size: number, projection: Projection}} ProjectionRecord */
|
|
@@ -135,12 +137,82 @@ export function closeCampaign(campaignPath, { at = new Date().toISOString(), eve
|
|
|
135
137
|
}
|
|
136
138
|
const repoRoot = campaignRepoRoot(campaignPath);
|
|
137
139
|
const ledgerFiles = preserveCampaignLedger(campaignPath, repoRoot);
|
|
138
|
-
|
|
140
|
+
// The closure travels on the record itself, computed in one deterministic
|
|
141
|
+
// pass over the linked runs' own files before the close is written.
|
|
142
|
+
const requirements = buildRequirementClosure(campaignPath, campaign);
|
|
143
|
+
/** @type {Campaign} */
|
|
144
|
+
const closed = { ...campaign, status: "closed", closedAt: at, updatedAt: at, requirements };
|
|
139
145
|
writeJsonAtomic(join(campaignPath, CAMPAIGN_FILE), closed);
|
|
140
146
|
appendJournal(campaignPath, { type: "campaign.closed", at, eventId });
|
|
141
147
|
return { path: campaignPath, campaign: closed, ledgerFiles };
|
|
142
148
|
}
|
|
143
149
|
|
|
150
|
+
/**
|
|
151
|
+
* The requirement closure a close records: one entry per requirement id the
|
|
152
|
+
* linked runs' contracts declared, correlated only by the identifiers the runs
|
|
153
|
+
* carried -- the contract node's declaration and the id the done node snapshot
|
|
154
|
+
* itself carries, stamped there by the engine -- never by requirement text. A
|
|
155
|
+
* requirement no done node carries is recorded as open rather than omitted.
|
|
156
|
+
* Deterministic and free of external calls: runs are visited in sorted id
|
|
157
|
+
* order, nodes in contract order, requirement ids sorted, and the only inputs
|
|
158
|
+
* are files already on disk. A run that never launched (or was pruned)
|
|
159
|
+
* contributes nothing.
|
|
160
|
+
*
|
|
161
|
+
* @param {string} campaignPath
|
|
162
|
+
* @param {Campaign} campaign
|
|
163
|
+
* @returns {RequirementClosure[]}
|
|
164
|
+
*/
|
|
165
|
+
function buildRequirementClosure(campaignPath, campaign) {
|
|
166
|
+
const runsDir = resolve(campaignPath, "..", "..");
|
|
167
|
+
/** @type {Map<string, RequirementNodeEvidence[]>} */
|
|
168
|
+
const covered = new Map();
|
|
169
|
+
/** @type {Set<string>} */
|
|
170
|
+
const declared = new Set();
|
|
171
|
+
for (const runId of [...campaign.linkedRunIds].sort()) {
|
|
172
|
+
const contract = readRunJson(join(runsDir, runId, "contract.json"));
|
|
173
|
+
const nodes = contract !== null && Array.isArray(contract.nodes) ? /** @type {JsonObject[]} */ (contract.nodes) : [];
|
|
174
|
+
for (const node of nodes) {
|
|
175
|
+
const nodeId = typeof node.id === "string" ? node.id : "";
|
|
176
|
+
const requirementIds = Array.isArray(node.requirementIds) ? node.requirementIds : [];
|
|
177
|
+
if (!nodeId || requirementIds.length === 0) continue;
|
|
178
|
+
const snapshot = readRunJson(join(runsDir, runId, "nodes", `${nodeId}.json`));
|
|
179
|
+
const done = snapshot !== null && snapshot.status === "done";
|
|
180
|
+
const carried = snapshot !== null && Array.isArray(snapshot.requirementIds) ? /** @type {unknown[]} */ (snapshot.requirementIds) : [];
|
|
181
|
+
const verification = snapshot !== null ? /** @type {JsonObject|null|undefined} */ (snapshot.verification) : undefined;
|
|
182
|
+
const gate = snapshot !== null ? /** @type {JsonObject|null|undefined} */ (snapshot.gate) : undefined;
|
|
183
|
+
const passed = verification && typeof verification.passed === "boolean" ? verification.passed : null;
|
|
184
|
+
const verdict = gate && typeof gate.verdict === "string" ? gate.verdict : null;
|
|
185
|
+
for (const requirementId of requirementIds) {
|
|
186
|
+
if (typeof requirementId !== "string") continue;
|
|
187
|
+
declared.add(requirementId);
|
|
188
|
+
if (!done || !carried.includes(requirementId)) continue;
|
|
189
|
+
const evidence = covered.get(requirementId) ?? [];
|
|
190
|
+
evidence.push({ runId, node: nodeId, passed, verdict });
|
|
191
|
+
covered.set(requirementId, evidence);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return [...declared].sort().map((requirementId) => {
|
|
196
|
+
const nodes = covered.get(requirementId) ?? [];
|
|
197
|
+
return { requirementId, status: nodes.length > 0 ? "covered" : "open", nodes };
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* @param {string} path
|
|
203
|
+
* @returns {JsonObject|null} null when the file is absent or not JSON: a run
|
|
204
|
+
* that never launched contributes nothing to the closure rather than failing
|
|
205
|
+
* the close, the same discipline `preserveCampaignLedger` applies to a run
|
|
206
|
+
* without a usage ledger.
|
|
207
|
+
*/
|
|
208
|
+
function readRunJson(path) {
|
|
209
|
+
try {
|
|
210
|
+
return /** @type {JsonObject} */ (JSON.parse(readFileSync(path, "utf8")));
|
|
211
|
+
} catch {
|
|
212
|
+
return null;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
144
216
|
/**
|
|
145
217
|
* The repository a campaign's ledger is preserved into. Under the home layout
|
|
146
218
|
* the campaign path carries the project id at the resolver's fixed position,
|
package/src/campaign/record.mjs
CHANGED
|
@@ -99,4 +99,32 @@ function validateCampaign(campaign) {
|
|
|
99
99
|
if (promotion.contractPath !== undefined) requireString(promotion.contractPath, `campaign.promotions[${index}].contractPath`);
|
|
100
100
|
}
|
|
101
101
|
}
|
|
102
|
+
// The requirement closure a close records: one entry per requirement id the
|
|
103
|
+
// linked runs' contracts declared, correlated by the identifiers the runs
|
|
104
|
+
// carried, with every uncovered requirement kept as open. Optional on read
|
|
105
|
+
// so a campaign closed before the field existed stays readable.
|
|
106
|
+
if (record.requirements !== undefined) {
|
|
107
|
+
if (!Array.isArray(record.requirements)) throw new TypeError("campaign.requirements must be an array");
|
|
108
|
+
for (const [index, entry] of record.requirements.entries()) {
|
|
109
|
+
assertObject(entry, `campaign.requirements[${index}]`);
|
|
110
|
+
const closure = /** @type {JsonObject} */ (entry);
|
|
111
|
+
requireId(closure.requirementId, `campaign.requirements[${index}].requirementId`);
|
|
112
|
+
if (closure.status !== "covered" && closure.status !== "open") {
|
|
113
|
+
throw new TypeError(`campaign.requirements[${index}].status must be covered or open`);
|
|
114
|
+
}
|
|
115
|
+
if (!Array.isArray(closure.nodes)) throw new TypeError(`campaign.requirements[${index}].nodes must be an array`);
|
|
116
|
+
for (const [position, evidence] of closure.nodes.entries()) {
|
|
117
|
+
assertObject(evidence, `campaign.requirements[${index}].nodes[${position}]`);
|
|
118
|
+
const node = /** @type {JsonObject} */ (evidence);
|
|
119
|
+
requireId(node.runId, `campaign.requirements[${index}].nodes[${position}].runId`);
|
|
120
|
+
requireId(node.node, `campaign.requirements[${index}].nodes[${position}].node`);
|
|
121
|
+
if (node.passed !== null && typeof node.passed !== "boolean") {
|
|
122
|
+
throw new TypeError(`campaign.requirements[${index}].nodes[${position}].passed must be a boolean or null`);
|
|
123
|
+
}
|
|
124
|
+
if (node.verdict !== null && typeof node.verdict !== "string") {
|
|
125
|
+
throw new TypeError(`campaign.requirements[${index}].nodes[${position}].verdict must be a string or null`);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
102
130
|
}
|
package/src/cli/plan.mjs
CHANGED
|
@@ -11,6 +11,8 @@ import { classifyRunProgress } from "../campaign/chain.mjs";
|
|
|
11
11
|
import { runProgress } from "../engine/supervise.mjs";
|
|
12
12
|
import { DISCOVERY_RUNTIME_DEFINITIONS } from "../engine/runtime-discovery.mjs";
|
|
13
13
|
import { validateRuntime } from "../contract/runtime.mjs";
|
|
14
|
+
import { validateFinalVerification, validateSharedVerification } from "../contract/final-verification.mjs";
|
|
15
|
+
import { colorLevel, statusToken } from "./brand.mjs";
|
|
14
16
|
import { delay } from "../util.mjs";
|
|
15
17
|
import { runPlanningPipeline } from "../plan/pipeline.mjs";
|
|
16
18
|
import { runDirectory } from "../run/paths.mjs";
|
|
@@ -79,9 +81,40 @@ export function loadRuntimesCatalogue(path) {
|
|
|
79
81
|
return runtimes;
|
|
80
82
|
}
|
|
81
83
|
|
|
84
|
+
/**
|
|
85
|
+
* A `--verification <path>` catalogue: a JSON object carrying either or both
|
|
86
|
+
* of the contract's own suite keys, `sharedVerification` and
|
|
87
|
+
* `finalVerification`, each validated with the same validator
|
|
88
|
+
* `validateContract` applies. A key that is not a contract suite is refused
|
|
89
|
+
* rather than ignored: a typo'd key would freeze a contract that looks
|
|
90
|
+
* ratcheted and is not, which is the failure mode this flag exists to close.
|
|
91
|
+
*
|
|
92
|
+
* @param {string} path
|
|
93
|
+
* @returns {{sharedVerification?: import("../contract/index.mjs").VerificationCommand[], finalVerification?: import("../contract/index.mjs").VerificationCommand[]}}
|
|
94
|
+
*/
|
|
95
|
+
export function loadVerificationSuites(path) {
|
|
96
|
+
const resolved = resolve(path);
|
|
97
|
+
/** @type {unknown} */
|
|
98
|
+
let raw;
|
|
99
|
+
try {
|
|
100
|
+
raw = JSON.parse(readFileSync(resolved, "utf8"));
|
|
101
|
+
} catch (error) {
|
|
102
|
+
throw new Error(`--verification ${path} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
|
|
103
|
+
}
|
|
104
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) throw new Error(`--verification ${path} must be a JSON object`);
|
|
105
|
+
const record = /** @type {Record<string, unknown>} */ (raw);
|
|
106
|
+
for (const key of Object.keys(record)) {
|
|
107
|
+
if (key !== "sharedVerification" && key !== "finalVerification") throw new Error(`--verification ${path} must carry only sharedVerification and finalVerification: ${key}`);
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
...(record.sharedVerification === undefined ? {} : { sharedVerification: validateSharedVerification(record.sharedVerification, "contract.sharedVerification") }),
|
|
111
|
+
...(record.finalVerification === undefined ? {} : { finalVerification: validateFinalVerification(record.finalVerification, "contract.finalVerification") }),
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
82
115
|
/**
|
|
83
116
|
* @param {string} target
|
|
84
|
-
* @param {{campaign?: string, phase?: string, "review-rounds"?: string, "approve-below"?: string, "runtime-defaults"?: string, runtimes?: string, detach?: boolean, json?: boolean}} values
|
|
117
|
+
* @param {{campaign?: string, phase?: string, "review-rounds"?: string, "approve-below"?: string, "runtime-defaults"?: string, runtimes?: string, verification?: string, detach?: boolean, json?: boolean}} values
|
|
85
118
|
* @returns {Promise<void>}
|
|
86
119
|
*/
|
|
87
120
|
export async function planCli(target, values) {
|
|
@@ -95,12 +128,16 @@ export async function planCli(target, values) {
|
|
|
95
128
|
const runtimes = typeof values.runtimes === "string" && values.runtimes
|
|
96
129
|
? loadRuntimesCatalogue(values.runtimes)
|
|
97
130
|
: DISCOVERY_RUNTIME_DEFINITIONS;
|
|
131
|
+
const verification = typeof values.verification === "string" && values.verification
|
|
132
|
+
? loadVerificationSuites(values.verification)
|
|
133
|
+
: {};
|
|
98
134
|
|
|
99
135
|
if (values.detach === true) {
|
|
100
136
|
const argv = ["plan", specPath, "--campaign", campaignId, "--phase", phase, "--review-rounds", String(reviewRounds)];
|
|
101
137
|
if (approveBelow !== undefined) argv.push("--approve-below", approveBelow);
|
|
102
138
|
if (values["runtime-defaults"] !== undefined) argv.push("--runtime-defaults", values["runtime-defaults"]);
|
|
103
139
|
if (typeof values.runtimes === "string" && values.runtimes) argv.push("--runtimes", resolve(values.runtimes));
|
|
140
|
+
if (typeof values.verification === "string" && values.verification) argv.push("--verification", resolve(values.verification));
|
|
104
141
|
const child = detachArgv(argv);
|
|
105
142
|
if (child.pid === undefined) throw new Error("detached plan has no pid");
|
|
106
143
|
process.stdout.write(`[plan] detached · pid ${child.pid} · ${specPath}\n`);
|
|
@@ -115,6 +152,7 @@ export async function planCli(target, values) {
|
|
|
115
152
|
approveBelow,
|
|
116
153
|
runtimeDefaults,
|
|
117
154
|
runtimes,
|
|
155
|
+
verification,
|
|
118
156
|
launch: async (contractPath, contract) => {
|
|
119
157
|
const child = detachSelf("run", contractPath);
|
|
120
158
|
if (child.pid === undefined) throw new Error("detached planning run has no pid");
|
|
@@ -139,5 +177,6 @@ export async function planCli(target, values) {
|
|
|
139
177
|
process.exitCode = 1;
|
|
140
178
|
return;
|
|
141
179
|
}
|
|
180
|
+
for (const warning of result.warnings) process.stdout.write(`${statusToken("warn", colorLevel(process.env, process.stdout.isTTY))} ${warning}\n`);
|
|
142
181
|
process.stdout.write(`[plan] ${campaignId} phase ${phase} frozen · approved ${result.approved} · ${result.contractPath}\n`);
|
|
143
182
|
}
|
package/src/cli.mjs
CHANGED
|
@@ -122,6 +122,7 @@ export const COMMAND_OPTIONS = {
|
|
|
122
122
|
"approve-below": { type: "string" },
|
|
123
123
|
"runtime-defaults": { type: "string" },
|
|
124
124
|
runtimes: { type: "string" },
|
|
125
|
+
verification: { type: "string" },
|
|
125
126
|
detach: { type: "boolean" },
|
|
126
127
|
json: { type: "boolean" },
|
|
127
128
|
},
|
package/src/contract/index.mjs
CHANGED
|
@@ -37,7 +37,7 @@ const CONTRACT_FIELDS = new Set([
|
|
|
37
37
|
]);
|
|
38
38
|
const DEFAULTS_FIELDS = new Set(["worker", "judge"]);
|
|
39
39
|
const NODE_FIELDS = new Set([
|
|
40
|
-
"id", "type", "phase", "runtime", "dependsOn", "taskPacket", "taskPacketFile", "prompt", "promptFile",
|
|
40
|
+
"id", "type", "phase", "requirementIds", "runtime", "dependsOn", "taskPacket", "taskPacketFile", "prompt", "promptFile",
|
|
41
41
|
"definitionOfDone", "gate", "timeoutSec", "maxTurns",
|
|
42
42
|
"requiredCapabilities", "packetHash", "sourceIdentity", "replayPolicy",
|
|
43
43
|
]);
|
|
@@ -68,7 +68,7 @@ const GATE_REVIEWS = new Set(["none", "advisory", "blocking"]);
|
|
|
68
68
|
|
|
69
69
|
/** @typedef {{enabled: boolean, review?: ("none"|"advisory"|"blocking"), runtime?: string, failOn?: ("minor"|"major"|"critical")[], maxRevisions?: number, requiredCapabilities?: CapabilityRequirements, skipWhen?: {verificationGreen: true, maxChangedPaths: number}}} ValidatedGate */
|
|
70
70
|
|
|
71
|
-
/** @typedef {{id: string, type: string, phase: string, runtime?: string, dependsOn: string[], taskPacket: TaskPacket, taskPacketFile?: string, prompt: string, definitionOfDone: import("./definition-of-done.mjs").DefinitionOfDoneItem[], gate: ValidatedGate, timeoutSec?: number, maxTurns?: number, requiredCapabilities: CapabilityRequirements, packetHash: string, sourceIdentity: SourceIdentity, replayPolicy: "safe"|"reconcile"|"never"}} ValidatedNode */
|
|
71
|
+
/** @typedef {{id: string, type: string, phase: string, requirementIds?: string[], runtime?: string, dependsOn: string[], taskPacket: TaskPacket, taskPacketFile?: string, prompt: string, definitionOfDone: import("./definition-of-done.mjs").DefinitionOfDoneItem[], gate: ValidatedGate, timeoutSec?: number, maxTurns?: number, requiredCapabilities: CapabilityRequirements, packetHash: string, sourceIdentity: SourceIdentity, replayPolicy: "safe"|"reconcile"|"never"}} ValidatedNode */
|
|
72
72
|
|
|
73
73
|
/** @typedef {{schemaVersion: number, contractVersion: string, id: string, campaignId: string, goal: string, cwd: string, sourceIdentity: SourceIdentity, runtimes: Record<string, ValidatedRuntime>, runtimeDefaults: {worker?: string, judge?: string}, nodes: ValidatedNode[], maxParallel: number, pollIntervalMs: number, stallTimeoutSec: number, timeoutSec: number, maxTurns: number, phaseSessionReuse: boolean, finalVerification?: VerificationCommand[], sharedVerification?: VerificationCommand[], nodeAdvisory?: NodeAdvisoryPolicy, warnings: string[]}} ValidatedContract */
|
|
74
74
|
/** @typedef {{costUsd?: number, durationSec?: number}} NodeAdvisoryPolicy */
|
|
@@ -98,11 +98,11 @@ const GATE_REVIEWS = new Set(["none", "advisory", "blocking"]);
|
|
|
98
98
|
/** @typedef {{history: RoutingHistoryEntry[], currentOverride: RoutingOverride|null, assignments?: RuntimeAssignments, availability?: Record<string, RuntimeAvailability>, tierExhaustion?: TierExhaustion, tierExhaustionCycle?: number}} RoutingState */
|
|
99
99
|
/** @typedef {{revision?: number, heartbeatCount: number, dryHeartbeatCount: number, progressSignature?: string|null, lastHeartbeatAt: string|null, lastProgressAt: string|null, nextCheckAt?: string|null}} ProgressState */
|
|
100
100
|
/** @typedef {{status: "unassigned"|"provisioning"|"ready"|"failed"|"removed", path: string|null, branch: string|null, commit: string|null, baseSha?: string|null, sealedSha?: string|null, sealError?: string|null, previousAttempt?: number|null}} WorktreeState */
|
|
101
|
-
/** @typedef {{schemaVersion: number, contractVersion: string, id: string, type: string, sourceIdentity: SourceIdentity, packetHash: string, status: NodeStatus, phase: NodePhase, attempt: number, revisions: number, judgeFailures?: number, review?: ("none"|"advisory"|"blocking"), runtime: RuntimeSnapshot|null, blockedBy: string[], startedAt: string|null, updatedAt: string, result: unknown, gate: GateResult|null, error: SnapshotError|null, usage?: Usage, costUsd?: number, routing?: RoutingState|null, progress?: ProgressState|null, worktree?: WorktreeState|null, integratedHead?: string|null, invocations?: Invocation[], executionOverrides?: ExecutionOverride[], verification?: VerificationState|null, scope?: BoundedScope|null, scopeFindings?: ScopeFindings|null, previousAttempt?: string, sessionPolicy?: {forceFresh?: boolean}|null, declaredReadBytes?: number|null}} NodeSnapshot */
|
|
101
|
+
/** @typedef {{schemaVersion: number, contractVersion: string, id: string, type: string, sourceIdentity: SourceIdentity, packetHash: string, requirementIds?: string[], status: NodeStatus, phase: NodePhase, attempt: number, revisions: number, judgeFailures?: number, review?: ("none"|"advisory"|"blocking"), runtime: RuntimeSnapshot|null, blockedBy: string[], startedAt: string|null, updatedAt: string, result: unknown, gate: GateResult|null, error: SnapshotError|null, usage?: Usage, costUsd?: number, routing?: RoutingState|null, progress?: ProgressState|null, worktree?: WorktreeState|null, integratedHead?: string|null, invocations?: Invocation[], executionOverrides?: ExecutionOverride[], verification?: VerificationState|null, scope?: BoundedScope|null, scopeFindings?: ScopeFindings|null, previousAttempt?: string, sessionPolicy?: {forceFresh?: boolean}|null, declaredReadBytes?: number|null}} NodeSnapshot */
|
|
102
102
|
/** @typedef {{path: string, sha: string}} ControllerIdentity */
|
|
103
103
|
/** @typedef {{schemaVersion: number, contractVersion: string, pid: number, processStartToken: string|null, startedAt: string, sourceIdentity: SourceIdentity, controllerIdentity?: ControllerIdentity, integrationRef?: string, identityWarnings?: string[], relaunchCount?: number, lastRelaunchProgressAt?: string|null, attention?: {code: string, message: string, at: string}|null, contractDigest?: string, scopeDecision?: ScopeDecision, autoRetries?: Record<string, {code: string, at: string}>}} RunMetadata */
|
|
104
104
|
/** @typedef {{at: string, base: string|null, dirtyTreeFingerprint: string|null}} ScopeDecision */
|
|
105
|
-
/** @typedef {{schemaVersion: number, contractVersion: string, at: string, node: string, from?: string, to: string, type?: string, phase?: string, attempt?: number, role?: "worker"|"judge", status?: NodeStatus, runtime?: string, currentRuntime?: string, errorCode?: string, error?: SnapshotError, verdict?: string, summary?: string, revisions?: number, sourceIdentity: SourceIdentity, packetHash: string, override?: unknown, recovery?: unknown, invocationId?: string, unexpectedPaths?: string[], unexpectedPathCount?: number}} EventRecord */
|
|
105
|
+
/** @typedef {{schemaVersion: number, contractVersion: string, at: string, node: string, from?: string, to: string, type?: string, phase?: string, attempt?: number, role?: "worker"|"judge", status?: NodeStatus, runtime?: string, currentRuntime?: string, errorCode?: string, error?: SnapshotError, verdict?: string, summary?: string, revisions?: number, requirementIds?: string[], sourceIdentity: SourceIdentity, packetHash: string, override?: unknown, recovery?: unknown, invocationId?: string, unexpectedPaths?: string[], unexpectedPathCount?: number}} EventRecord */
|
|
106
106
|
|
|
107
107
|
/**
|
|
108
108
|
* Validate and canonicalize the versioned contract. Runtime JSON remains
|
|
@@ -179,6 +179,17 @@ export function validateContract(raw, contractPath, options = {}) {
|
|
|
179
179
|
ids.add(node.id);
|
|
180
180
|
requireString(node.type, `nodes[${index}].type`);
|
|
181
181
|
boundedString(node.phase, `nodes[${index}].phase`, 128);
|
|
182
|
+
// A node's requirementIds are inherited from its phase at freeze time
|
|
183
|
+
// (src/plan/freeze.mjs stamps them) and are optional on read: a contract
|
|
184
|
+
// without the field loads unchanged, which is what keeps CONTRACT_VERSION
|
|
185
|
+
// at 0.3.0 while requirement ids reach the node.
|
|
186
|
+
const requirementIds = node.requirementIds;
|
|
187
|
+
if (requirementIds !== undefined) {
|
|
188
|
+
if (!Array.isArray(requirementIds) || requirementIds.length > 64) {
|
|
189
|
+
throw new TypeError(`nodes[${index}].requirementIds must be an array of at most 64 requirement ids`);
|
|
190
|
+
}
|
|
191
|
+
requirementIds.forEach((id, position) => boundedString(id, `nodes[${index}].requirementIds[${position}]`, 128));
|
|
192
|
+
}
|
|
182
193
|
if (node.runtime !== undefined) requireRuntime(runtimes, node.runtime, `nodes[${index}].runtime`);
|
|
183
194
|
const dependsOn = node.dependsOn ?? [];
|
|
184
195
|
if (!Array.isArray(dependsOn) || dependsOn.some((id) => typeof id !== "string")) {
|
package/src/contract/runtime.mjs
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* A declared runtime: its fields, the harness names it may name, whether its
|
|
3
|
-
* permission mode can execute a command, and how a role resolves to one
|
|
3
|
+
* permission mode can execute a command, and how a role resolves to one -- the
|
|
4
|
+
* latter including the strategies a routing rule may name, which are authored
|
|
5
|
+
* protocol surface a reader can reject by name just like a harness name.
|
|
4
6
|
*
|
|
5
7
|
* Split out because both the contract validator and the snapshot validator need
|
|
6
8
|
* it -- a persisted `runtime` on a node snapshot is the shape the contract
|
|
7
9
|
* declared -- and the snapshot validator should not import the contract
|
|
8
10
|
* validator to reach it.
|
|
9
11
|
*/
|
|
10
|
-
import { assertObject, nonNegativeNumber, positiveInteger, positiveNumber, rejectUnknown, requireId, requireString, requireStringArray } from "./assert.mjs";
|
|
12
|
+
import { assertObject, nonNegativeNumber, positiveInteger, positiveNumber, rejectUnknown, requireId, requireString, requireStringArray, requireTimestamp } from "./assert.mjs";
|
|
11
13
|
import { composeAssignments } from "../engine/runtime-discovery.mjs";
|
|
12
14
|
import { harnessCapabilities, resolvePermissionExecution, resolveVendor, validateCapabilityRequirements } from "../harnesses/index.mjs";
|
|
13
15
|
import { stableJson } from "../util.mjs";
|
|
@@ -26,6 +28,19 @@ const RUNTIME_FIELDS = new Set([
|
|
|
26
28
|
]);
|
|
27
29
|
const RUNTIME_HARNESSES = new Set(["claude", "codex", "agy", "dsh", "zcode", "exec-jsonl", "replay"]);
|
|
28
30
|
|
|
31
|
+
/**
|
|
32
|
+
* The strategies a routing rule may name: how a rule consumes its `prefer`
|
|
33
|
+
* list. `priority` takes the first admissible candidate; `cost` the lowest
|
|
34
|
+
* declared `costRank`; `reset-proximity` the least observed `remaining`
|
|
35
|
+
* allowance -- the window nearest its reset is spent first; `attempt-affinity`
|
|
36
|
+
* the previous attempt's runtime for the same node. A strategy whose datum a
|
|
37
|
+
* given runtime does not expose is inert for that runtime -- never a failure.
|
|
38
|
+
* An assignment records `declared` instead of any of these when an operator's
|
|
39
|
+
* runtime instruction prevailed over the table and every strategy.
|
|
40
|
+
* @typedef {"priority" | "cost" | "reset-proximity" | "attempt-affinity"} RoutingStrategy
|
|
41
|
+
*/
|
|
42
|
+
export const ROUTING_STRATEGIES = Object.freeze(new Set(["priority", "cost", "reset-proximity", "attempt-affinity"]));
|
|
43
|
+
|
|
29
44
|
/**
|
|
30
45
|
* Harness-specific stall thresholds where the contract's single default is
|
|
31
46
|
* wrong for every turn the harness runs. `zcode` has no streaming flag: its
|
|
@@ -54,6 +69,14 @@ const CAPABILITY_FIELDS = new Set([
|
|
|
54
69
|
*/
|
|
55
70
|
export function routeRuntime(contract, node, role = "worker", event = {}) {
|
|
56
71
|
if (role !== "worker" && role !== "judge") throw new TypeError("route role must be worker or judge");
|
|
72
|
+
// Catalogue records entering a routing decision are validated here, the one
|
|
73
|
+
// boundary every runtime-routing reader shares; the copies persisted on node
|
|
74
|
+
// snapshots are validated where they are written.
|
|
75
|
+
if (event.availability) {
|
|
76
|
+
for (const [id, availability] of Object.entries(event.availability)) {
|
|
77
|
+
validateRuntimeAvailability(availability, `routing availability ${id}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
57
80
|
const initialRuntimeId = role === "judge"
|
|
58
81
|
? node.gate.runtime ?? contract.runtimeDefaults?.judge
|
|
59
82
|
: node.runtime ?? contract.runtimeDefaults?.worker;
|
|
@@ -149,6 +172,31 @@ function validatePricing(value, label) {
|
|
|
149
172
|
if (rate < 0) throw new TypeError(`${label}.${key} must not be negative`);
|
|
150
173
|
}
|
|
151
174
|
}
|
|
175
|
+
/** The fields of one runtime-catalogue record (`RuntimeAvailability`). */
|
|
176
|
+
const AVAILABILITY_FIELDS = new Set(["available", "exhaustedUntil", "reason", "observedAt", "window", "remaining"]);
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* One runtime-catalogue record: what the harness de facto reported, and when.
|
|
180
|
+
* The three classified fields are required; the observables may be absent (a
|
|
181
|
+
* record that predates the field) or null (the harness exposes nothing -- never
|
|
182
|
+
* zero and never full allowance), but a present one must be typed: a record is
|
|
183
|
+
* persisted or read into a routing decision only through validators that can
|
|
184
|
+
* say what each datum means.
|
|
185
|
+
*
|
|
186
|
+
* @param {unknown} value
|
|
187
|
+
* @param {string} label
|
|
188
|
+
*/
|
|
189
|
+
export function validateRuntimeAvailability(value, label) {
|
|
190
|
+
assertObject(value, label);
|
|
191
|
+
rejectUnknown(value, AVAILABILITY_FIELDS, label);
|
|
192
|
+
if (typeof value.available !== "boolean") throw new TypeError(`${label}.available must be boolean`);
|
|
193
|
+
if (value.exhaustedUntil !== undefined && value.exhaustedUntil !== null) requireTimestamp(value.exhaustedUntil, `${label}.exhaustedUntil`);
|
|
194
|
+
requireString(value.reason, `${label}.reason`);
|
|
195
|
+
if (value.observedAt !== undefined && value.observedAt !== null) requireTimestamp(value.observedAt, `${label}.observedAt`);
|
|
196
|
+
if (value.window !== undefined && value.window !== null) requireString(value.window, `${label}.window`);
|
|
197
|
+
if (value.remaining !== undefined && value.remaining !== null) nonNegativeNumber(value.remaining, `${label}.remaining`);
|
|
198
|
+
}
|
|
199
|
+
|
|
152
200
|
/**
|
|
153
201
|
* @param {Record<string, ValidatedRuntime>} runtimes
|
|
154
202
|
* @param {string} runtimeId
|
|
@@ -127,7 +127,7 @@ export function validateNodeSnapshot(value, expectedNode = null) {
|
|
|
127
127
|
assertObject(value, "node snapshot");
|
|
128
128
|
rejectUnknown(value, new Set([
|
|
129
129
|
"schemaVersion", "contractVersion", "id", "type", "sourceIdentity", "packetHash", "status", "phase",
|
|
130
|
-
"attempt", "revisions", "judgeFailures", "runtime", "blockedBy", "startedAt", "updatedAt", "result", "gate", "error", "usage",
|
|
130
|
+
"attempt", "revisions", "judgeFailures", "requirementIds", "runtime", "blockedBy", "startedAt", "updatedAt", "result", "gate", "error", "usage",
|
|
131
131
|
"costUsd", "routing", "progress", "worktree", "invocations", "executionOverrides", "verification", "scope",
|
|
132
132
|
"scopeFindings", "review", "previousAttempt", "sessionPolicy", "integratedHead", "declaredReadBytes",
|
|
133
133
|
]), "node snapshot");
|
|
@@ -139,6 +139,10 @@ export function validateNodeSnapshot(value, expectedNode = null) {
|
|
|
139
139
|
nonNegativeInteger(value.attempt, "node snapshot.attempt");
|
|
140
140
|
nonNegativeInteger(value.revisions, "node snapshot.revisions");
|
|
141
141
|
if (value.judgeFailures !== undefined) nonNegativeInteger(value.judgeFailures, "node snapshot.judgeFailures");
|
|
142
|
+
// The phase requirement ids the engine stamped onto the node when its result
|
|
143
|
+
// was accepted; optional on read so snapshots written before the stamp load
|
|
144
|
+
// unchanged.
|
|
145
|
+
if (value.requirementIds !== undefined) validateRequirementIdList(value.requirementIds, "node snapshot.requirementIds");
|
|
142
146
|
// The review mode that governed the attempt's gate, recorded so a status
|
|
143
147
|
// surface can tell an advisory finding from a below-threshold blocking one.
|
|
144
148
|
if (value.review !== undefined && !REVIEW_MODES.has(/** @type {string} */ (value.review))) {
|
|
@@ -206,7 +210,7 @@ export function validateEvent(value) {
|
|
|
206
210
|
assertObject(value, "event");
|
|
207
211
|
rejectUnknown(value, new Set([
|
|
208
212
|
"schemaVersion", "contractVersion", "at", "node", "from", "to", "type", "phase", "attempt", "runtime",
|
|
209
|
-
"role", "status", "currentRuntime", "errorCode", "error", "verdict", "summary", "revisions", "sourceIdentity", "packetHash", "override", "recovery", "invocationId", "unexpectedPaths", "unexpectedPathCount",
|
|
213
|
+
"role", "status", "currentRuntime", "errorCode", "error", "verdict", "summary", "revisions", "requirementIds", "sourceIdentity", "packetHash", "override", "recovery", "invocationId", "unexpectedPaths", "unexpectedPathCount",
|
|
210
214
|
]), "event");
|
|
211
215
|
validateMetadata(value, "event");
|
|
212
216
|
requireString(value.at, "event.at");
|
|
@@ -230,8 +234,23 @@ export function validateEvent(value) {
|
|
|
230
234
|
}
|
|
231
235
|
}
|
|
232
236
|
if (value.unexpectedPathCount !== undefined) nonNegativeInteger(value.unexpectedPathCount, "event.unexpectedPathCount");
|
|
237
|
+
if (value.requirementIds !== undefined) validateRequirementIdList(value.requirementIds, "event.requirementIds");
|
|
233
238
|
return /** @type {EventRecord} */ (value);
|
|
234
239
|
}
|
|
240
|
+
/**
|
|
241
|
+
* The requirement ids a node inherited from its phase, stamped by the engine
|
|
242
|
+
* and carried beside the result on both the snapshot and the transition event.
|
|
243
|
+
* Bounded like every persisted list: at most 64 ids of at most 128 bytes.
|
|
244
|
+
*
|
|
245
|
+
* @param {unknown} value
|
|
246
|
+
* @param {string} label
|
|
247
|
+
*/
|
|
248
|
+
function validateRequirementIdList(value, label) {
|
|
249
|
+
if (!Array.isArray(value) || value.length > 64) {
|
|
250
|
+
throw new TypeError(`${label} must be an array of at most 64 requirement ids`);
|
|
251
|
+
}
|
|
252
|
+
for (const [index, id] of value.entries()) boundedString(id, `${label}[${index}]`, 128);
|
|
253
|
+
}
|
|
235
254
|
/**
|
|
236
255
|
* @param {unknown} value
|
|
237
256
|
* @param {string} label
|
|
@@ -21,12 +21,28 @@ export const VERIFICATION_LIMITS = Object.freeze({
|
|
|
21
21
|
snapshotPathBytes: 1024,
|
|
22
22
|
});
|
|
23
23
|
|
|
24
|
+
/**
|
|
25
|
+
* The declared risk tiers and the fraction of sampled mutants a suite must kill
|
|
26
|
+
* for a `mutation` verification entry to pass. The tier is what the entry
|
|
27
|
+
* declares; the fraction is not re-picked per entry, so two nodes at the same
|
|
28
|
+
* tier sit the same bar. The wall-clock budget that bounds how many mutants run
|
|
29
|
+
* at all is a measurement, not policy, and lives with the runner that enforces
|
|
30
|
+
* it (`MUTATION_TIME_BUDGET_MS`, `src/engine/mutation.mjs`).
|
|
31
|
+
*/
|
|
32
|
+
export const MUTATION_TIERS = Object.freeze({
|
|
33
|
+
high: 1,
|
|
34
|
+
medium: 0.75,
|
|
35
|
+
low: 0.5,
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
/** @typedef {keyof typeof MUTATION_TIERS} MutationTier */
|
|
39
|
+
|
|
24
40
|
/** @typedef {"active"|"closed"|"failed"|"crashed"|"canceled"} VerificationAttemptStatus */
|
|
25
41
|
|
|
26
42
|
/**
|
|
27
43
|
* One declared deterministic check: an argv command run by the controller.
|
|
28
44
|
*
|
|
29
|
-
* @typedef {{argv: string[], cwd?: string, timeoutSec?: number, repeat?: number, env?: string[], mutation?: {
|
|
45
|
+
* @typedef {{argv: string[], cwd?: string, timeoutSec?: number, repeat?: number, env?: string[], mutation?: {tier: MutationTier}}} VerificationCommand
|
|
30
46
|
*/
|
|
31
47
|
|
|
32
48
|
/**
|
|
@@ -126,20 +142,20 @@ function validateVerificationCommand(command, label = "verification command") {
|
|
|
126
142
|
const envBytes = env.reduce((sum, name) => sum + Buffer.byteLength(/** @type {string} */ (name), "utf8"), 0);
|
|
127
143
|
if (envBytes > VERIFICATION_LIMITS.maxEnvBytes) throw new TypeError(`${label}.env exceeds aggregate byte limit`);
|
|
128
144
|
// Mutation testing is opt-in per entry: it re-runs the same argv against
|
|
129
|
-
// deliberately broken copies of the node's written files.
|
|
130
|
-
//
|
|
131
|
-
//
|
|
132
|
-
/** @type {{
|
|
145
|
+
// deliberately broken copies of the node's written files. The entry declares
|
|
146
|
+
// its risk tier; `MUTATION_TIERS` fixes the kill fraction each tier demands,
|
|
147
|
+
// so the bar is a property of the tier, not of the author's caution.
|
|
148
|
+
/** @type {{tier: MutationTier}|undefined} */
|
|
133
149
|
let mutation;
|
|
134
150
|
if (record.mutation !== undefined) {
|
|
135
151
|
const rawMutation = record.mutation;
|
|
136
|
-
if (!rawMutation || typeof rawMutation !== "object" || Array.isArray(rawMutation)) throw new TypeError(`${label}.mutation must be an object with a
|
|
152
|
+
if (!rawMutation || typeof rawMutation !== "object" || Array.isArray(rawMutation)) throw new TypeError(`${label}.mutation must be an object with a declared risk tier`);
|
|
137
153
|
const mutationRecord = /** @type {Record<string, unknown>} */ (rawMutation);
|
|
138
|
-
for (const key of Object.keys(mutationRecord)) if (key !== "
|
|
139
|
-
if (typeof mutationRecord.
|
|
140
|
-
throw new TypeError(`${label}.mutation.
|
|
154
|
+
for (const key of Object.keys(mutationRecord)) if (key !== "tier") throw new TypeError(`${label}.mutation has unexpected field ${key}`);
|
|
155
|
+
if (typeof mutationRecord.tier !== "string" || !Object.hasOwn(MUTATION_TIERS, mutationRecord.tier)) {
|
|
156
|
+
throw new TypeError(`${label}.mutation.tier must be one of ${Object.keys(MUTATION_TIERS).join(", ")}`);
|
|
141
157
|
}
|
|
142
|
-
mutation = {
|
|
158
|
+
mutation = { tier: /** @type {MutationTier} */ (mutationRecord.tier) };
|
|
143
159
|
}
|
|
144
160
|
/** @type {VerificationCommand} */
|
|
145
161
|
const normalized = { argv: [.../** @type {string[]} */ (record.argv)], timeoutSec, repeat, env: [.../** @type {string[]} */ (env)] };
|
|
@@ -20,9 +20,17 @@ import { transition } from "./state.mjs";
|
|
|
20
20
|
/**
|
|
21
21
|
* Resolve role assignments once at run creation. Discovery is used only for
|
|
22
22
|
* omitted roles; the resulting pair is persisted so resume is deterministic.
|
|
23
|
+
* Alongside it, `decisions` records, for every assignment, the strategy that
|
|
24
|
+
* was applied and the reason for the choice: `declared` with the declaring
|
|
25
|
+
* field when the contract named the runtime -- an operator instruction
|
|
26
|
+
* prevails over every strategy -- or the discovery ranking that composed an
|
|
27
|
+
* omitted role. The record lives at this boundary, not on the persisted
|
|
28
|
+
* assignment entries, because the snapshot's routing.allowlist still carries
|
|
29
|
+
* the classified fields only (the same declared lag as the catalogue
|
|
30
|
+
* observables); it widens when a reader needs the record durably.
|
|
23
31
|
*
|
|
24
32
|
* @param {ValidatedContract} contract
|
|
25
|
-
* @returns {Promise<{assignments: Record<string, {worker: string, judge: string, composedWorker: boolean, composedJudge: boolean}>, availability: Record<string, import("./runtime-discovery.mjs").RuntimeAvailability>}>}
|
|
33
|
+
* @returns {Promise<{assignments: Record<string, {worker: string, judge: string, composedWorker: boolean, composedJudge: boolean}>, decisions: Record<string, {worker: {strategy: string|null, reason: string}, judge: {strategy: string|null, reason: string}}>, availability: Record<string, import("./runtime-discovery.mjs").RuntimeAvailability>}>}
|
|
26
34
|
*/
|
|
27
35
|
export async function runtimeAssignments(contract) {
|
|
28
36
|
const needsComposition = contract.nodes.some((node) =>
|
|
@@ -31,15 +39,34 @@ export async function runtimeAssignments(contract) {
|
|
|
31
39
|
const availability = needsComposition ? await discoverRuntimes(contract.runtimes, { cwd: contract.cwd }) : {};
|
|
32
40
|
const config = readUserConfig(process.env);
|
|
33
41
|
const assignments = composeAssignments(contract, availability, { config });
|
|
42
|
+
/** @type {Record<string, {worker: {strategy: string|null, reason: string}, judge: {strategy: string|null, reason: string}}>} */
|
|
43
|
+
const decisions = {};
|
|
34
44
|
return {
|
|
35
45
|
assignments: Object.fromEntries(Object.entries(assignments).map(([nodeId, assignment]) => {
|
|
36
46
|
const node = contract.nodes.find((candidate) => candidate.id === nodeId);
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
47
|
+
const composedWorker = node?.runtime === undefined && contract.runtimeDefaults?.worker === undefined;
|
|
48
|
+
const composedJudge = Boolean(node?.gate.enabled && node.gate.runtime === undefined && contract.runtimeDefaults?.judge === undefined);
|
|
49
|
+
// A judge the gate never asks for is no choice at all: no strategy
|
|
50
|
+
// decided it, so none is recorded.
|
|
51
|
+
const judgeSource = node?.gate.enabled && node.gate.runtime !== undefined
|
|
52
|
+
? "gate runtime"
|
|
53
|
+
: contract.runtimeDefaults?.judge !== undefined ? "runtimeDefaults.judge" : null;
|
|
54
|
+
const workerSource = node?.runtime !== undefined
|
|
55
|
+
? "node runtime"
|
|
56
|
+
: contract.runtimeDefaults?.worker !== undefined ? "runtimeDefaults.worker" : null;
|
|
57
|
+
decisions[nodeId] = {
|
|
58
|
+
worker: {
|
|
59
|
+
strategy: composedWorker ? "cost" : workerSource !== null ? "declared" : null,
|
|
60
|
+
reason: composedWorker ? "discovery: cheapest available runtime" : /** @type {string} */ (workerSource),
|
|
61
|
+
},
|
|
62
|
+
judge: {
|
|
63
|
+
strategy: composedJudge ? "priority" : judgeSource !== null ? "declared" : null,
|
|
64
|
+
reason: composedJudge ? "discovery: strongest available runtime" : /** @type {string} */ (judgeSource ?? "no judge required"),
|
|
65
|
+
},
|
|
66
|
+
};
|
|
67
|
+
return [nodeId, { ...assignment, composedWorker, composedJudge }];
|
|
42
68
|
})),
|
|
69
|
+
decisions,
|
|
43
70
|
availability,
|
|
44
71
|
};
|
|
45
72
|
}
|
package/src/engine/backoff.mjs
CHANGED
|
@@ -476,6 +476,12 @@ const MAX_ROUTING_HISTORY = 64;
|
|
|
476
476
|
/**
|
|
477
477
|
* The override reason recorded on the node, in the operator's words.
|
|
478
478
|
*
|
|
479
|
+
* The reason also names the attempt-affinity outcome, because the override is
|
|
480
|
+
* the role's working assignment and the record of why it names the runtime it
|
|
481
|
+
* names: a reset holds affinity (the previous attempt's runtime stays warm
|
|
482
|
+
* for the retry), an edge yields it (the previous runtime is the one that
|
|
483
|
+
* just failed, and the reason quotes its code).
|
|
484
|
+
*
|
|
479
485
|
* @param {Transition} schedule
|
|
480
486
|
* @param {"worker"|"judge"} role
|
|
481
487
|
* @param {string} current
|
|
@@ -484,10 +490,10 @@ const MAX_ROUTING_HISTORY = 64;
|
|
|
484
490
|
*/
|
|
485
491
|
function routeReason(schedule, role, current, error) {
|
|
486
492
|
if (schedule.kind === "reset" && schedule.reason === "quota_reset") {
|
|
487
|
-
return `${role} provider ${current} quota resets at ${schedule.at}: ${error.message}`;
|
|
493
|
+
return `${role} provider ${current} quota resets at ${schedule.at}: ${error.message}; attempt-affinity held: ${current} keeps the node's context for the retry`;
|
|
488
494
|
}
|
|
489
|
-
if (schedule.kind === "reset") return `${role} provider ${current} hit a transient network failure, retrying at ${schedule.at}: ${error.message}`;
|
|
490
|
-
if (schedule.reason === "network_backoff") return `${role} provider ${current} kept failing on the network: ${error.message}`;
|
|
491
|
-
if (schedule.reason === "protocol_failure") return `${role} provider ${current} could not hold the result protocol: ${error.message}`;
|
|
492
|
-
return `${role} provider ${current} exhausted: ${error.message}`;
|
|
495
|
+
if (schedule.kind === "reset") return `${role} provider ${current} hit a transient network failure, retrying at ${schedule.at}: ${error.message}; attempt-affinity held: ${current} keeps the node's context for the retry`;
|
|
496
|
+
if (schedule.reason === "network_backoff") return `${role} provider ${current} kept failing on the network: ${error.message}; attempt-affinity yielded: ${current} reported ${error.code}`;
|
|
497
|
+
if (schedule.reason === "protocol_failure") return `${role} provider ${current} could not hold the result protocol: ${error.message}; attempt-affinity yielded: ${current} reported ${error.code}`;
|
|
498
|
+
return `${role} provider ${current} exhausted: ${error.message}; attempt-affinity yielded: ${current} reported ${error.code}`;
|
|
493
499
|
}
|