faberun 0.19.0 → 0.19.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -42,7 +42,14 @@ export const MUTATION_TIERS = Object.freeze({
42
42
  /**
43
43
  * One declared deterministic check: an argv command run by the controller.
44
44
  *
45
- * @typedef {{argv: string[], cwd?: string, timeoutSec?: number, repeat?: number, env?: string[], mutation?: {tier: MutationTier}}} VerificationCommand
45
+ * `requirementId` names the spec requirement this command is the proof of. It
46
+ * changes nothing about how the command runs; it is what makes a duplicated
47
+ * proof visible. Measured 2026-09-22: one broken command lived in a spec's R3,
48
+ * in its R4 and in seven nodes' verification, and the repair reached one of
49
+ * them — nothing could tell that the other copies had stopped agreeing,
50
+ * because nothing recorded that they were copies of one claim.
51
+ *
52
+ * @typedef {{argv: string[], cwd?: string, timeoutSec?: number, repeat?: number, env?: string[], mutation?: {tier: MutationTier}, requirementId?: string}} VerificationCommand
46
53
  */
47
54
 
48
55
  /**
@@ -122,7 +129,7 @@ export function validateVerificationCommands(commands, label = "verification") {
122
129
  function validateVerificationCommand(command, label = "verification command") {
123
130
  if (!command || typeof command !== "object" || Array.isArray(command)) throw new TypeError(`${label} must be an argv command object`);
124
131
  const record = /** @type {Record<string, unknown>} */ (command);
125
- const allowed = new Set(["argv", "cwd", "timeoutSec", "repeat", "env", "mutation"]);
132
+ const allowed = new Set(["argv", "cwd", "timeoutSec", "repeat", "env", "mutation", "requirementId"]);
126
133
  for (const key of Object.keys(record)) if (!allowed.has(key)) throw new TypeError(`${label} has unexpected field ${key}`);
127
134
  if (!Array.isArray(record.argv) || record.argv.length === 0 || record.argv.length > 64 || record.argv.some((item) => typeof item !== "string" || !item.trim() || Buffer.byteLength(item, "utf8") > 8 * 1024)) {
128
135
  throw new TypeError(`${label}.argv must be a non-empty array of strings`);
@@ -157,10 +164,18 @@ function validateVerificationCommand(command, label = "verification command") {
157
164
  }
158
165
  mutation = { tier: /** @type {MutationTier} */ (mutationRecord.tier) };
159
166
  }
167
+ // Bounded exactly like the node-level `requirementIds` it must match against
168
+ // (at most 128 bytes), so the two sides of the claim cannot accept different
169
+ // ids.
170
+ if (record.requirementId !== undefined
171
+ && (typeof record.requirementId !== "string" || !record.requirementId.trim() || Buffer.byteLength(record.requirementId, "utf8") > 128)) {
172
+ throw new TypeError(`${label}.requirementId must be a requirement id of at most 128 bytes`);
173
+ }
160
174
  /** @type {VerificationCommand} */
161
175
  const normalized = { argv: [.../** @type {string[]} */ (record.argv)], timeoutSec, repeat, env: [.../** @type {string[]} */ (env)] };
162
176
  if (record.cwd !== undefined) normalized.cwd = /** @type {string} */ (record.cwd);
163
177
  if (mutation !== undefined) normalized.mutation = mutation;
178
+ if (record.requirementId !== undefined) normalized.requirementId = /** @type {string} */ (record.requirementId);
164
179
  return normalized;
165
180
  }
166
181
 
@@ -199,3 +214,49 @@ export function compactVerification(result) {
199
214
  return { passed: Boolean(result?.passed), commands };
200
215
  }
201
216
 
217
+
218
+ /**
219
+ * One proof, one source. A command that declares `requirementId` says it is
220
+ * the proof of that requirement; this reports the two ways such a claim can
221
+ * be false.
222
+ *
223
+ * A claim the node does not carry is a mislabel: the node's own
224
+ * `requirementIds` are what the phase assigned it, and a command proving
225
+ * something outside them is either the wrong id or the wrong node.
226
+ *
227
+ * Copies that stopped agreeing are the measured one. 2026-09-22: a broken
228
+ * command lived in a spec's R3, its R4, and seven nodes' verification, and
229
+ * the repair reached one copy. Nothing could see that the others had drifted,
230
+ * because nothing recorded that they were copies of a single claim. Argv is
231
+ * compared against argv, never a joined string against a shell command: a
232
+ * joined argv loses argument boundaries, which is the same reason a
233
+ * `verification` proof references an index instead of comparing text.
234
+ *
235
+ * @param {Array<{id: string, requirementIds?: string[], commands: VerificationCommand[]}>} owners
236
+ * @returns {string[]}
237
+ */
238
+ export function requirementProofWarnings(owners) {
239
+ /** @type {string[]} */
240
+ const warnings = [];
241
+ /** @type {Map<string, Array<{owner: string, position: number, argv: string[]}>>} */
242
+ const claims = new Map();
243
+ for (const owner of owners) {
244
+ owner.commands.forEach((command, position) => {
245
+ const requirementId = command.requirementId;
246
+ if (requirementId === undefined) return;
247
+ if (owner.requirementIds !== undefined && !owner.requirementIds.includes(requirementId)) {
248
+ warnings.push(`${owner.id}: verification[${position}] declares requirementId "${requirementId}", which this node does not carry in requirementIds`);
249
+ }
250
+ const claimed = claims.get(requirementId) ?? [];
251
+ claimed.push({ owner: owner.id, position, argv: command.argv });
252
+ claims.set(requirementId, claimed);
253
+ });
254
+ }
255
+ for (const [requirementId, claimed] of claims) {
256
+ const distinct = new Map(claimed.map((claim) => [JSON.stringify(claim.argv), claim]));
257
+ if (distinct.size < 2) continue;
258
+ const listed = [...distinct.values()].map((claim) => `${claim.owner}: verification[${claim.position}] runs ${JSON.stringify(claim.argv)}`).join("; ");
259
+ warnings.push(`requirementId "${requirementId}" is proven by commands that no longer agree: ${listed}`);
260
+ }
261
+ return warnings;
262
+ }
@@ -16,7 +16,7 @@ import { invocationOwned } from "./process-identity.mjs";
16
16
  import { terminateInvocation } from "./process.mjs";
17
17
  import { join, resolve } from "node:path";
18
18
  import { readFileSync } from "node:fs";
19
- import { readRunNodes } from "./scheduler.mjs";
19
+ import { readRunNodes } from "../run/node-store.mjs";
20
20
  import { createPreservedRef, deleteRef, releaseAttemptWorktree, runRefName } from "../repo/worktree.mjs";
21
21
  import { syncAgentSignal } from "../repo/signal.mjs";
22
22
  import { transition, writeNode } from "./state.mjs";
@@ -55,7 +55,7 @@ export async function cancelRun(runDirPath) {
55
55
  }
56
56
  const controllerLock = await acquireStaleLock(runDir);
57
57
  try {
58
- const states = readRunNodes(runDir, contract);
58
+ const states = readRunNodes(runDir, contract, { tolerateMissing: true });
59
59
  /** @type {Error[]} */
60
60
  const failures = [];
61
61
  for (const state of states) {
@@ -196,7 +196,7 @@ async function waitForTerminal(runDir, timeoutMs) {
196
196
  while (Date.now() < deadline) {
197
197
  const contractPath = join(runDir, "contract.json");
198
198
  const contract = validateContract(JSON.parse(readFileSync(contractPath, "utf8")), contractPath, { persisted: true });
199
- const states = readRunNodes(runDir, contract);
199
+ const states = readRunNodes(runDir, contract, { tolerateMissing: true });
200
200
  if (states.every((state) => SETTLED.has(state.status)) && states.every((state) => (state.invocations ?? []).every((invocation) => !invocationOwned(invocation)))) return true;
201
201
  await delay(100);
202
202
  }
@@ -25,6 +25,7 @@ import { attemptWorkspace, createAttemptWorktree, sealAttempt } from "../repo/wo
25
25
  import { attemptWorktreePath } from "../run/paths.mjs";
26
26
  import { basename, dirname, join } from "node:path";
27
27
  import { errorCode, errorMessage } from "../util.mjs";
28
+ import { gateProofTimeoutMs } from "../contract/final-verification.mjs";
28
29
  import { captureWorkspaceScope, captureWorkspaceSnapshot } from "../repo/workspace.mjs";
29
30
  import { deterministicGate, judgeReaskReason, judgeRequired, judgeSkippedByScope } from "./judge-gate.mjs";
30
31
  import { emptyScope, persistedScopeBoundary, workerScope } from "./scope.mjs";
@@ -459,7 +460,7 @@ export async function startJudge(contract, node, state, runDir, running, workerR
459
460
  node,
460
461
  workspace,
461
462
  reask,
462
- Math.max(1_000, Math.min((node.timeoutSec ?? contract.timeoutSec ?? 60) * 1000, 120_000)),
463
+ gateProofTimeoutMs(node, contract),
463
464
  /** @type {import("../contract/index.mjs").VerificationState|null} */ (state.verification),
464
465
  );
465
466
  state.review = reviewMode(node.gate);
@@ -136,15 +136,22 @@ export function failoverEdges(contract) {
136
136
  * The first snapshot wins; every later requirement set is accumulated, so a
137
137
  * runtime reached twice is still checked against both callers' demands.
138
138
  *
139
- * @param {Map<string, {runtime: RuntimeSnapshot, requiredCapabilitySets: import("../harnesses/index.mjs").CapabilityRequirements[]}>} runtimes
139
+ * `routed` says whether some node or default names this runtime, directly or
140
+ * as a declared failover target. A runtime reached only because a role named
141
+ * none -- every catalogue entry is then a candidate availability discovery may
142
+ * pick -- is added with `routed: false`, and one route that names it at all
143
+ * makes it routed for good.
144
+ *
145
+ * @param {Map<string, {runtime: RuntimeSnapshot, requiredCapabilitySets: import("../harnesses/index.mjs").CapabilityRequirements[], routed: boolean}>} runtimes
140
146
  * @param {RuntimeSnapshot} runtime
141
147
  * @param {import("../harnesses/index.mjs").CapabilityRequirements[]} requiredCapabilitySets
148
+ * @param {boolean} [routed]
142
149
  */
143
- export function addRuntimeRequirement(runtimes, runtime, requiredCapabilitySets) {
150
+ export function addRuntimeRequirement(runtimes, runtime, requiredCapabilitySets, routed = true) {
144
151
  const incoming = requiredCapabilitySets.filter((requirements) => requirements && Object.keys(requirements).length);
145
152
  const current = runtimes.get(runtime.id);
146
- if (!current) runtimes.set(runtime.id, { runtime, requiredCapabilitySets: incoming });
147
- else runtimes.set(runtime.id, { runtime: current.runtime, requiredCapabilitySets: [...current.requiredCapabilitySets, ...incoming] });
153
+ if (!current) runtimes.set(runtime.id, { runtime, requiredCapabilitySets: incoming, routed });
154
+ else runtimes.set(runtime.id, { runtime: current.runtime, requiredCapabilitySets: [...current.requiredCapabilitySets, ...incoming], routed: current.routed || routed });
148
155
  }
149
156
 
150
157
  /**
@@ -16,6 +16,7 @@ import { reviewMode, UNCITED_REJECTION_REASON } from "../contract/review-modes.m
16
16
  import { JUDGE_LIMITS } from "../contract/judge-envelope.mjs";
17
17
  import { sharedVerificationCommands } from "../contract/final-verification.mjs";
18
18
  import { killTarget } from "../host/platform.mjs";
19
+ import { shellWords } from "../util.mjs";
19
20
 
20
21
  /** @typedef {import("../contract/definition-of-done.mjs").DefinitionOfDoneItem} DefinitionOfDoneItem */
21
22
  /** @typedef {import("../contract/verification.mjs").VerificationCommand} VerificationCommand */
@@ -174,31 +175,26 @@ function envForFilteredProof() {
174
175
  /**
175
176
  * The node:test filters a command string declares, in argv order, as flag and
176
177
  * value. Presence alone changes behaviour (the appended reporter and the
177
- * zero-plan look-up); the value is read for the refusal detail alone, which is
178
- * why a whitespace split is close enough even though the command runs through
179
- * a shell.
178
+ * zero-plan look-up); the value lands in the refusal detail, and it is read
179
+ * the way the shell this command runs under reads it -- `shellWords` groups a
180
+ * quoted pattern into one word instead of splitting the pattern itself.
180
181
  *
181
182
  * @param {string} ref
182
183
  * @returns {Array<{flag: string, value: string}>}
183
184
  */
184
185
  function declaredTestFilters(ref) {
185
- const tokens = ref.split(/\s+/u).filter(Boolean);
186
+ const tokens = shellWords(ref);
186
187
  /** @type {Array<{flag: string, value: string}>} */
187
188
  const filters = [];
188
189
  for (const [index, token] of tokens.entries()) {
189
190
  for (const flag of TEST_FILTER_FLAGS) {
190
- if (token.startsWith(`${flag}=`)) filters.push({ flag, value: unquote(token.slice(flag.length + 1)) });
191
- else if (token === flag) filters.push({ flag, value: unquote(tokens[index + 1] ?? "") });
191
+ if (token.startsWith(`${flag}=`)) filters.push({ flag, value: token.slice(flag.length + 1) });
192
+ else if (token === flag) filters.push({ flag, value: tokens[index + 1] ?? "" });
192
193
  }
193
194
  }
194
195
  return filters;
195
196
  }
196
197
 
197
- /** @param {string} value @returns {string} */
198
- function unquote(value) {
199
- return value.replace(/^['"]|['"]$/gu, "");
200
- }
201
-
202
198
  /** @param {Array<{flag: string, value: string}>} filters @returns {string} */
203
199
  function filterNames(filters) {
204
200
  return filters.map(({ flag, value }) => `${flag} "${value}"`).join(", ");
@@ -36,7 +36,7 @@ import { killTarget } from "../host/platform.mjs";
36
36
  /** @typedef {{prompt: string|null, stdout: string, stderr: string}} PathSet */
37
37
  /** @typedef {{id: string, pid: number, processGroupId: number|null, processStartToken: string|null, harness: string, runtimeId: string|null, runtimeFingerprint?: string, revision?: number, phase: string, promptPath: string|null, stdoutPath: string, stderrPath: string, startedAt: string, deadlineAt: string|null, updatedAt: string, closedAt: string|null, exitCode: number|null, signal: string|null, status: "active"|"closed"|"terminated", executable: string, snapshotPath?: string, usage?: Usage, usageEstimated?: boolean, costUsd?: number|null, costProvenance?: "priced", runId?: string, campaignId?: string, nodeId?: string, attempt?: number, workspace?: string, worktreeBranch?: string|null, worktreeBaseSha?: string|null, planPhase?: string, role?: "worker"|"judge", model?: string, reasoning?: string|null, sandbox?: string|null, continuationId?: string|null, continuationMode?: "fresh"|"reuse"|"rotate", session?: import("../harnesses/session-metrics.mjs").SessionLedger|null}} Invocation */
38
38
  /** @typedef {{pid: number|null, processGroupId?: number|null, processStartToken?: string|null}} InvocationProbe */
39
- /** @typedef {{child: ChildProcess, contract: ValidatedContract, node: ValidatedNode, state: NodeSnapshot, runtime: HarnessRuntime & {id: string|null}, cwd: string, paths: PathSet, phase: string, invocation: Invocation, startedAt: string, startedTicks: bigint, progressTicks: bigint, lastOutputAt: number, closed: boolean, exitCode: number|null, signal: string|null, spawnError: Error|null, terminating: Promise<void>|null, gateConfigPath: string, gateReleasePath: string, scopeBaseline?: unknown, scopeChecked?: boolean, scopeViolation?: boolean, resultMaterialization?: boolean, recoveryBaseline?: unknown, observeTimer?: ReturnType<typeof setInterval>, monitorOffset?: number, monitorParser?: import("../harnesses/session-metrics.mjs").SessionMetricsParser, lastEventCount?: number, observedOnce?: boolean, onClose?: (invocation: Invocation) => void, onInvocationUpdate?: (invocation: Invocation) => void, onProgress?: (state: NodeSnapshot) => void}} Job */
39
+ /** @typedef {{child: ChildProcess, contract: ValidatedContract, node: ValidatedNode, state: NodeSnapshot, runtime: HarnessRuntime & {id: string|null}, cwd: string, paths: PathSet, phase: string, invocation: Invocation, startedAt: string, startedTicks: bigint, progressTicks: bigint, lastOutputAt: number, closed: boolean, exitCode: number|null, signal: string|null, spawnError: Error|null, terminating: Promise<void>|null, gateConfigPath: string, gateReleasePath: string, scopeBaseline?: unknown, scopeChecked?: boolean, scopeViolation?: boolean, resultMaterialization?: boolean, recoveryBaseline?: unknown, observeTimer?: ReturnType<typeof setInterval>, monitorOffset?: number, monitorParser?: import("../harnesses/session-metrics.mjs").SessionMetricsParser, lastEventCount?: number, lastMonitorOffset?: number, observedOnce?: boolean, turnCapWarned?: boolean, onClose?: (invocation: Invocation) => void, onInvocationUpdate?: (invocation: Invocation) => void, onProgress?: (state: NodeSnapshot) => void}} Job */
40
40
  /** @typedef {{graceMs?: number, killGraceMs?: number, escalate?: boolean, runDir?: string, kill?: (pid: number, signal: string|number) => unknown, child?: ChildProcess|null}} TerminateOptions */
41
41
 
42
42
  const HERE = dirname(fileURLToPath(import.meta.url));
@@ -278,6 +278,14 @@ export async function terminateInvocation(invocation, options = {}) {
278
278
  */
279
279
  const SEAL_BEFORE_KILL_CODES = new Set(["wall_clock_timeout", "stall_timeout", "turn_limit"]);
280
280
 
281
+ /**
282
+ * How far into its request ceiling an attempt gets before it says so. Four
283
+ * fifths: late enough that an ordinary attempt never mentions it (measured
284
+ * 2026-09-20 over 200 completed claude worker turns, p90 was 83 of a 150
285
+ * default), early enough that the remaining fifth is still room to act in.
286
+ */
287
+ const TURN_CAP_WARN_FRACTION = 0.8;
288
+
281
289
  /**
282
290
  * How long a `SIGSTOP`ped process group is given to actually stop before the
283
291
  * seal begins. The stop is asynchronous; this bounded settle keeps the seal
@@ -434,8 +442,26 @@ export async function detectStalls(contract, running, onTimeout, onProgress, onB
434
442
  if (streaming) {
435
443
  const monitored = monitorInvocation(job);
436
444
  const events = monitored.turns + monitored.toolCalls;
437
- if (events !== job.lastEventCount || job.observedOnce !== true) {
445
+ // Transcript bytes the monitor consumed count as liveness beside the
446
+ // events, because a turn can transmit for a long time without finishing
447
+ // one. Codex meters progress as `turn.completed` plus tool calls, so a
448
+ // single long reasoning stretch -- streaming `item.completed` records
449
+ // that are neither -- advanced no counter and was killed as a stall
450
+ // while it was actively transmitting. This is not the mtime the module
451
+ // header rejects: mtime moves for a buffered harness that has written
452
+ // nothing a provider produced, and this branch is the streaming
453
+ // harnesses only, where new bytes on the transcript are the provider's
454
+ // own output and the monitor's offset only ever advances. A process
455
+ // that transmits forever without ending is still held by the wall clock
456
+ // and the turn cap below.
457
+ // An offset this job has never recorded is not growth: `observedOnce`
458
+ // below is what covers the first pass, and reading `undefined` as a
459
+ // change would make every first observation look like progress.
460
+ const consumed = job.monitorOffset ?? 0;
461
+ const grew = consumed !== (job.lastMonitorOffset ?? consumed);
462
+ if (events !== job.lastEventCount || grew || job.observedOnce !== true) {
438
463
  job.lastEventCount = events;
464
+ job.lastMonitorOffset = consumed;
439
465
  job.progressTicks = now;
440
466
  // `lastOutputAt` is the supervised controller's provider-progress
441
467
  // signal (scheduler.mjs): keep it advancing for an event that counts
@@ -450,6 +476,18 @@ export async function detectStalls(contract, running, onTimeout, onProgress, onB
450
476
  // (`turns` counts provider requests for claude, dsh and agy; codex
451
477
  // reports whole turns, so its cap is in effect a turn count.)
452
478
  const turnCap = job.node?.maxTurns ?? contract.maxTurns;
479
+ // The ceiling used to arrive only as the kill. `maxTurns` appeared once
480
+ // in the whole documentation set and not at all in the contract
481
+ // reference, so the author of a long-running contract raised
482
+ // `timeoutSec` and `stallTimeoutSec` -- everything they knew existed --
483
+ // and left this at its default; two Opus attempts at maximum effort
484
+ // were then cut mid-turn with `turn_limit`, after the cost was already
485
+ // paid. Said once per attempt as the ceiling comes into view, it is a
486
+ // decision the operator can still make.
487
+ if (typeof turnCap === "number" && job.turnCapWarned !== true && monitored.turns >= Math.floor(turnCap * TURN_CAP_WARN_FRACTION)) {
488
+ job.turnCapWarned = true;
489
+ process.stdout.write(`[node] ${nodeId} ${job.phase} · ${monitored.turns} of the attempt's maxTurns of ${turnCap} provider requests · raise maxTurns to give it more\n`);
490
+ }
453
491
  if (typeof turnCap === "number" && monitored.turns >= turnCap) {
454
492
  const limit = {
455
493
  code: "turn_limit",
@@ -23,7 +23,8 @@ import { attemptWorktreePath } from "../run/paths.mjs";
23
23
  import { canonicalWorkerResultText, isResultMaterializationInvocation, materializeAttemptResult, recoverWorkerResult } from "./result-file.mjs";
24
24
  import { checkPersistedWorkerScope, persistedScopeBoundary, reconcileAmbiguousWorkerRestart, resolveUnknownEffect } from "./scope.mjs";
25
25
  import { closePersistedInvocation, recoverOrphan, recoveryFromOverride } from "./recover.mjs";
26
- import { driveRun, readRunNodes } from "./scheduler.mjs";
26
+ import { driveRun } from "./scheduler.mjs";
27
+ import { readRunNodes } from "../run/node-store.mjs";
27
28
  import { emptyUsage, invocationCost, invocationUsage, persistRecoveryUsage } from "../run/usage.mjs";
28
29
  import { ensureTerminalEvent, hasDoneEvent, recordExecutionOverride, transition, writeNode } from "./state.mjs";
29
30
  import { errorMessage, excerpt } from "../util.mjs";
@@ -60,7 +60,18 @@ export const DISCOVERY_RUNTIME_DEFINITIONS = Object.freeze({
60
60
  costRank: 1,
61
61
  },
62
62
  "agy-gemini": { harness: "agy", model: "gemini-3.8-flash-low", vendor: "google", tier: 1, costRank: 1 },
63
+ // Three codex rows, because the harness declares three models and an
64
+ // account is entitled to only some of them: measured 2026-09-21 on the
65
+ // owner's ChatGPT account, plain `gpt-5.6` answers HTTP 400 while
66
+ // `gpt-5.6-sol` answers normally. One row meant `setup` could offer only the
67
+ // model that account cannot use, and the operator's only way out was to hand
68
+ // every contract its own catalogue through `--runtimes`. Declaration order is
69
+ // unchanged, so the composed default is still `codex-gpt`: which of the three
70
+ // an account can reach is not something this file can know, and the live
71
+ // preflight is what reports it per id.
63
72
  "codex-gpt": { harness: "codex", model: "gpt-5.6", vendor: "openai", tier: 2, costRank: 2 },
73
+ "codex-sol": { harness: "codex", model: "gpt-5.6-sol", vendor: "openai", tier: 2, costRank: 2 },
74
+ "codex-luna": { harness: "codex", model: "gpt-5.6-luna", vendor: "openai", tier: 2, costRank: 2 },
64
75
  "claude-sonnet": { harness: "claude", model: "claude-sonnet-5", vendor: "anthropic", tier: 2, costRank: 2 },
65
76
  });
66
77
 
@@ -37,14 +37,12 @@ import { delay, errorCode } from "../util.mjs";
37
37
  import { alreadyNotified, emitNodeAdvisories, notifyQueueFor, notifyQueuesByRun, renderCampaignHandoffSafely } from "./notify-queue.mjs";
38
38
  import { detectStalls, invocationAlive, terminateProcess } from "./process.mjs";
39
39
  import { transition, writeNode } from "./state.mjs";
40
- import { listNodeSnapshots, readNodeSnapshot } from "../run/node-store.mjs";
41
40
  import { render, renderFinalReport, writeFindingsArtifact } from "../report/final.mjs";
42
41
  import { operationNextState, providerReceipts, settleInvocation } from "../run/operations.mjs";
43
42
  import { appendUsageRecord, invocationCost, invocationUsage, recordInvocationUsage } from "../run/usage.mjs";
44
43
  import { captureNodeScopeBoundaries, checkWorkerScope, emptyScope } from "./scope.mjs";
45
44
  import { validateContractForLaunch } from "../campaign/chain.mjs";
46
- import { validateNodeSnapshot } from "../contract/snapshot.mjs";
47
- import { finalVerificationCommands, sharedVerificationCommands } from "../contract/final-verification.mjs";
45
+ import { finalVerificationCommands, gateProofTimeoutMs, sharedVerificationCommands } from "../contract/final-verification.mjs";
48
46
  import { startJudge, startWorker } from "./dispatch.mjs";
49
47
  import { assertEnvironmentReady, captureRunIdentity, createRunMetadata, serializableContract, statesFingerprint } from "./run-identity.mjs";
50
48
  import { blockDependents, runtimeAssignments } from "./assignment.mjs";
@@ -120,7 +118,7 @@ export function nodeBudgetBasisMs(contract, node) {
120
118
  // finalVerification set with each.
121
119
  const attemptMs = packetMs + sharedMs;
122
120
  const candidateMs = attemptMs + finalMs;
123
- const gateTimeoutMs = Math.max(1_000, Math.min(defaultTimeoutMs, 120_000));
121
+ const gateTimeoutMs = gateProofTimeoutMs(node, contract);
124
122
  const commandProofs = (node.definitionOfDone ?? []).filter((item) => item.proof?.kind === "command").length;
125
123
  return defaultTimeoutMs + attemptMs + candidateMs + commandProofs * gateTimeoutMs + finalMs;
126
124
  }
@@ -238,6 +236,19 @@ export async function runContract(contractPath, options = {}) {
238
236
  }
239
237
  const lock = acquireLock(runDir);
240
238
  try {
239
+ // `contract.json` is written before anything else claims a name outside
240
+ // this directory, because `cancel` is the only verb that releases those
241
+ // names and it reads the contract from here. Everything below can fail or
242
+ // be killed -- `runtimeAssignments` probes providers, `captureRunIdentity`
243
+ // shells out to git, `createRunRef` claims `refs/faberun/<id>/run` -- and a
244
+ // launch that died between the ref and this write used to leave an
245
+ // occupied ref plus a run directory `cancel` could not parse: the operator
246
+ // was refused the directory, deleted it, was then refused the ref, and had
247
+ // no single verb for either. Written first, the directory is always
248
+ // cancellable from the instant it exists.
249
+ mkdirSync(join(runDir, "nodes"), { recursive: true });
250
+ mkdirSync(join(runDir, "logs"), { recursive: true });
251
+ writeJsonAtomic(join(runDir, "contract.json"), serializableContract(contract));
241
252
  const runtimePlan = await runtimeAssignments(contract);
242
253
  const scopeBoundaries = captureNodeScopeBoundaries(contract);
243
254
  const sourceIdentity = await captureRunIdentity(contract, scopeBoundaries);
@@ -245,9 +256,6 @@ export async function runContract(contractPath, options = {}) {
245
256
  lock.assert();
246
257
  const runsDir = runsRoot(contract.cwd);
247
258
  const campaign = resolveCampaign(runsDir, contract.campaignId);
248
- mkdirSync(join(runDir, "nodes"), { recursive: true });
249
- mkdirSync(join(runDir, "logs"), { recursive: true });
250
- writeJsonAtomic(join(runDir, "contract.json"), serializableContract(contract));
251
259
  writeJsonAtomic(join(runDir, "judge.schema.json"), JUDGE_SCHEMA);
252
260
  writeJsonAtomic(join(runDir, "run.json"), createRunMetadata(lock, sourceIdentity, {}, integrationRef));
253
261
  registerRun(campaign.path, contract.id);
@@ -755,20 +763,3 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
755
763
  }
756
764
  return { runDir, states, ok: failed.length === 0 };
757
765
  }
758
-
759
- /**
760
- * @param {string} runDir
761
- * @param {ValidatedContract} contract
762
- * @returns {NodeSnapshot[]}
763
- */
764
- export function readRunNodes(runDir, contract) {
765
- const names = listNodeSnapshots(runDir);
766
- const expected = new Map(contract.nodes.map((node) => [`${node.id}.json`, node]));
767
- for (const name of names) if (!expected.has(name)) throw new TypeError(`unexpected persisted node snapshot ${name}`);
768
- return contract.nodes.map((node) => {
769
- const name = `${node.id}.json`;
770
- if (!names.includes(name)) throw new TypeError(`missing persisted node snapshot ${name}`);
771
- return validateNodeSnapshot(readNodeSnapshot(runDir, node.id), node);
772
- });
773
- }
774
-
@@ -15,7 +15,7 @@ import { SETTLED } from "./prompts.mjs";
15
15
  import { appendTransitionEvent, recordExecutionOverride, transition, writeNode } from "./state.mjs";
16
16
  import { attemptWorkspace } from "../repo/worktree.mjs";
17
17
 
18
- import { errorCode, errorMessage, excerpt, isContained } from "../util.mjs";
18
+ import { errorCode, errorMessage, excerpt, isContained, shellWords } from "../util.mjs";
19
19
  import { executeControllerVerification } from "./verify.mjs";
20
20
  import { providerReceiptsFromInvocationTail, settleInvocation } from "../run/operations.mjs";
21
21
  import { readJson } from "../run/store.mjs";
@@ -108,10 +108,17 @@ export function workerScope(taskPacket) {
108
108
 
109
109
  /**
110
110
  * Everything a node's own proofs name: a Definition of Done `path` proof's
111
- * path, the words of a `command` proof (a command proof carries a display
112
- * string, not an argv, so a quoted path holding a space is not recovered), the
113
- * argv of the verification entry a `verification` proof references, and the
114
- * argv of every verification command the packet declares.
111
+ * path, the words of a `command` proof, the argv of the verification entry a
112
+ * `verification` proof references, and the argv of every verification command
113
+ * the packet declares.
114
+ *
115
+ * A command proof carries a display string, not an argv, so its words are
116
+ * recovered with `shellWords` rather than a whitespace split -- the split lost
117
+ * exactly what the shell it runs under preserves. Measured 2026-09-22:
118
+ * `spawn(ref, {shell: true})` hands the whole string to `sh -c`, which groups
119
+ * `"b c"` into one argument, while the split here cut it into `"b` and `c"`,
120
+ * so a proof naming a real path with a space cited two fragments that matched
121
+ * no file and the write it excused read as unexpected.
115
122
  *
116
123
  * @param {ValidatedNode} node
117
124
  * @returns {ProofCitation[]}
@@ -129,7 +136,7 @@ function proofCitations(node) {
129
136
  if (proof.kind === "path") {
130
137
  citations.push({ tokens: [proof.ref], cwd: ".", literal: true, citation: `${item.id} path proof` });
131
138
  } else if (proof.kind === "command") {
132
- citations.push({ tokens: proof.ref.split(/\s+/u), cwd: ".", literal: false, citation: `${item.id} command proof` });
139
+ citations.push({ tokens: shellWords(proof.ref), cwd: ".", literal: false, citation: `${item.id} command proof` });
133
140
  } else {
134
141
  const command = commands[Number.parseInt(proof.ref, 10)];
135
142
  if (command) citations.push({ tokens: command.argv, cwd: command.cwd ?? ".", literal: false, citation: `${item.id} verification[${proof.ref}] proof` });
package/src/host/home.mjs CHANGED
@@ -154,3 +154,41 @@ function parseVersion(text) {
154
154
  if (!match) return null;
155
155
  return [Number(match[1]), Number(match[2]), Number(match[3])];
156
156
  }
157
+
158
+ /**
159
+ * How old a cached update check may be and still be repeated as a fact about
160
+ * the world. The banner and the notification message read the cache and never
161
+ * the network -- a version hint must not put a request in front of every
162
+ * command -- so the cache is only ever as fresh as the last
163
+ * `faberun update --check` somebody remembered to run. Measured 2026-09-22 on
164
+ * the owner's machine: `update-check.json` was five days old and still named
165
+ * 0.10.0 as the latest release while 0.19.0 was being cut.
166
+ *
167
+ * A week is the bound, not a refresh: past it the hint stops being shown
168
+ * rather than being re-fetched, because a stale claim asserted with authority
169
+ * is worse than no claim. The command that does reach the network is
170
+ * unaffected, and the file keeps its `checkedAt` either way.
171
+ */
172
+ export const UPDATE_CHECK_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
173
+
174
+ /**
175
+ * The newer release a still-fresh cached check names, or null: no cache, a
176
+ * cache older than the window, a cache timestamped in the future (a record
177
+ * that would otherwise never expire), or a `latest` that is not newer than
178
+ * what is running. The one home of the "is there an update to mention" rule,
179
+ * so the banner and the notification message cannot come to disagree.
180
+ *
181
+ * @param {string} home
182
+ * @param {string} version the version actually running
183
+ * @param {number} [now] epoch milliseconds, injected by tests
184
+ * @returns {string|null}
185
+ */
186
+ export function availableUpdate(home, version, now = Date.now()) {
187
+ const check = readUpdateCheck(home);
188
+ if (!check) return null;
189
+ const checkedAt = Date.parse(check.checkedAt);
190
+ if (!Number.isFinite(checkedAt)) return null;
191
+ const age = now - checkedAt;
192
+ if (age < 0 || age > UPDATE_CHECK_MAX_AGE_MS) return null;
193
+ return compareVersions(check.latest, version) > 0 ? check.latest : null;
194
+ }
@@ -50,7 +50,7 @@ import { availabilityKey, readAvailability, recordAvailability } from "../run/av
50
50
  /** @typedef {import("../harnesses/index.mjs").CapabilityRequirements} CapabilityRequirements */
51
51
  /** @typedef {import("../harnesses/index.mjs").ProbeResult} ProbeResult */
52
52
  /** @typedef {import("../engine/runtime-discovery.mjs").RuntimeAvailability} RuntimeAvailability */
53
- /** @typedef {Map<string, {runtime: RuntimeSnapshot, requiredCapabilitySets: CapabilityRequirements[]}>} ReachableRuntimes */
53
+ /** @typedef {Map<string, {runtime: RuntimeSnapshot, requiredCapabilitySets: CapabilityRequirements[], routed: boolean}>} ReachableRuntimes */
54
54
  /** @typedef {{name: string, ok: boolean, advisory: boolean, detail: string}} EnvCheck */
55
55
  /** @typedef {{schemaVersion: number, ok: boolean, checks: EnvCheck[]}} EnvReport */
56
56
 
@@ -169,9 +169,19 @@ export function checkWorktree(cwd, requireClean) {
169
169
  }
170
170
 
171
171
  /**
172
- * Every runtime the run can route to — initial and failover must resolve to
173
- * a binary that exists and reports a version. A version-less runtime is fatal
174
- * up front because a resume refuses a runtime whose probe came back null.
172
+ * Every runtime the run committed to — a node's or a default's named runtime,
173
+ * and the failover target it declares must resolve to a binary that exists
174
+ * and reports a version. A version-less runtime is fatal up front because a
175
+ * resume refuses a runtime whose probe came back null.
176
+ *
177
+ * A runtime that is merely a candidate is held to a weaker rule: at least one
178
+ * of them has to resolve. A role that names no runtime lets availability
179
+ * discovery choose at dispatch, so every catalogue entry is reachable without
180
+ * any of them being chosen, and demanding a binary for each of them refused a
181
+ * run over a harness it would never have started. Measured 2026-09-21 on the
182
+ * owner's machine: `codex reported no version` blocked a launch whose work was
183
+ * routed to `zcode`. What is still fatal is a catalogue where nothing resolves
184
+ * — then there is no route at all, and discovery has nothing to choose.
175
185
  *
176
186
  * @param {ReachableRuntimes} runtimes
177
187
  * @param {Record<string, string|null>} harnessVersions
@@ -182,20 +192,34 @@ export function checkRuntimeBinaries(runtimes, harnessVersions, cwd = ".") {
182
192
  /** @type {string[]} */
183
193
  const problems = [];
184
194
  /** @type {string[]} */
195
+ const unavailableCandidates = [];
196
+ /** @type {string[]} */
185
197
  const resolved = [];
186
- for (const [id, { runtime }] of runtimes) {
198
+ let candidates = 0;
199
+ let candidatesResolved = 0;
200
+ for (const [id, { runtime, routed }] of runtimes) {
187
201
  // The harness owns the resolution: a per-runtime executable, an
188
202
  // FABERUN_*_BIN override, and each harness's default binary all
189
203
  // land here, and a relative path belongs to the run cwd, not to ours.
190
204
  const executable = getHarness(runtime.harness).executable(runtime);
191
205
  const found = findExecutable(executable.includes("/") || executable.includes("\\") ? resolve(cwd, executable) : executable);
192
206
  const version = harnessVersions[id] ?? null;
193
- if (found === null) problems.push(`${id}: ${executable} not found on PATH`);
194
- else if (version === null) problems.push(`${id}: ${executable} reported no version`);
195
- else resolved.push(`${id} ${version}`);
207
+ if (!routed) candidates += 1;
208
+ const problem = found === null
209
+ ? `${id}: ${executable} not found on PATH`
210
+ : version === null ? `${id}: ${executable} reported no version` : null;
211
+ if (problem === null) {
212
+ resolved.push(`${id} ${version}`);
213
+ if (!routed) candidatesResolved += 1;
214
+ } else if (routed) problems.push(problem);
215
+ else unavailableCandidates.push(problem);
196
216
  }
197
217
  if (problems.length) return fail("runtime binaries", problems.join(" · "));
198
- return pass("runtime binaries", resolved.length ? resolved.join(" · ") : "no routed runtime");
218
+ if (candidates > 0 && candidatesResolved === 0) {
219
+ return fail("runtime binaries", `no catalogue runtime resolves, so availability discovery has nothing to choose: ${unavailableCandidates.join(" · ")}`);
220
+ }
221
+ const aside = unavailableCandidates.length ? ` · not candidates: ${unavailableCandidates.join(" · ")}` : "";
222
+ return pass("runtime binaries", resolved.length ? `${resolved.join(" · ")}${aside}` : "no routed runtime");
199
223
  }
200
224
 
201
225
  /**
@@ -360,10 +384,10 @@ export function timeVerificationCommands(contract, probes = {}) {
360
384
  * can take only one hop, and its reachable set stops at B.
361
385
  *
362
386
  * @param {ValidatedContract} contract
363
- * @returns {Map<string, {runtime: RuntimeSnapshot, requiredCapabilitySets: import("../harnesses/index.mjs").CapabilityRequirements[]}>}
387
+ * @returns {ReachableRuntimes}
364
388
  */
365
389
  export function reachableRuntimes(contract) {
366
- /** @type {Map<string, {runtime: RuntimeSnapshot, requiredCapabilitySets: import("../harnesses/index.mjs").CapabilityRequirements[]}>} */
390
+ /** @type {ReachableRuntimes} */
367
391
  const runtimes = new Map();
368
392
  for (const node of contract.nodes) {
369
393
  for (const role of /** @type {("worker"|"judge")[]} */ (["worker", ...(node.gate.enabled ? ["judge"] : [])])) {
@@ -384,14 +408,22 @@ export function reachableRuntimes(contract) {
384
408
  ? [runtime.requiredCapabilities, node.gate.requiredCapabilities]
385
409
  : [runtime.requiredCapabilities, node.requiredCapabilities];
386
410
  const requiredCapabilitySets = required.filter((item) => item !== undefined);
387
- addRuntimeRequirement(runtimes, runtime, requiredCapabilitySets);
411
+ if (explicit) addRuntimeRequirement(runtimes, runtime, requiredCapabilitySets);
388
412
  const current = { node, role, runtimeId: /** @type {string} */ (runtime.id) };
389
413
  for (const fallbackRuntime of failoverTargets(contract, current)) {
390
- addRuntimeRequirement(runtimes, fallbackRuntime, requiredCapabilitySets);
414
+ addRuntimeRequirement(runtimes, fallbackRuntime, requiredCapabilitySets, Boolean(explicit));
391
415
  }
392
416
  if (!explicit) {
417
+ // Nothing names a runtime for this role, so availability discovery
418
+ // picks one at dispatch and every catalogue entry is a candidate --
419
+ // including the stand-in above, which was chosen as "the first entry"
420
+ // and is no more routed than the rest. They are reachable, so their
421
+ // capabilities still have to hold, but none of them is a runtime this
422
+ // run committed to: requiring a binary for each turned one absent
423
+ // harness into a refusal of a run that would never have used it. The
424
+ // loop covers the stand-in as well -- it is one of the keys.
393
425
  for (const candidate of Object.keys(contract.runtimes)) {
394
- addRuntimeRequirement(runtimes, runtimeSnapshot(contract, candidate), requiredCapabilitySets);
426
+ addRuntimeRequirement(runtimes, runtimeSnapshot(contract, candidate), requiredCapabilitySets, false);
395
427
  }
396
428
  }
397
429
  }