faberun 0.22.0 → 0.23.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,13 +10,15 @@ import { readFileSync, writeFileSync } from "node:fs";
10
10
  import { resolve } from "node:path";
11
11
  import { fileURLToPath } from "node:url";
12
12
  import { COMMAND_OPTIONS } from "../cli.mjs";
13
+ import { METRICS_MANUAL } from "../campaign/metrics-command.mjs";
13
14
  import CAMPAIGN_OPERATIONS from "./campaign.mjs";
14
15
  import SEAT_OPERATIONS from "./seat.mjs";
15
16
  import CONTRACT_OPERATIONS from "./contract.mjs";
16
17
  import SKILLS_OPERATIONS from "./skills.mjs";
17
18
  import SPEC_OPERATIONS from "./spec.mjs";
18
19
 
19
- /** @typedef {{type: "string"|"boolean", multiple?: boolean}} FlagSpec */
20
+ /** @typedef {{value: string, effect: string, default: string}} ManualFlag */
21
+ /** @typedef {{type: "string"|"boolean", multiple?: boolean, manual?: ManualFlag}} FlagSpec */
20
22
  /** @typedef {{flags?: Record<string, FlagSpec>, operations?: Record<string, Record<string, FlagSpec>>}} VerbSurface */
21
23
  /** @typedef {{verbs: Record<string, VerbSurface>}} Surface */
22
24
 
@@ -51,7 +53,12 @@ const CONTAINER_OPERATIONS = {
51
53
  export function collectSurface() {
52
54
  /** @type {Record<string, VerbSurface>} */
53
55
  const verbs = {};
54
- for (const [verb, flags] of Object.entries(COMMAND_OPTIONS)) verbs[verb] = { flags };
56
+ for (const [verb, flags] of Object.entries(COMMAND_OPTIONS)) {
57
+ const manualFlags = verb === "metrics"
58
+ ? Object.fromEntries(Object.entries(flags).map(([name, spec]) => [name, { ...spec, manual: METRICS_MANUAL[name] }]))
59
+ : flags;
60
+ verbs[verb] = { flags: manualFlags };
61
+ }
55
62
  if (verbs.supervise) verbs.supervise.operations = { campaign: CAMPAIGN_OPERATIONS.supervise };
56
63
  for (const [verb, operations] of Object.entries(CONTAINER_OPERATIONS)) verbs[verb] = { operations };
57
64
  return { verbs };
@@ -291,9 +298,10 @@ function buildRows(flags, existingRows) {
291
298
  }
292
299
  return names.map((name) => {
293
300
  const existing = existingRows.get(name);
294
- const value = existing ? existing.value : "<value>";
295
- const effect = existing ? existing.effect : "";
296
- const fallback = existing ? existing.default : "—";
301
+ const manual = flags[name].manual;
302
+ const value = existing?.value && existing.value !== "<value>" ? existing.value : manual?.value ?? existing?.value ?? "<value>";
303
+ const effect = existing?.effect || manual?.effect || "";
304
+ const fallback = existing?.default && existing.default !== "—" ? existing.default : manual?.default ?? existing?.default ?? "—";
297
305
  return `| \`--${name}\` | ${value} | ${effect} | ${fallback} |`;
298
306
  });
299
307
  }
package/src/cli.mjs CHANGED
@@ -39,7 +39,7 @@ import { updateCommand } from "./cli/update.mjs";
39
39
  import { contractCli, validateContractFile } from "./cli/contract.mjs";
40
40
  import { specCli } from "./cli/spec.mjs";
41
41
  import { planCli } from "./cli/plan.mjs";
42
- import { METRICS_OPTIONS, renderCampaignMetrics } from "./campaign/metrics.mjs";
42
+ import { METRICS_OPTIONS, renderCampaignMetrics } from "./campaign/metrics-command.mjs";
43
43
  import { runContract } from "./engine/scheduler.mjs";
44
44
  import { resumeRun } from "./engine/resume.mjs";
45
45
  import { cancelRun } from "./engine/cancel.mjs";
@@ -259,13 +259,14 @@ function accountDelegation(runtimeId, runtime, envelope, priced, observation) {
259
259
  nodeId,
260
260
  role: "worker",
261
261
  runtimeId,
262
+ harness: runtime.harness,
262
263
  model: runtime.model,
263
264
  usage: envelope.usage,
264
265
  costUsd: priced.costUsd,
265
266
  costProvenance: priced.costProvenance,
266
267
  startedAt: observation.startedAt,
267
268
  closedAt: new Date().toISOString(),
268
- }));
269
+ }), { runtime });
269
270
  }
270
271
 
271
272
  /** @param {string} reason @returns {BulkReadResult} */
@@ -50,7 +50,7 @@ import { transition, writeNode } from "./state.mjs";
50
50
  * `dispatched`: a judge is running (or failed to start and the node is already
51
51
  * marked failed).
52
52
  *
53
- * @typedef {{kind: "rejected", verdict: JudgeVerdict}|{kind: "settle", gate: JudgeVerdict}|{kind: "dispatched"}} JudgeRound
53
+ * @typedef {{kind: "rejected", verdict: JudgeVerdict}|{kind: "settle", gate: JudgeVerdict|null}|{kind: "dispatched"}} JudgeRound
54
54
  */
55
55
  /** @typedef {import("./prompts.mjs").JudgeVerdict} JudgeVerdict */
56
56
 
@@ -439,6 +439,28 @@ export function startResultMaterialization(contract, node, state, runDir, runnin
439
439
  transition(runDir, state, "failed", { phase: "worker", error: { code: "result_materialization_failed", message: errorMessage(error) } }, lock);
440
440
  }
441
441
  }
442
+ /**
443
+ * The mechanical half of the gate, for a node whose gate is disabled: its
444
+ * Definition of Done command and path proofs still run, and no judge is
445
+ * asked. Measured 2026-09-23 on evidence-you-can-recompute: every node carried
446
+ * `gate: false`, and not one of its declared proofs ran.
447
+ *
448
+ * @param {ValidatedContract} contract
449
+ * @param {ValidatedNode} node
450
+ * @param {NodeSnapshot} state
451
+ * @returns {Promise<JudgeRound>}
452
+ */
453
+ export async function startMechanicalGate(contract, node, state) {
454
+ const { verdict, results } = await deterministicGate(
455
+ node,
456
+ attemptWorkspace(state) ?? contract.cwd,
457
+ false,
458
+ gateProofTimeoutMs(node, contract),
459
+ /** @type {import("../contract/index.mjs").VerificationState|null} */ (state.verification),
460
+ );
461
+ if (verdict.verdict === "fail") return { kind: "rejected", verdict };
462
+ return { kind: "settle", gate: results.length ? verdict : null };
463
+ }
442
464
  /**
443
465
  * Gate a completed worker: mechanical proofs gate first, the judge arbitrates
444
466
  * only judgment items and is skipped when the review mode is `none` or no
@@ -52,7 +52,7 @@ import {
52
52
  workerResultPath,
53
53
  } from "./result-file.mjs";
54
54
  import { canReuseResultEvidence, checkResultMaterializationScope, checkWorkerScope, recordScopeFinding, sourceWorkerRuntime } from "./scope.mjs";
55
- import { startJudge, startResultMaterialization } from "./dispatch.mjs";
55
+ import { startJudge, startMechanicalGate, startResultMaterialization } from "./dispatch.mjs";
56
56
  import { raiseNodeAttention, settleDone } from "./settle.mjs";
57
57
  import { applyRejection, applyVerificationFailure } from "./settle.mjs";
58
58
  import { emitNodeAdvisories } from "./notify-queue.mjs";
@@ -409,7 +409,7 @@ export async function finalizeClosedJobs(contract, runDir, states, closed, lock,
409
409
  error: envelope.error ?? null,
410
410
  nextState: operationNextState(state),
411
411
  });
412
- appendUsageRecord(runDir, state.invocations.find((invocation) => invocation.id === job.invocation.id));
412
+ appendUsageRecord(runDir, state.invocations.find((invocation) => invocation.id === job.invocation.id), { runtime: job.runtime });
413
413
  state.usage = invocationUsage(state);
414
414
  state.costUsd = invocationCost(state);
415
415
  // Checked before any other branch can act on how this invocation closed --
@@ -587,11 +587,10 @@ export async function finalizeClosedJobs(contract, runDir, states, closed, lock,
587
587
  continue;
588
588
  }
589
589
  if (job.scopeViolation) recordScopeFinding(runDir, state, lock);
590
- if (job.node.gate.enabled) {
591
- await applyJudgeRound(await startJudge(contract, job.node, state, runDir, running, workerResult, lock, states, campaignPath),
592
- contract, job.node, state, runDir, running, lock, states, campaignPath, workerResult);
593
- }
594
- else await settleDone(contract, job.node, state, runDir, lock, states, campaignPath, { phase: "complete", result: workerResult });
590
+ const round = job.node.gate.enabled
591
+ ? await startJudge(contract, job.node, state, runDir, running, workerResult, lock, states, campaignPath)
592
+ : await startMechanicalGate(contract, job.node, state);
593
+ await applyJudgeRound(round, contract, job.node, state, runDir, running, lock, states, campaignPath, workerResult);
595
594
  continue;
596
595
  }
597
596
  }
@@ -786,4 +785,3 @@ export async function applyInvalidWorkerResult(contract, node, state, runDir, ru
786
785
  * else there.
787
786
  *
788
787
  * @param {string} runDir @param {string} nodeId @returns {string} */
789
-
@@ -539,6 +539,7 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
539
539
  for (const job of jobs) envelopes.set(job.invocation.id, recordInvocationUsage(job, { accumulate: false }));
540
540
  for (const job of jobs) {
541
541
  const invocation = job.state.invocations?.find((item) => item.id === job.invocation.id) ?? job.invocation;
542
+ appendUsageRecord(runDir, invocation, { runtime: job.runtime, unknownReason: "invocation-killed" });
542
543
  const scopeOk = job.phase !== "worker" || checkWorkerScope(contract, runDir, job, lock);
543
544
  settleInvocation(runDir, invocation, {
544
545
  status: scopeOk ? "canceled" : "failed",
@@ -569,7 +570,7 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
569
570
  job.state.usage = invocationUsage(job.state);
570
571
  job.state.costUsd = invocationCost(job.state);
571
572
  const invocation = job.state.invocations?.find((item) => item.id === job.invocation.id) ?? job.invocation;
572
- appendUsageRecord(runDir, invocation);
573
+ appendUsageRecord(runDir, invocation, { runtime: job.runtime, unknownReason: "invocation-killed" });
573
574
  if (job.phase === "worker" && !checkWorkerScope(contract, runDir, job, lock)) {
574
575
  settleInvocation(runDir, invocation, {
575
576
  status: "failed",
@@ -46,7 +46,7 @@ import { verifyCandidateWorkspace } from "./verify.mjs";
46
46
  * (`gate.maxRevisions`, default 1) whether or not the gate reviews: a red
47
47
  * deterministic verification earns the same fresh attempt with the failure in
48
48
  * front of the worker that a judge rejection does. Measured 2026-09-20 in the
49
- * orchestration-arms campaign: with the budget behind `gate.enabled`, a node
49
+ * `docs/campaigns/orchestration-arms/`: with the budget behind `gate.enabled`, a node
50
50
  * under `gate: false` died on one timing test that flaked under load, and its
51
51
  * dependant with it, while the judged twin of the same node got its retry.
52
52
  * @param {ValidatedContract} contract @param {ValidatedNode} node @param {NodeSnapshot} state @param {string} runDir @param {Map<string, Job>|null} running @param {LockHandle} lock @param {Map<string, NodeSnapshot>} states @param {string} campaignPath @param {JudgeVerdict} verdict @param {{code: string, label: string, phase?: "worker"|"judge", message?: string, forceFresh?: boolean}} options */
@@ -65,18 +65,60 @@ export const MEASURE_SIDE_EFFECT_ENV_KEYS = [
65
65
  * @returns {{output: string, exitCode: number|null, truncated: boolean}}
66
66
  */
67
67
  export function runShellCapture(cwd, command, probes = {}) {
68
+ const { output, exitCode, truncated } = runShellFull(cwd, command, probes);
69
+ const bytes = Buffer.from(output, "utf8");
70
+ return {
71
+ output: truncated ? bytes.subarray(0, MEASURE_OUTPUT_CAP_BYTES).toString("utf8") : output,
72
+ exitCode,
73
+ truncated,
74
+ };
75
+ }
76
+
77
+ /**
78
+ * The shell run behind `runShellCapture`, returning the whole output: a
79
+ * pinned-test check has to see every test the runner reported, and the
80
+ * matching line can sit far past the 4 KiB a measurement keeps.
81
+ *
82
+ * @param {string} cwd
83
+ * @param {string} command
84
+ * @param {MeasureProbes} [probes]
85
+ * @returns {{output: string, exitCode: number|null, truncated: boolean}}
86
+ */
87
+ function runShellFull(cwd, command, probes = {}) {
68
88
  const run = probes.run ?? spawnSync;
69
89
  const env = { ...process.env };
70
90
  for (const key of MEASURE_SIDE_EFFECT_ENV_KEYS) delete env[key];
91
+ // A nested `node --test` that inherits NODE_TEST_CONTEXT stays a runner
92
+ // child and prints no result at all (measured 2026-09-21, AGENTS.md), so a
93
+ // proof run from inside a test runner would have nothing to show.
94
+ delete env.NODE_TEST_CONTEXT;
71
95
  const result = run(command, { shell: true, cwd, timeout: MEASURE_TIMEOUT_MS, encoding: "utf8", env });
72
96
  const combined = `${result.stdout ?? ""}${result.stderr ?? ""}`;
73
- const bytes = Buffer.from(combined, "utf8");
74
- const truncated = bytes.length > MEASURE_OUTPUT_CAP_BYTES;
75
- return {
76
- output: truncated ? bytes.subarray(0, MEASURE_OUTPUT_CAP_BYTES).toString("utf8") : combined,
77
- exitCode: result.status,
78
- truncated,
79
- };
97
+ return { output: combined, exitCode: result.status, truncated: Buffer.byteLength(combined, "utf8") > MEASURE_OUTPUT_CAP_BYTES };
98
+ }
99
+
100
+ /** `--test-name-pattern` in a shell command, quoted or bare, `=` or space. */
101
+ const NAME_PATTERN_FLAG = /--test-name-pattern(?:=|\s+)(?:"([^"]*)"|'([^']*)'|(\S+))/u;
102
+ /** A passing test as the spec reporter (`✔ name (1ms)`) or TAP (`ok 1 - name`) prints it. */
103
+ const PASSING_TEST_LINE = /^\s*(?:\u2714\s+|ok \d+ - )(.+?)(?:\s+\(\d[\d.]*m?s\))?\s*$/gmu;
104
+
105
+ /**
106
+ * Whether a `node --test --test-name-pattern=<p>` proof ran a test named by
107
+ * its pattern. Node exits 0 when the pattern matches nothing, printing only
108
+ * the file, so the exit code alone cannot tell a proof from its absence.
109
+ * Returns null when the command carries no pattern.
110
+ *
111
+ * @param {string} command
112
+ * @param {string} output
113
+ * @returns {boolean|null}
114
+ */
115
+ function patternMatchedATest(command, output) {
116
+ if (!/\bnode\b[^|;&]*--test\b/u.test(command)) return null;
117
+ const flag = NAME_PATTERN_FLAG.exec(command);
118
+ if (!flag) return null;
119
+ const pattern = new RegExp(flag[1] ?? flag[2] ?? flag[3], "u");
120
+ const plain = output.replace(/\u001b\[[0-9;]*m/gu, "");
121
+ return [...plain.matchAll(PASSING_TEST_LINE)].some((line) => pattern.test(line[1]) && !/\.m?[jt]s$/u.test(line[1].trim()));
80
122
  }
81
123
 
82
124
  /**
@@ -104,11 +146,12 @@ export function proveRequirements(cwd, requirements, probes = {}) {
104
146
  const proof = requirement.proof;
105
147
  if (!proof?.ref) continue;
106
148
  if (proof.kind === "command") {
107
- const { output, exitCode } = runShellCapture(cwd, proof.ref, probes);
108
- const detail = exitCode === 0
109
- ? "exit 0"
110
- : `exit ${exitCode ?? "no exit code (killed or never started)"}${output.trim() ? `: ${output.trim().slice(0, PROOF_DETAIL_CAP)}` : ""}`;
111
- results.push({ requirementId: requirement.id, kind: "command", ref: proof.ref, pass: exitCode === 0, detail });
149
+ const { output, exitCode } = runShellFull(cwd, proof.ref, probes);
150
+ const matched = exitCode === 0 ? patternMatchedATest(proof.ref, output) : null;
151
+ const detail = exitCode !== 0
152
+ ? `exit ${exitCode ?? "no exit code (killed or never started)"}${output.trim() ? `: ${output.trim().slice(0, PROOF_DETAIL_CAP)}` : ""}`
153
+ : matched === false ? "exit 0, but its --test-name-pattern matched no test: the proof ran nothing" : "exit 0";
154
+ results.push({ requirementId: requirement.id, kind: "command", ref: proof.ref, pass: exitCode === 0 && matched !== false, detail });
112
155
  continue;
113
156
  }
114
157
  if (proof.kind === "path") {
@@ -384,7 +384,10 @@ export function sealAttempt({ repo, path, baseSha, runId, nodeId, attempt, exclu
384
384
  // schedule it chose. The identity and signing overrides above are the
385
385
  // same argument: this commit answers to the factory, not to the repo's
386
386
  // conventions for human commits.
387
- "commit", "--no-verify", "-qm", `faberun ${runId} ${nodeId} attempt ${attempt}`,
387
+ // Conventional, so a campaign branch passes a PR's commitlint check
388
+ // (measured 2026-09-23: every `faberun <run> <node> attempt N` seal was
389
+ // refused); the run id moves to the body to keep the header short.
390
+ "commit", "--no-verify", "-q", "-m", `chore(faberun): seal ${nodeId} attempt ${attempt}`, "-m", `run ${runId}`,
388
391
  ]);
389
392
  }
390
393
  const sha = gitHead(path);
@@ -13,7 +13,7 @@
13
13
  */
14
14
 
15
15
  /** @typedef {import("../campaign/metrics.mjs").CampaignMetrics} CampaignMetrics */
16
- /** @typedef {import("../campaign/metrics.mjs").MetricsSources} MetricsSources */
16
+ /** @typedef {import("../campaign/metrics-command.mjs").MetricsSources} MetricsSources */
17
17
 
18
18
  /** Schema of the `--json` form; bumped when a consumer would have to change. */
19
19
  const METRICS_SCHEMA_VERSION = 2;
@@ -39,19 +39,30 @@ const HOURS_PRECISION = 10_000;
39
39
  * @returns {string}
40
40
  */
41
41
  export function renderMetricsReport(sources, metrics) {
42
+ const missingSources = sources.missingSources ?? [];
42
43
  const lines = [
43
44
  `[metrics] ${sources.campaignId} · ${sources.runIds.length} runs · ${sources.events.length} events · ${Object.keys(metrics).length} indicators`,
44
45
  ];
46
+ if (missingSources.length > 0) lines.push(`missing sources: ${missingSources.join(", ")}`);
45
47
  for (const [name, indicator] of Object.entries(metrics)) {
46
48
  const records = indicator.count === 1 ? "1 record" : `${indicator.count} records`;
47
49
  const unknown = typeof (/** @type {{unknownCount?: number}} */ (indicator).unknownCount) === "number"
48
- ? ` · ${(/** @type {{unknownCount: number}} */ (indicator)).unknownCount} unknown`
50
+ ? `${unknownCostSuffix(/** @type {{unknownCount: number, unknownFractionByReason?: Record<string, number>}} */ (indicator))}`
49
51
  : "";
50
- lines.push(`${name.padEnd(NAME_WIDTH)} ${indicator.direction.padEnd(11)} ${formatValue(name, indicator.value).padEnd(24)} · ${records}${unknown}`);
52
+ const missing = Array.isArray(indicator.missingSources) ? ` · missing ${indicator.missingSources.join(", ")}` : "";
53
+ lines.push(`${name.padEnd(NAME_WIDTH)} ${indicator.direction.padEnd(11)} ${formatValue(name, indicator.value).padEnd(24)} · ${records}${unknown}${missing}`);
51
54
  }
52
55
  return `${lines.join("\n")}\n`;
53
56
  }
54
57
 
58
+ /** @param {{unknownCount: number, unknownFractionByReason?: Record<string, number>}} indicator @returns {string} */
59
+ function unknownCostSuffix(indicator) {
60
+ const reasons = Object.entries(indicator.unknownFractionByReason ?? {})
61
+ .map(([reason, fraction]) => `${reason}=${fraction}`)
62
+ .join(", ");
63
+ return ` · ${indicator.unknownCount} unknown${reasons ? ` (${reasons})` : ""}`;
64
+ }
65
+
55
66
  /**
56
67
  * The machine-readable form, one JSON line like `preflight --json`.
57
68
  *
@@ -60,11 +71,13 @@ export function renderMetricsReport(sources, metrics) {
60
71
  * @returns {string}
61
72
  */
62
73
  export function renderMetricsJson(sources, metrics) {
74
+ const missingSources = sources.missingSources ?? [];
63
75
  return `${JSON.stringify({
64
76
  schemaVersion: METRICS_SCHEMA_VERSION,
65
77
  campaignId: sources.campaignId,
66
78
  runs: sources.runIds.length,
67
79
  events: sources.events.length,
80
+ missingSources,
68
81
  indicators: metrics,
69
82
  })}\n`;
70
83
  }
package/src/run/lock.mjs CHANGED
@@ -17,7 +17,7 @@ import { closeSync, fsyncSync, linkSync, mkdirSync, openSync, readFileSync, rena
17
17
  import { hostname } from "node:os";
18
18
  import { join } from "node:path";
19
19
  import { fsyncDirectory } from "./store.mjs";
20
- import { errorCode } from "../util.mjs";
20
+ import { errorCode, sleepSync } from "../util.mjs";
21
21
 
22
22
  const LOCK_FILE = "controller.lock";
23
23
  const TAKEOVER_ATTEMPTS = 20;
@@ -84,7 +84,7 @@ export function processStartToken(pid) {
84
84
  // and report nothing for a pid that is already alive. One short retry
85
85
  // closes that window; a still-empty answer stays null.
86
86
  if (!pidAlive(pid)) return null;
87
- sleepMs(20);
87
+ sleepSync(20);
88
88
  const retried = psStartTime(pid);
89
89
  return retried.length > 0 ? retried : null;
90
90
  }
@@ -106,16 +106,6 @@ function psStartTime(pid) {
106
106
  }
107
107
  }
108
108
 
109
- /**
110
- * Block without a timer so the synchronous darwin retry above can wait out the
111
- * spawn race. The duration is tiny and bounded, so blocking the loop is safe.
112
- *
113
- * @param {number} milliseconds
114
- */
115
- function sleepMs(milliseconds) {
116
- Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);
117
- }
118
-
119
109
  /** @param {number|null|undefined} pid @returns {boolean} */
120
110
  export function pidAlive(pid) {
121
111
  if (!Number.isInteger(pid) || /** @type {number} */ (pid) <= 0) return false;
package/src/run/store.mjs CHANGED
@@ -14,9 +14,19 @@ import {
14
14
  } from "node:fs";
15
15
  import { randomUUID } from "node:crypto";
16
16
  import { dirname, join } from "node:path";
17
- import { errorCode } from "../util.mjs";
17
+ import { errorCode, sleepSync } from "../util.mjs";
18
18
 
19
19
  const BOOTSTRAP_FILE = "bootstrap.json";
20
+ // Windows refuses to replace a file another process holds open: MoveFileEx
21
+ // answers EPERM (or EACCES, EBUSY) while a reader, an indexer or a virus
22
+ // scanner has the destination, where a POSIX rename replaces it regardless.
23
+ // Measured 2026-09-23 on Windows 11 with node 24: one process polling a file
24
+ // with readFileSync failed 882 of 2190 atomic writes onto it, and on CI that
25
+ // killed controllers mid-run (a heartbeat write threw out of driveRun). The
26
+ // retry and its bounds are graceful-fs's: back off 10 ms more per attempt up to
27
+ // 100 ms, and give up after 60 s.
28
+ const WINDOWS_RENAME_BUSY = new Set(["EPERM", "EACCES", "EBUSY"]);
29
+ const RENAME_RETRY_BUDGET_MS = 60_000;
20
30
  const JSONL_RECOVERY_TAIL_BYTES = 64 * 1024;
21
31
 
22
32
  /**
@@ -51,7 +61,7 @@ export function writeTextAtomic(path, text) {
51
61
  } finally {
52
62
  closeSync(fd);
53
63
  }
54
- renameSync(temporary, path);
64
+ renameReplacing(temporary, path);
55
65
  fsyncDirectory(dirname(path));
56
66
  committed = true;
57
67
  } catch (error) {
@@ -68,6 +78,24 @@ export function writeTextAtomic(path, text) {
68
78
  }
69
79
  }
70
80
 
81
+ /**
82
+ * @param {string} from
83
+ * @param {string} to
84
+ */
85
+ function renameReplacing(from, to) {
86
+ const deadline = Date.now() + RENAME_RETRY_BUDGET_MS;
87
+ for (let backoff = 0; ; backoff = Math.min(backoff + 10, 100)) {
88
+ try {
89
+ renameSync(from, to);
90
+ return;
91
+ } catch (error) {
92
+ const busy = process.platform === "win32" && WINDOWS_RENAME_BUSY.has(errorCode(error) ?? "");
93
+ if (!busy || Date.now() >= deadline) throw error;
94
+ sleepSync(backoff);
95
+ }
96
+ }
97
+ }
98
+
71
99
  /**
72
100
  * @param {string} path
73
101
  * @param {unknown} value
package/src/run/usage.mjs CHANGED
@@ -15,7 +15,7 @@ import { liveUsage } from "../harnesses/session-metrics.mjs";
15
15
  import { priceUsage } from "../engine/process.mjs";
16
16
  import { readBoundedTail, sessionLedger } from "../engine/transcript.mjs";
17
17
  import { writeNode } from "../engine/state.mjs";
18
- import { normalizeProviderResult } from "../harnesses/index.mjs";
18
+ import { harnessCapabilities, normalizeProviderResult } from "../harnesses/index.mjs";
19
19
 
20
20
  // `priceUsage` is defined beside `invocationResult`, the second source point,
21
21
  // and re-exported here so the ledger's public surface is unchanged. This module
@@ -31,6 +31,15 @@ export { priceUsage };
31
31
  /** @typedef {ProviderEnvelope & {costProvenance?: "priced"}} PricedEnvelope */
32
32
  /** @typedef {{kind: "adopted"|"rejudge"|"restart"|"reconciled"|"exhausted"|"stalled", phase?: "worker"|"judge", result?: unknown, usage?: Usage, costUsd?: number|null, costProvenance?: "priced", exhaustedUntil?: string|null, error?: {code: string, message: string}|null, invocationId?: string, reason?: string}} RecoveryOutcome */
33
33
  /** @typedef {import("../contract/index.mjs").Usage} Usage */
34
+ /** @typedef {"no-usage-stream"|"model-not-priced"|"provider-reported-nothing"|"invocation-killed"} UnknownCostReason */
35
+
36
+ /** The only reasons a newly-written unknown cost may carry. `legacy` is read-only compatibility for older ledgers. */
37
+ export const UNKNOWN_COST_REASONS = Object.freeze([
38
+ "no-usage-stream",
39
+ "model-not-priced",
40
+ "provider-reported-nothing",
41
+ "invocation-killed",
42
+ ]);
34
43
 
35
44
  /**
36
45
  * Extract the invocation's provider envelope from the bounded transcript tail
@@ -144,10 +153,15 @@ export function usageRecordIds(runDir) {
144
153
  *
145
154
  * @param {string} runDir
146
155
  * @param {Invocation|undefined|null} invocation
156
+ * @param {{runtime?: {harness?: string, capabilities?: {usage?: boolean}}, unknownReason?: UnknownCostReason}} [options]
147
157
  */
148
- export function appendUsageRecord(runDir, invocation) {
158
+ export function appendUsageRecord(runDir, invocation, options = {}) {
149
159
  if (!invocation?.id || usageRecordIds(runDir).has(invocation.id)) return;
150
160
  const usage = /** @type {Usage} */ (invocation.usage ?? { inputTokens: null, outputTokens: null, cacheReadInputTokens: null });
161
+ const costProvenance = invocation.costProvenance ?? (typeof invocation.costUsd === "number" ? "provider" : "unknown");
162
+ const unknownReason = costProvenance === "unknown"
163
+ ? unknownCostReason(invocation, usage, options)
164
+ : undefined;
151
165
  appendJsonl(join(runDir, USAGE_LOG_NAME), {
152
166
  invocationId: invocation.id,
153
167
  runId: invocation.runId ?? basename(runDir),
@@ -162,12 +176,51 @@ export function appendUsageRecord(runDir, invocation) {
162
176
  costUsd: typeof invocation.costUsd === "number" ? invocation.costUsd : null,
163
177
  // A persisted `priced` marker wins; otherwise the pre-Phase-4 rule applies
164
178
  // unchanged: a reported number is `provider`, absence is `unknown`.
165
- costProvenance: invocation.costProvenance ?? (typeof invocation.costUsd === "number" ? "provider" : "unknown"),
179
+ costProvenance,
180
+ ...(unknownReason === undefined ? {} : { unknownReason }),
166
181
  session: invocation.session ?? null,
167
182
  startedAt: invocation.startedAt ?? null,
168
183
  finishedAt: invocation.closedAt ?? null,
169
184
  });
170
185
  }
186
+
187
+ /**
188
+ * Explain an unknown cost at the point where the invocation's counters and
189
+ * pricing have been resolved. A runtime can be supplied by the live engine;
190
+ * older or recovery callers fall back to the invocation's harness identity.
191
+ *
192
+ * @param {Invocation} invocation
193
+ * @param {Usage} usage
194
+ * @param {{runtime?: {harness?: string, capabilities?: {usage?: boolean}}, unknownReason?: UnknownCostReason}} [options]
195
+ * @returns {UnknownCostReason}
196
+ */
197
+ function unknownCostReason(invocation, usage, options = {}) {
198
+ if (options.unknownReason !== undefined) return options.unknownReason;
199
+ if (invocation.signal !== null && invocation.signal !== undefined || invocation.status === "terminated") {
200
+ return "invocation-killed";
201
+ }
202
+ const runtimeUsage = options.runtime?.capabilities?.usage;
203
+ if (runtimeUsage === false || (runtimeUsage === undefined && invocation.harness !== undefined && !harnessHasUsage(invocation.harness))) {
204
+ return "no-usage-stream";
205
+ }
206
+ if (!hasMeasuredUsage(usage)) return "provider-reported-nothing";
207
+ // A partial token stream cannot be called "nothing". It also cannot be
208
+ // priced because one or more counters are absent, so the model-not-priced
209
+ // bucket is the only remaining vocabulary that preserves the unknown cost.
210
+ return "model-not-priced";
211
+ }
212
+
213
+ /** @param {string} harness @returns {boolean} */
214
+ function harnessHasUsage(harness) {
215
+ try {
216
+ return harnessCapabilities({ harness }).usage;
217
+ } catch (error) {
218
+ // An unknown harness cannot prove that it emits usage; callers still get
219
+ // the provider-reported-nothing fallback rather than a guessed adapter.
220
+ void error;
221
+ return true;
222
+ }
223
+ }
171
224
  /**
172
225
  * Recovery can discover usage after the run synchronized its records. Attach
173
226
  * it to the authoritative invocation first, then write the updated usage
@@ -201,7 +254,11 @@ export async function persistRecoveryUsage(runDir, state, recovery, lock) {
201
254
  writeNode(runDir, state, lock);
202
255
  }
203
256
  const updated = state.invocations?.find((invocation) => invocation.id === current.id);
204
- if (updated) appendUsageRecord(runDir, updated);
257
+ if (updated) {
258
+ const killed = recovery.kind === "stalled"
259
+ || (recovery.kind === "restart" && /wall-clock budget|died without a completed stream/iu.test(recovery.reason ?? ""));
260
+ appendUsageRecord(runDir, updated, killed ? { unknownReason: "invocation-killed" } : {});
261
+ }
205
262
  }
206
263
  /**
207
264
  * @param {Usage|undefined} left
package/src/util.mjs CHANGED
@@ -275,3 +275,13 @@ export function shellWords(text) {
275
275
  if (current !== null) words.push(current);
276
276
  return words;
277
277
  }
278
+
279
+ /**
280
+ * Block the thread without a timer, for a synchronous path that has to wait
281
+ * out a short race. Callers keep the duration tiny and bounded.
282
+ *
283
+ * @param {number} milliseconds
284
+ */
285
+ export function sleepSync(milliseconds) {
286
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);
287
+ }