faberun 0.22.0 → 0.23.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/campaign/chain.mjs +21 -8
- package/src/campaign/index.mjs +70 -49
- package/src/campaign/journal.mjs +8 -0
- package/src/campaign/layout.mjs +1 -0
- package/src/campaign/ledger.mjs +270 -0
- package/src/campaign/metrics-command.mjs +232 -0
- package/src/campaign/metrics.mjs +234 -153
- package/src/campaign/record.mjs +12 -0
- package/src/cli/campaign.mjs +43 -3
- package/src/cli/manual.mjs +13 -5
- package/src/cli.mjs +1 -1
- package/src/engine/bulk-read.mjs +2 -1
- package/src/engine/dispatch.mjs +23 -1
- package/src/engine/lifecycle.mjs +6 -8
- package/src/engine/scheduler.mjs +2 -1
- package/src/engine/settle.mjs +1 -1
- package/src/plan/proof-run.mjs +55 -12
- package/src/repo/worktree.mjs +4 -1
- package/src/report/metrics-report.mjs +16 -3
- package/src/run/lock.mjs +2 -12
- package/src/run/store.mjs +30 -2
- package/src/run/usage.mjs +61 -4
- package/src/util.mjs +10 -0
package/src/cli/manual.mjs
CHANGED
|
@@ -10,13 +10,15 @@ import { readFileSync, writeFileSync } from "node:fs";
|
|
|
10
10
|
import { resolve } from "node:path";
|
|
11
11
|
import { fileURLToPath } from "node:url";
|
|
12
12
|
import { COMMAND_OPTIONS } from "../cli.mjs";
|
|
13
|
+
import { METRICS_MANUAL } from "../campaign/metrics-command.mjs";
|
|
13
14
|
import CAMPAIGN_OPERATIONS from "./campaign.mjs";
|
|
14
15
|
import SEAT_OPERATIONS from "./seat.mjs";
|
|
15
16
|
import CONTRACT_OPERATIONS from "./contract.mjs";
|
|
16
17
|
import SKILLS_OPERATIONS from "./skills.mjs";
|
|
17
18
|
import SPEC_OPERATIONS from "./spec.mjs";
|
|
18
19
|
|
|
19
|
-
/** @typedef {{
|
|
20
|
+
/** @typedef {{value: string, effect: string, default: string}} ManualFlag */
|
|
21
|
+
/** @typedef {{type: "string"|"boolean", multiple?: boolean, manual?: ManualFlag}} FlagSpec */
|
|
20
22
|
/** @typedef {{flags?: Record<string, FlagSpec>, operations?: Record<string, Record<string, FlagSpec>>}} VerbSurface */
|
|
21
23
|
/** @typedef {{verbs: Record<string, VerbSurface>}} Surface */
|
|
22
24
|
|
|
@@ -51,7 +53,12 @@ const CONTAINER_OPERATIONS = {
|
|
|
51
53
|
export function collectSurface() {
|
|
52
54
|
/** @type {Record<string, VerbSurface>} */
|
|
53
55
|
const verbs = {};
|
|
54
|
-
for (const [verb, flags] of Object.entries(COMMAND_OPTIONS))
|
|
56
|
+
for (const [verb, flags] of Object.entries(COMMAND_OPTIONS)) {
|
|
57
|
+
const manualFlags = verb === "metrics"
|
|
58
|
+
? Object.fromEntries(Object.entries(flags).map(([name, spec]) => [name, { ...spec, manual: METRICS_MANUAL[name] }]))
|
|
59
|
+
: flags;
|
|
60
|
+
verbs[verb] = { flags: manualFlags };
|
|
61
|
+
}
|
|
55
62
|
if (verbs.supervise) verbs.supervise.operations = { campaign: CAMPAIGN_OPERATIONS.supervise };
|
|
56
63
|
for (const [verb, operations] of Object.entries(CONTAINER_OPERATIONS)) verbs[verb] = { operations };
|
|
57
64
|
return { verbs };
|
|
@@ -291,9 +298,10 @@ function buildRows(flags, existingRows) {
|
|
|
291
298
|
}
|
|
292
299
|
return names.map((name) => {
|
|
293
300
|
const existing = existingRows.get(name);
|
|
294
|
-
const
|
|
295
|
-
const
|
|
296
|
-
const
|
|
301
|
+
const manual = flags[name].manual;
|
|
302
|
+
const value = existing?.value && existing.value !== "<value>" ? existing.value : manual?.value ?? existing?.value ?? "<value>";
|
|
303
|
+
const effect = existing?.effect || manual?.effect || "";
|
|
304
|
+
const fallback = existing?.default && existing.default !== "—" ? existing.default : manual?.default ?? existing?.default ?? "—";
|
|
297
305
|
return `| \`--${name}\` | ${value} | ${effect} | ${fallback} |`;
|
|
298
306
|
});
|
|
299
307
|
}
|
package/src/cli.mjs
CHANGED
|
@@ -39,7 +39,7 @@ import { updateCommand } from "./cli/update.mjs";
|
|
|
39
39
|
import { contractCli, validateContractFile } from "./cli/contract.mjs";
|
|
40
40
|
import { specCli } from "./cli/spec.mjs";
|
|
41
41
|
import { planCli } from "./cli/plan.mjs";
|
|
42
|
-
import { METRICS_OPTIONS, renderCampaignMetrics } from "./campaign/metrics.mjs";
|
|
42
|
+
import { METRICS_OPTIONS, renderCampaignMetrics } from "./campaign/metrics-command.mjs";
|
|
43
43
|
import { runContract } from "./engine/scheduler.mjs";
|
|
44
44
|
import { resumeRun } from "./engine/resume.mjs";
|
|
45
45
|
import { cancelRun } from "./engine/cancel.mjs";
|
package/src/engine/bulk-read.mjs
CHANGED
|
@@ -259,13 +259,14 @@ function accountDelegation(runtimeId, runtime, envelope, priced, observation) {
|
|
|
259
259
|
nodeId,
|
|
260
260
|
role: "worker",
|
|
261
261
|
runtimeId,
|
|
262
|
+
harness: runtime.harness,
|
|
262
263
|
model: runtime.model,
|
|
263
264
|
usage: envelope.usage,
|
|
264
265
|
costUsd: priced.costUsd,
|
|
265
266
|
costProvenance: priced.costProvenance,
|
|
266
267
|
startedAt: observation.startedAt,
|
|
267
268
|
closedAt: new Date().toISOString(),
|
|
268
|
-
}));
|
|
269
|
+
}), { runtime });
|
|
269
270
|
}
|
|
270
271
|
|
|
271
272
|
/** @param {string} reason @returns {BulkReadResult} */
|
package/src/engine/dispatch.mjs
CHANGED
|
@@ -50,7 +50,7 @@ import { transition, writeNode } from "./state.mjs";
|
|
|
50
50
|
* `dispatched`: a judge is running (or failed to start and the node is already
|
|
51
51
|
* marked failed).
|
|
52
52
|
*
|
|
53
|
-
* @typedef {{kind: "rejected", verdict: JudgeVerdict}|{kind: "settle", gate: JudgeVerdict}|{kind: "dispatched"}} JudgeRound
|
|
53
|
+
* @typedef {{kind: "rejected", verdict: JudgeVerdict}|{kind: "settle", gate: JudgeVerdict|null}|{kind: "dispatched"}} JudgeRound
|
|
54
54
|
*/
|
|
55
55
|
/** @typedef {import("./prompts.mjs").JudgeVerdict} JudgeVerdict */
|
|
56
56
|
|
|
@@ -439,6 +439,28 @@ export function startResultMaterialization(contract, node, state, runDir, runnin
|
|
|
439
439
|
transition(runDir, state, "failed", { phase: "worker", error: { code: "result_materialization_failed", message: errorMessage(error) } }, lock);
|
|
440
440
|
}
|
|
441
441
|
}
|
|
442
|
+
/**
|
|
443
|
+
* The mechanical half of the gate, for a node whose gate is disabled: its
|
|
444
|
+
* Definition of Done command and path proofs still run, and no judge is
|
|
445
|
+
* asked. Measured 2026-09-23 on evidence-you-can-recompute: every node carried
|
|
446
|
+
* `gate: false`, and not one of its declared proofs ran.
|
|
447
|
+
*
|
|
448
|
+
* @param {ValidatedContract} contract
|
|
449
|
+
* @param {ValidatedNode} node
|
|
450
|
+
* @param {NodeSnapshot} state
|
|
451
|
+
* @returns {Promise<JudgeRound>}
|
|
452
|
+
*/
|
|
453
|
+
export async function startMechanicalGate(contract, node, state) {
|
|
454
|
+
const { verdict, results } = await deterministicGate(
|
|
455
|
+
node,
|
|
456
|
+
attemptWorkspace(state) ?? contract.cwd,
|
|
457
|
+
false,
|
|
458
|
+
gateProofTimeoutMs(node, contract),
|
|
459
|
+
/** @type {import("../contract/index.mjs").VerificationState|null} */ (state.verification),
|
|
460
|
+
);
|
|
461
|
+
if (verdict.verdict === "fail") return { kind: "rejected", verdict };
|
|
462
|
+
return { kind: "settle", gate: results.length ? verdict : null };
|
|
463
|
+
}
|
|
442
464
|
/**
|
|
443
465
|
* Gate a completed worker: mechanical proofs gate first, the judge arbitrates
|
|
444
466
|
* only judgment items and is skipped when the review mode is `none` or no
|
package/src/engine/lifecycle.mjs
CHANGED
|
@@ -52,7 +52,7 @@ import {
|
|
|
52
52
|
workerResultPath,
|
|
53
53
|
} from "./result-file.mjs";
|
|
54
54
|
import { canReuseResultEvidence, checkResultMaterializationScope, checkWorkerScope, recordScopeFinding, sourceWorkerRuntime } from "./scope.mjs";
|
|
55
|
-
import { startJudge, startResultMaterialization } from "./dispatch.mjs";
|
|
55
|
+
import { startJudge, startMechanicalGate, startResultMaterialization } from "./dispatch.mjs";
|
|
56
56
|
import { raiseNodeAttention, settleDone } from "./settle.mjs";
|
|
57
57
|
import { applyRejection, applyVerificationFailure } from "./settle.mjs";
|
|
58
58
|
import { emitNodeAdvisories } from "./notify-queue.mjs";
|
|
@@ -409,7 +409,7 @@ export async function finalizeClosedJobs(contract, runDir, states, closed, lock,
|
|
|
409
409
|
error: envelope.error ?? null,
|
|
410
410
|
nextState: operationNextState(state),
|
|
411
411
|
});
|
|
412
|
-
appendUsageRecord(runDir, state.invocations.find((invocation) => invocation.id === job.invocation.id));
|
|
412
|
+
appendUsageRecord(runDir, state.invocations.find((invocation) => invocation.id === job.invocation.id), { runtime: job.runtime });
|
|
413
413
|
state.usage = invocationUsage(state);
|
|
414
414
|
state.costUsd = invocationCost(state);
|
|
415
415
|
// Checked before any other branch can act on how this invocation closed --
|
|
@@ -587,11 +587,10 @@ export async function finalizeClosedJobs(contract, runDir, states, closed, lock,
|
|
|
587
587
|
continue;
|
|
588
588
|
}
|
|
589
589
|
if (job.scopeViolation) recordScopeFinding(runDir, state, lock);
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
else await settleDone(contract, job.node, state, runDir, lock, states, campaignPath, { phase: "complete", result: workerResult });
|
|
590
|
+
const round = job.node.gate.enabled
|
|
591
|
+
? await startJudge(contract, job.node, state, runDir, running, workerResult, lock, states, campaignPath)
|
|
592
|
+
: await startMechanicalGate(contract, job.node, state);
|
|
593
|
+
await applyJudgeRound(round, contract, job.node, state, runDir, running, lock, states, campaignPath, workerResult);
|
|
595
594
|
continue;
|
|
596
595
|
}
|
|
597
596
|
}
|
|
@@ -786,4 +785,3 @@ export async function applyInvalidWorkerResult(contract, node, state, runDir, ru
|
|
|
786
785
|
* else there.
|
|
787
786
|
*
|
|
788
787
|
* @param {string} runDir @param {string} nodeId @returns {string} */
|
|
789
|
-
|
package/src/engine/scheduler.mjs
CHANGED
|
@@ -539,6 +539,7 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
|
|
|
539
539
|
for (const job of jobs) envelopes.set(job.invocation.id, recordInvocationUsage(job, { accumulate: false }));
|
|
540
540
|
for (const job of jobs) {
|
|
541
541
|
const invocation = job.state.invocations?.find((item) => item.id === job.invocation.id) ?? job.invocation;
|
|
542
|
+
appendUsageRecord(runDir, invocation, { runtime: job.runtime, unknownReason: "invocation-killed" });
|
|
542
543
|
const scopeOk = job.phase !== "worker" || checkWorkerScope(contract, runDir, job, lock);
|
|
543
544
|
settleInvocation(runDir, invocation, {
|
|
544
545
|
status: scopeOk ? "canceled" : "failed",
|
|
@@ -569,7 +570,7 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
|
|
|
569
570
|
job.state.usage = invocationUsage(job.state);
|
|
570
571
|
job.state.costUsd = invocationCost(job.state);
|
|
571
572
|
const invocation = job.state.invocations?.find((item) => item.id === job.invocation.id) ?? job.invocation;
|
|
572
|
-
appendUsageRecord(runDir, invocation);
|
|
573
|
+
appendUsageRecord(runDir, invocation, { runtime: job.runtime, unknownReason: "invocation-killed" });
|
|
573
574
|
if (job.phase === "worker" && !checkWorkerScope(contract, runDir, job, lock)) {
|
|
574
575
|
settleInvocation(runDir, invocation, {
|
|
575
576
|
status: "failed",
|
package/src/engine/settle.mjs
CHANGED
|
@@ -46,7 +46,7 @@ import { verifyCandidateWorkspace } from "./verify.mjs";
|
|
|
46
46
|
* (`gate.maxRevisions`, default 1) whether or not the gate reviews: a red
|
|
47
47
|
* deterministic verification earns the same fresh attempt with the failure in
|
|
48
48
|
* front of the worker that a judge rejection does. Measured 2026-09-20 in the
|
|
49
|
-
* orchestration-arms
|
|
49
|
+
* `docs/campaigns/orchestration-arms/`: with the budget behind `gate.enabled`, a node
|
|
50
50
|
* under `gate: false` died on one timing test that flaked under load, and its
|
|
51
51
|
* dependant with it, while the judged twin of the same node got its retry.
|
|
52
52
|
* @param {ValidatedContract} contract @param {ValidatedNode} node @param {NodeSnapshot} state @param {string} runDir @param {Map<string, Job>|null} running @param {LockHandle} lock @param {Map<string, NodeSnapshot>} states @param {string} campaignPath @param {JudgeVerdict} verdict @param {{code: string, label: string, phase?: "worker"|"judge", message?: string, forceFresh?: boolean}} options */
|
package/src/plan/proof-run.mjs
CHANGED
|
@@ -65,18 +65,60 @@ export const MEASURE_SIDE_EFFECT_ENV_KEYS = [
|
|
|
65
65
|
* @returns {{output: string, exitCode: number|null, truncated: boolean}}
|
|
66
66
|
*/
|
|
67
67
|
export function runShellCapture(cwd, command, probes = {}) {
|
|
68
|
+
const { output, exitCode, truncated } = runShellFull(cwd, command, probes);
|
|
69
|
+
const bytes = Buffer.from(output, "utf8");
|
|
70
|
+
return {
|
|
71
|
+
output: truncated ? bytes.subarray(0, MEASURE_OUTPUT_CAP_BYTES).toString("utf8") : output,
|
|
72
|
+
exitCode,
|
|
73
|
+
truncated,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* The shell run behind `runShellCapture`, returning the whole output: a
|
|
79
|
+
* pinned-test check has to see every test the runner reported, and the
|
|
80
|
+
* matching line can sit far past the 4 KiB a measurement keeps.
|
|
81
|
+
*
|
|
82
|
+
* @param {string} cwd
|
|
83
|
+
* @param {string} command
|
|
84
|
+
* @param {MeasureProbes} [probes]
|
|
85
|
+
* @returns {{output: string, exitCode: number|null, truncated: boolean}}
|
|
86
|
+
*/
|
|
87
|
+
function runShellFull(cwd, command, probes = {}) {
|
|
68
88
|
const run = probes.run ?? spawnSync;
|
|
69
89
|
const env = { ...process.env };
|
|
70
90
|
for (const key of MEASURE_SIDE_EFFECT_ENV_KEYS) delete env[key];
|
|
91
|
+
// A nested `node --test` that inherits NODE_TEST_CONTEXT stays a runner
|
|
92
|
+
// child and prints no result at all (measured 2026-09-21, AGENTS.md), so a
|
|
93
|
+
// proof run from inside a test runner would have nothing to show.
|
|
94
|
+
delete env.NODE_TEST_CONTEXT;
|
|
71
95
|
const result = run(command, { shell: true, cwd, timeout: MEASURE_TIMEOUT_MS, encoding: "utf8", env });
|
|
72
96
|
const combined = `${result.stdout ?? ""}${result.stderr ?? ""}`;
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
97
|
+
return { output: combined, exitCode: result.status, truncated: Buffer.byteLength(combined, "utf8") > MEASURE_OUTPUT_CAP_BYTES };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** `--test-name-pattern` in a shell command, quoted or bare, `=` or space. */
|
|
101
|
+
const NAME_PATTERN_FLAG = /--test-name-pattern(?:=|\s+)(?:"([^"]*)"|'([^']*)'|(\S+))/u;
|
|
102
|
+
/** A passing test as the spec reporter (`✔ name (1ms)`) or TAP (`ok 1 - name`) prints it. */
|
|
103
|
+
const PASSING_TEST_LINE = /^\s*(?:\u2714\s+|ok \d+ - )(.+?)(?:\s+\(\d[\d.]*m?s\))?\s*$/gmu;
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Whether a `node --test --test-name-pattern=<p>` proof ran a test named by
|
|
107
|
+
* its pattern. Node exits 0 when the pattern matches nothing, printing only
|
|
108
|
+
* the file, so the exit code alone cannot tell a proof from its absence.
|
|
109
|
+
* Returns null when the command carries no pattern.
|
|
110
|
+
*
|
|
111
|
+
* @param {string} command
|
|
112
|
+
* @param {string} output
|
|
113
|
+
* @returns {boolean|null}
|
|
114
|
+
*/
|
|
115
|
+
function patternMatchedATest(command, output) {
|
|
116
|
+
if (!/\bnode\b[^|;&]*--test\b/u.test(command)) return null;
|
|
117
|
+
const flag = NAME_PATTERN_FLAG.exec(command);
|
|
118
|
+
if (!flag) return null;
|
|
119
|
+
const pattern = new RegExp(flag[1] ?? flag[2] ?? flag[3], "u");
|
|
120
|
+
const plain = output.replace(/\u001b\[[0-9;]*m/gu, "");
|
|
121
|
+
return [...plain.matchAll(PASSING_TEST_LINE)].some((line) => pattern.test(line[1]) && !/\.m?[jt]s$/u.test(line[1].trim()));
|
|
80
122
|
}
|
|
81
123
|
|
|
82
124
|
/**
|
|
@@ -104,11 +146,12 @@ export function proveRequirements(cwd, requirements, probes = {}) {
|
|
|
104
146
|
const proof = requirement.proof;
|
|
105
147
|
if (!proof?.ref) continue;
|
|
106
148
|
if (proof.kind === "command") {
|
|
107
|
-
const { output, exitCode } =
|
|
108
|
-
const
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
149
|
+
const { output, exitCode } = runShellFull(cwd, proof.ref, probes);
|
|
150
|
+
const matched = exitCode === 0 ? patternMatchedATest(proof.ref, output) : null;
|
|
151
|
+
const detail = exitCode !== 0
|
|
152
|
+
? `exit ${exitCode ?? "no exit code (killed or never started)"}${output.trim() ? `: ${output.trim().slice(0, PROOF_DETAIL_CAP)}` : ""}`
|
|
153
|
+
: matched === false ? "exit 0, but its --test-name-pattern matched no test: the proof ran nothing" : "exit 0";
|
|
154
|
+
results.push({ requirementId: requirement.id, kind: "command", ref: proof.ref, pass: exitCode === 0 && matched !== false, detail });
|
|
112
155
|
continue;
|
|
113
156
|
}
|
|
114
157
|
if (proof.kind === "path") {
|
package/src/repo/worktree.mjs
CHANGED
|
@@ -384,7 +384,10 @@ export function sealAttempt({ repo, path, baseSha, runId, nodeId, attempt, exclu
|
|
|
384
384
|
// schedule it chose. The identity and signing overrides above are the
|
|
385
385
|
// same argument: this commit answers to the factory, not to the repo's
|
|
386
386
|
// conventions for human commits.
|
|
387
|
-
|
|
387
|
+
// Conventional, so a campaign branch passes a PR's commitlint check
|
|
388
|
+
// (measured 2026-09-23: every `faberun <run> <node> attempt N` seal was
|
|
389
|
+
// refused); the run id moves to the body to keep the header short.
|
|
390
|
+
"commit", "--no-verify", "-q", "-m", `chore(faberun): seal ${nodeId} attempt ${attempt}`, "-m", `run ${runId}`,
|
|
388
391
|
]);
|
|
389
392
|
}
|
|
390
393
|
const sha = gitHead(path);
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
15
|
/** @typedef {import("../campaign/metrics.mjs").CampaignMetrics} CampaignMetrics */
|
|
16
|
-
/** @typedef {import("../campaign/metrics.mjs").MetricsSources} MetricsSources */
|
|
16
|
+
/** @typedef {import("../campaign/metrics-command.mjs").MetricsSources} MetricsSources */
|
|
17
17
|
|
|
18
18
|
/** Schema of the `--json` form; bumped when a consumer would have to change. */
|
|
19
19
|
const METRICS_SCHEMA_VERSION = 2;
|
|
@@ -39,19 +39,30 @@ const HOURS_PRECISION = 10_000;
|
|
|
39
39
|
* @returns {string}
|
|
40
40
|
*/
|
|
41
41
|
export function renderMetricsReport(sources, metrics) {
|
|
42
|
+
const missingSources = sources.missingSources ?? [];
|
|
42
43
|
const lines = [
|
|
43
44
|
`[metrics] ${sources.campaignId} · ${sources.runIds.length} runs · ${sources.events.length} events · ${Object.keys(metrics).length} indicators`,
|
|
44
45
|
];
|
|
46
|
+
if (missingSources.length > 0) lines.push(`missing sources: ${missingSources.join(", ")}`);
|
|
45
47
|
for (const [name, indicator] of Object.entries(metrics)) {
|
|
46
48
|
const records = indicator.count === 1 ? "1 record" : `${indicator.count} records`;
|
|
47
49
|
const unknown = typeof (/** @type {{unknownCount?: number}} */ (indicator).unknownCount) === "number"
|
|
48
|
-
?
|
|
50
|
+
? `${unknownCostSuffix(/** @type {{unknownCount: number, unknownFractionByReason?: Record<string, number>}} */ (indicator))}`
|
|
49
51
|
: "";
|
|
50
|
-
|
|
52
|
+
const missing = Array.isArray(indicator.missingSources) ? ` · missing ${indicator.missingSources.join(", ")}` : "";
|
|
53
|
+
lines.push(`${name.padEnd(NAME_WIDTH)} ${indicator.direction.padEnd(11)} ${formatValue(name, indicator.value).padEnd(24)} · ${records}${unknown}${missing}`);
|
|
51
54
|
}
|
|
52
55
|
return `${lines.join("\n")}\n`;
|
|
53
56
|
}
|
|
54
57
|
|
|
58
|
+
/** @param {{unknownCount: number, unknownFractionByReason?: Record<string, number>}} indicator @returns {string} */
|
|
59
|
+
function unknownCostSuffix(indicator) {
|
|
60
|
+
const reasons = Object.entries(indicator.unknownFractionByReason ?? {})
|
|
61
|
+
.map(([reason, fraction]) => `${reason}=${fraction}`)
|
|
62
|
+
.join(", ");
|
|
63
|
+
return ` · ${indicator.unknownCount} unknown${reasons ? ` (${reasons})` : ""}`;
|
|
64
|
+
}
|
|
65
|
+
|
|
55
66
|
/**
|
|
56
67
|
* The machine-readable form, one JSON line like `preflight --json`.
|
|
57
68
|
*
|
|
@@ -60,11 +71,13 @@ export function renderMetricsReport(sources, metrics) {
|
|
|
60
71
|
* @returns {string}
|
|
61
72
|
*/
|
|
62
73
|
export function renderMetricsJson(sources, metrics) {
|
|
74
|
+
const missingSources = sources.missingSources ?? [];
|
|
63
75
|
return `${JSON.stringify({
|
|
64
76
|
schemaVersion: METRICS_SCHEMA_VERSION,
|
|
65
77
|
campaignId: sources.campaignId,
|
|
66
78
|
runs: sources.runIds.length,
|
|
67
79
|
events: sources.events.length,
|
|
80
|
+
missingSources,
|
|
68
81
|
indicators: metrics,
|
|
69
82
|
})}\n`;
|
|
70
83
|
}
|
package/src/run/lock.mjs
CHANGED
|
@@ -17,7 +17,7 @@ import { closeSync, fsyncSync, linkSync, mkdirSync, openSync, readFileSync, rena
|
|
|
17
17
|
import { hostname } from "node:os";
|
|
18
18
|
import { join } from "node:path";
|
|
19
19
|
import { fsyncDirectory } from "./store.mjs";
|
|
20
|
-
import { errorCode } from "../util.mjs";
|
|
20
|
+
import { errorCode, sleepSync } from "../util.mjs";
|
|
21
21
|
|
|
22
22
|
const LOCK_FILE = "controller.lock";
|
|
23
23
|
const TAKEOVER_ATTEMPTS = 20;
|
|
@@ -84,7 +84,7 @@ export function processStartToken(pid) {
|
|
|
84
84
|
// and report nothing for a pid that is already alive. One short retry
|
|
85
85
|
// closes that window; a still-empty answer stays null.
|
|
86
86
|
if (!pidAlive(pid)) return null;
|
|
87
|
-
|
|
87
|
+
sleepSync(20);
|
|
88
88
|
const retried = psStartTime(pid);
|
|
89
89
|
return retried.length > 0 ? retried : null;
|
|
90
90
|
}
|
|
@@ -106,16 +106,6 @@ function psStartTime(pid) {
|
|
|
106
106
|
}
|
|
107
107
|
}
|
|
108
108
|
|
|
109
|
-
/**
|
|
110
|
-
* Block without a timer so the synchronous darwin retry above can wait out the
|
|
111
|
-
* spawn race. The duration is tiny and bounded, so blocking the loop is safe.
|
|
112
|
-
*
|
|
113
|
-
* @param {number} milliseconds
|
|
114
|
-
*/
|
|
115
|
-
function sleepMs(milliseconds) {
|
|
116
|
-
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);
|
|
117
|
-
}
|
|
118
|
-
|
|
119
109
|
/** @param {number|null|undefined} pid @returns {boolean} */
|
|
120
110
|
export function pidAlive(pid) {
|
|
121
111
|
if (!Number.isInteger(pid) || /** @type {number} */ (pid) <= 0) return false;
|
package/src/run/store.mjs
CHANGED
|
@@ -14,9 +14,19 @@ import {
|
|
|
14
14
|
} from "node:fs";
|
|
15
15
|
import { randomUUID } from "node:crypto";
|
|
16
16
|
import { dirname, join } from "node:path";
|
|
17
|
-
import { errorCode } from "../util.mjs";
|
|
17
|
+
import { errorCode, sleepSync } from "../util.mjs";
|
|
18
18
|
|
|
19
19
|
const BOOTSTRAP_FILE = "bootstrap.json";
|
|
20
|
+
// Windows refuses to replace a file another process holds open: MoveFileEx
|
|
21
|
+
// answers EPERM (or EACCES, EBUSY) while a reader, an indexer or a virus
|
|
22
|
+
// scanner has the destination, where a POSIX rename replaces it regardless.
|
|
23
|
+
// Measured 2026-09-23 on Windows 11 with node 24: one process polling a file
|
|
24
|
+
// with readFileSync failed 882 of 2190 atomic writes onto it, and on CI that
|
|
25
|
+
// killed controllers mid-run (a heartbeat write threw out of driveRun). The
|
|
26
|
+
// retry and its bounds are graceful-fs's: back off 10 ms more per attempt up to
|
|
27
|
+
// 100 ms, and give up after 60 s.
|
|
28
|
+
const WINDOWS_RENAME_BUSY = new Set(["EPERM", "EACCES", "EBUSY"]);
|
|
29
|
+
const RENAME_RETRY_BUDGET_MS = 60_000;
|
|
20
30
|
const JSONL_RECOVERY_TAIL_BYTES = 64 * 1024;
|
|
21
31
|
|
|
22
32
|
/**
|
|
@@ -51,7 +61,7 @@ export function writeTextAtomic(path, text) {
|
|
|
51
61
|
} finally {
|
|
52
62
|
closeSync(fd);
|
|
53
63
|
}
|
|
54
|
-
|
|
64
|
+
renameReplacing(temporary, path);
|
|
55
65
|
fsyncDirectory(dirname(path));
|
|
56
66
|
committed = true;
|
|
57
67
|
} catch (error) {
|
|
@@ -68,6 +78,24 @@ export function writeTextAtomic(path, text) {
|
|
|
68
78
|
}
|
|
69
79
|
}
|
|
70
80
|
|
|
81
|
+
/**
|
|
82
|
+
* @param {string} from
|
|
83
|
+
* @param {string} to
|
|
84
|
+
*/
|
|
85
|
+
function renameReplacing(from, to) {
|
|
86
|
+
const deadline = Date.now() + RENAME_RETRY_BUDGET_MS;
|
|
87
|
+
for (let backoff = 0; ; backoff = Math.min(backoff + 10, 100)) {
|
|
88
|
+
try {
|
|
89
|
+
renameSync(from, to);
|
|
90
|
+
return;
|
|
91
|
+
} catch (error) {
|
|
92
|
+
const busy = process.platform === "win32" && WINDOWS_RENAME_BUSY.has(errorCode(error) ?? "");
|
|
93
|
+
if (!busy || Date.now() >= deadline) throw error;
|
|
94
|
+
sleepSync(backoff);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
71
99
|
/**
|
|
72
100
|
* @param {string} path
|
|
73
101
|
* @param {unknown} value
|
package/src/run/usage.mjs
CHANGED
|
@@ -15,7 +15,7 @@ import { liveUsage } from "../harnesses/session-metrics.mjs";
|
|
|
15
15
|
import { priceUsage } from "../engine/process.mjs";
|
|
16
16
|
import { readBoundedTail, sessionLedger } from "../engine/transcript.mjs";
|
|
17
17
|
import { writeNode } from "../engine/state.mjs";
|
|
18
|
-
import { normalizeProviderResult } from "../harnesses/index.mjs";
|
|
18
|
+
import { harnessCapabilities, normalizeProviderResult } from "../harnesses/index.mjs";
|
|
19
19
|
|
|
20
20
|
// `priceUsage` is defined beside `invocationResult`, the second source point,
|
|
21
21
|
// and re-exported here so the ledger's public surface is unchanged. This module
|
|
@@ -31,6 +31,15 @@ export { priceUsage };
|
|
|
31
31
|
/** @typedef {ProviderEnvelope & {costProvenance?: "priced"}} PricedEnvelope */
|
|
32
32
|
/** @typedef {{kind: "adopted"|"rejudge"|"restart"|"reconciled"|"exhausted"|"stalled", phase?: "worker"|"judge", result?: unknown, usage?: Usage, costUsd?: number|null, costProvenance?: "priced", exhaustedUntil?: string|null, error?: {code: string, message: string}|null, invocationId?: string, reason?: string}} RecoveryOutcome */
|
|
33
33
|
/** @typedef {import("../contract/index.mjs").Usage} Usage */
|
|
34
|
+
/** @typedef {"no-usage-stream"|"model-not-priced"|"provider-reported-nothing"|"invocation-killed"} UnknownCostReason */
|
|
35
|
+
|
|
36
|
+
/** The only reasons a newly-written unknown cost may carry. `legacy` is read-only compatibility for older ledgers. */
|
|
37
|
+
export const UNKNOWN_COST_REASONS = Object.freeze([
|
|
38
|
+
"no-usage-stream",
|
|
39
|
+
"model-not-priced",
|
|
40
|
+
"provider-reported-nothing",
|
|
41
|
+
"invocation-killed",
|
|
42
|
+
]);
|
|
34
43
|
|
|
35
44
|
/**
|
|
36
45
|
* Extract the invocation's provider envelope from the bounded transcript tail
|
|
@@ -144,10 +153,15 @@ export function usageRecordIds(runDir) {
|
|
|
144
153
|
*
|
|
145
154
|
* @param {string} runDir
|
|
146
155
|
* @param {Invocation|undefined|null} invocation
|
|
156
|
+
* @param {{runtime?: {harness?: string, capabilities?: {usage?: boolean}}, unknownReason?: UnknownCostReason}} [options]
|
|
147
157
|
*/
|
|
148
|
-
export function appendUsageRecord(runDir, invocation) {
|
|
158
|
+
export function appendUsageRecord(runDir, invocation, options = {}) {
|
|
149
159
|
if (!invocation?.id || usageRecordIds(runDir).has(invocation.id)) return;
|
|
150
160
|
const usage = /** @type {Usage} */ (invocation.usage ?? { inputTokens: null, outputTokens: null, cacheReadInputTokens: null });
|
|
161
|
+
const costProvenance = invocation.costProvenance ?? (typeof invocation.costUsd === "number" ? "provider" : "unknown");
|
|
162
|
+
const unknownReason = costProvenance === "unknown"
|
|
163
|
+
? unknownCostReason(invocation, usage, options)
|
|
164
|
+
: undefined;
|
|
151
165
|
appendJsonl(join(runDir, USAGE_LOG_NAME), {
|
|
152
166
|
invocationId: invocation.id,
|
|
153
167
|
runId: invocation.runId ?? basename(runDir),
|
|
@@ -162,12 +176,51 @@ export function appendUsageRecord(runDir, invocation) {
|
|
|
162
176
|
costUsd: typeof invocation.costUsd === "number" ? invocation.costUsd : null,
|
|
163
177
|
// A persisted `priced` marker wins; otherwise the pre-Phase-4 rule applies
|
|
164
178
|
// unchanged: a reported number is `provider`, absence is `unknown`.
|
|
165
|
-
costProvenance
|
|
179
|
+
costProvenance,
|
|
180
|
+
...(unknownReason === undefined ? {} : { unknownReason }),
|
|
166
181
|
session: invocation.session ?? null,
|
|
167
182
|
startedAt: invocation.startedAt ?? null,
|
|
168
183
|
finishedAt: invocation.closedAt ?? null,
|
|
169
184
|
});
|
|
170
185
|
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Explain an unknown cost at the point where the invocation's counters and
|
|
189
|
+
* pricing have been resolved. A runtime can be supplied by the live engine;
|
|
190
|
+
* older or recovery callers fall back to the invocation's harness identity.
|
|
191
|
+
*
|
|
192
|
+
* @param {Invocation} invocation
|
|
193
|
+
* @param {Usage} usage
|
|
194
|
+
* @param {{runtime?: {harness?: string, capabilities?: {usage?: boolean}}, unknownReason?: UnknownCostReason}} [options]
|
|
195
|
+
* @returns {UnknownCostReason}
|
|
196
|
+
*/
|
|
197
|
+
function unknownCostReason(invocation, usage, options = {}) {
|
|
198
|
+
if (options.unknownReason !== undefined) return options.unknownReason;
|
|
199
|
+
if (invocation.signal !== null && invocation.signal !== undefined || invocation.status === "terminated") {
|
|
200
|
+
return "invocation-killed";
|
|
201
|
+
}
|
|
202
|
+
const runtimeUsage = options.runtime?.capabilities?.usage;
|
|
203
|
+
if (runtimeUsage === false || (runtimeUsage === undefined && invocation.harness !== undefined && !harnessHasUsage(invocation.harness))) {
|
|
204
|
+
return "no-usage-stream";
|
|
205
|
+
}
|
|
206
|
+
if (!hasMeasuredUsage(usage)) return "provider-reported-nothing";
|
|
207
|
+
// A partial token stream cannot be called "nothing". It also cannot be
|
|
208
|
+
// priced because one or more counters are absent, so the model-not-priced
|
|
209
|
+
// bucket is the only remaining vocabulary that preserves the unknown cost.
|
|
210
|
+
return "model-not-priced";
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** @param {string} harness @returns {boolean} */
|
|
214
|
+
function harnessHasUsage(harness) {
|
|
215
|
+
try {
|
|
216
|
+
return harnessCapabilities({ harness }).usage;
|
|
217
|
+
} catch (error) {
|
|
218
|
+
// An unknown harness cannot prove that it emits usage; callers still get
|
|
219
|
+
// the provider-reported-nothing fallback rather than a guessed adapter.
|
|
220
|
+
void error;
|
|
221
|
+
return true;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
171
224
|
/**
|
|
172
225
|
* Recovery can discover usage after the run synchronized its records. Attach
|
|
173
226
|
* it to the authoritative invocation first, then write the updated usage
|
|
@@ -201,7 +254,11 @@ export async function persistRecoveryUsage(runDir, state, recovery, lock) {
|
|
|
201
254
|
writeNode(runDir, state, lock);
|
|
202
255
|
}
|
|
203
256
|
const updated = state.invocations?.find((invocation) => invocation.id === current.id);
|
|
204
|
-
if (updated)
|
|
257
|
+
if (updated) {
|
|
258
|
+
const killed = recovery.kind === "stalled"
|
|
259
|
+
|| (recovery.kind === "restart" && /wall-clock budget|died without a completed stream/iu.test(recovery.reason ?? ""));
|
|
260
|
+
appendUsageRecord(runDir, updated, killed ? { unknownReason: "invocation-killed" } : {});
|
|
261
|
+
}
|
|
205
262
|
}
|
|
206
263
|
/**
|
|
207
264
|
* @param {Usage|undefined} left
|
package/src/util.mjs
CHANGED
|
@@ -275,3 +275,13 @@ export function shellWords(text) {
|
|
|
275
275
|
if (current !== null) words.push(current);
|
|
276
276
|
return words;
|
|
277
277
|
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Block the thread without a timer, for a synchronous path that has to wait
|
|
281
|
+
* out a short race. Callers keep the duration tiny and bounded.
|
|
282
|
+
*
|
|
283
|
+
* @param {number} milliseconds
|
|
284
|
+
*/
|
|
285
|
+
export function sleepSync(milliseconds) {
|
|
286
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);
|
|
287
|
+
}
|