faberun 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +131 -0
- package/bin/faberun.mjs +25 -0
- package/integrations/claude-code/statusline-bench.sh +42 -0
- package/integrations/claude-code/statusline.sh +80 -0
- package/package.json +33 -0
- package/skills/faberun/SKILL.md +24 -0
- package/skills/faberun/references/contract.md +380 -0
- package/skills/faberun/references/engineering.md +29 -0
- package/skills/faberun/references/handoffs.md +26 -0
- package/skills/faberun/references/operations.md +184 -0
- package/skills/faberun/references/rules.md +35 -0
- package/skills/faberun/references/workflow.md +23 -0
- package/skills/init-agentkit/SKILL.md +108 -0
- package/skills/init-agentkit/scripts/install-agentkit.sh +127 -0
- package/skills/init-agentkit/templates/.claude/commands/create-adr.md +44 -0
- package/skills/init-agentkit/templates/.github/workflows/quality.yml +43 -0
- package/skills/init-agentkit/templates/.sentrux/baseline.json +9 -0
- package/skills/init-agentkit/templates/.sentrux/rules.toml +21 -0
- package/skills/init-agentkit/templates/AGENTS.md +110 -0
- package/skills/init-agentkit/templates/docs/ABSTRACTIONS.md +30 -0
- package/skills/init-agentkit/templates/docs/ARCHITECTURE.md +31 -0
- package/skills/init-agentkit/templates/docs/GETTING-STARTED.md +44 -0
- package/skills/init-agentkit/templates/docs/VISION.md +33 -0
- package/skills/init-agentkit/templates/docs/adr/0001-record-architecture-decisions.md +36 -0
- package/skills/init-agentkit/templates/docs/adr/0002-root-managed-ai-guidance.md +37 -0
- package/skills/init-agentkit/templates/docs/adr/0003-sentrux-structural-quality-gates.md +49 -0
- package/skills/init-agentkit/templates/docs/adr/README.md +52 -0
- package/skills/init-agentkit/templates/docs/sentrux.md +66 -0
- package/skills/init-agentkit/templates/githooks/commit-msg +22 -0
- package/skills/init-agentkit/templates/githooks/pre-commit +32 -0
- package/src/campaign/brief.mjs +394 -0
- package/src/campaign/chain.mjs +555 -0
- package/src/campaign/handoff.mjs +516 -0
- package/src/campaign/index.mjs +300 -0
- package/src/campaign/journal.mjs +347 -0
- package/src/campaign/layout.mjs +51 -0
- package/src/campaign/metrics-evals.mjs +25 -0
- package/src/campaign/metrics.mjs +517 -0
- package/src/campaign/projection.mjs +250 -0
- package/src/campaign/record.mjs +102 -0
- package/src/campaign/unpark.mjs +56 -0
- package/src/cli/brand.mjs +205 -0
- package/src/cli/campaign.mjs +730 -0
- package/src/cli/contract.mjs +67 -0
- package/src/cli/init.mjs +170 -0
- package/src/cli/launch.mjs +239 -0
- package/src/cli/seat.mjs +139 -0
- package/src/cli/setup.mjs +294 -0
- package/src/cli/skills.mjs +105 -0
- package/src/cli/update.mjs +216 -0
- package/src/cli.mjs +525 -0
- package/src/contract/articles.mjs +12 -0
- package/src/contract/assert.mjs +162 -0
- package/src/contract/definition-of-done.mjs +97 -0
- package/src/contract/final-verification.mjs +96 -0
- package/src/contract/index.mjs +641 -0
- package/src/contract/judge-envelope.mjs +25 -0
- package/src/contract/review-modes.mjs +151 -0
- package/src/contract/runtime.mjs +204 -0
- package/src/contract/schema-version.mjs +25 -0
- package/src/contract/scope-findings.mjs +77 -0
- package/src/contract/snapshot.mjs +639 -0
- package/src/contract/task-packet.mjs +495 -0
- package/src/contract/untrusted.mjs +75 -0
- package/src/contract/verification.mjs +185 -0
- package/src/contract/worker-result.mjs +138 -0
- package/src/engine/assignment.mjs +63 -0
- package/src/engine/backoff.mjs +492 -0
- package/src/engine/bulk-read.mjs +361 -0
- package/src/engine/cancel.mjs +177 -0
- package/src/engine/detach.mjs +101 -0
- package/src/engine/dispatch.mjs +752 -0
- package/src/engine/failover.mjs +192 -0
- package/src/engine/gate.mjs +183 -0
- package/src/engine/judge-gate.mjs +517 -0
- package/src/engine/lifecycle.mjs +772 -0
- package/src/engine/live-preflight.mjs +299 -0
- package/src/engine/mutation.mjs +146 -0
- package/src/engine/notify-queue.mjs +327 -0
- package/src/engine/process-identity.mjs +72 -0
- package/src/engine/process.mjs +774 -0
- package/src/engine/prompts.mjs +289 -0
- package/src/engine/recover.mjs +300 -0
- package/src/engine/result-file.mjs +222 -0
- package/src/engine/resume.mjs +635 -0
- package/src/engine/retry.mjs +334 -0
- package/src/engine/review.mjs +228 -0
- package/src/engine/run-command.mjs +287 -0
- package/src/engine/run-identity.mjs +411 -0
- package/src/engine/runtime-discovery.mjs +235 -0
- package/src/engine/scheduler.mjs +526 -0
- package/src/engine/scope.mjs +378 -0
- package/src/engine/settle.mjs +207 -0
- package/src/engine/state.mjs +148 -0
- package/src/engine/supervise.mjs +713 -0
- package/src/engine/verify.mjs +167 -0
- package/src/harnesses/agy/index.mjs +62 -0
- package/src/harnesses/catalogue.mjs +509 -0
- package/src/harnesses/claude/index.mjs +90 -0
- package/src/harnesses/codex/index.mjs +87 -0
- package/src/harnesses/dsh/closed-packet.patch.yml +42 -0
- package/src/harnesses/dsh/index.mjs +210 -0
- package/src/harnesses/dsh/runner.mjs +259 -0
- package/src/harnesses/exec-jsonl/index.mjs +788 -0
- package/src/harnesses/index.mjs +508 -0
- package/src/harnesses/protocol.mjs +531 -0
- package/src/harnesses/replay/bin.mjs +386 -0
- package/src/harnesses/replay/index.mjs +238 -0
- package/src/harnesses/zcode/index.mjs +276 -0
- package/src/host/config.mjs +87 -0
- package/src/host/home.mjs +149 -0
- package/src/host/package.mjs +23 -0
- package/src/host/preflight.mjs +520 -0
- package/src/host/tool-policy-decisions.mjs +341 -0
- package/src/host/tool-policy-hook.mjs +270 -0
- package/src/notify/index.mjs +359 -0
- package/src/notify/os-macos.mjs +81 -0
- package/src/repo/declared-paths.mjs +220 -0
- package/src/repo/integrate.mjs +546 -0
- package/src/repo/scope-closure.mjs +665 -0
- package/src/repo/signal-block.mjs +16 -0
- package/src/repo/signal.mjs +222 -0
- package/src/repo/source-identity.mjs +295 -0
- package/src/repo/workspace.mjs +557 -0
- package/src/repo/worktree.mjs +352 -0
- package/src/report/final.mjs +200 -0
- package/src/report/metrics-report.mjs +99 -0
- package/src/report/next.mjs +383 -0
- package/src/report/render.mjs +716 -0
- package/src/run/disk-gc.mjs +251 -0
- package/src/run/lock.mjs +329 -0
- package/src/run/node-store.mjs +62 -0
- package/src/run/operations.mjs +286 -0
- package/src/run/store.mjs +187 -0
- package/src/run/usage.mjs +337 -0
- package/src/seat/harnesses.mjs +83 -0
- package/src/seat/index.mjs +239 -0
- package/src/seat/tmux.mjs +208 -0
- package/src/util.mjs +0 -0
- package/src/web/api.mjs +371 -0
- package/src/web/boundary.mjs +88 -0
- package/src/web/index.html +299 -0
- package/src/web/server.mjs +552 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where a campaign's files live and how big each one may be.
|
|
3
|
+
*
|
|
4
|
+
* Every byte ceiling in the campaign layer is here rather than beside the code
|
|
5
|
+
* that enforces it, because the ceilings interact: the handoff has a total
|
|
6
|
+
* budget, the journal a per-entry cap, the projection a list cap, and a change
|
|
7
|
+
* to one is a decision about the others. They were scattered across one
|
|
8
|
+
* 1,418-line file, which is how a "small" limit bump becomes a surprise.
|
|
9
|
+
*/
|
|
10
|
+
import { join } from "node:path";
|
|
11
|
+
import { requireId } from "../contract/assert.mjs";
|
|
12
|
+
|
|
13
|
+
export const CAMPAIGN_DIR_NAME = "campaigns";
|
|
14
|
+
export const CAMPAIGN_FILE = "campaign.json";
|
|
15
|
+
export const JOURNAL_FILE = "journal.jsonl";
|
|
16
|
+
export const HANDOFF_FILE = "HANDOFF.md";
|
|
17
|
+
export const PROJECTION_FILE = "projection.json";
|
|
18
|
+
export const JOURNAL_WATCH_CURSOR_DIR = "watch-cursors";
|
|
19
|
+
export const JOURNAL_WATCH_CURSOR_SCHEMA_VERSION = 1;
|
|
20
|
+
export const HANDOFF_LIMIT = 20;
|
|
21
|
+
export const HANDOFF_BYTES = 16 * 1024;
|
|
22
|
+
export const BRIEF_FILE = "operator-brief.md";
|
|
23
|
+
export const BRIEF_BYTES = 4 * 1024;
|
|
24
|
+
export const JOURNAL_TEXT_BYTES = 2 * 1024;
|
|
25
|
+
export const GOAL_TEXT_BYTES = 4 * 1024;
|
|
26
|
+
export const RENDER_NOTE_BYTES = 300;
|
|
27
|
+
export const PROJECTION_LIST_CAP = 60;
|
|
28
|
+
export const PROJECTION_ACTIVE_CAP = 100;
|
|
29
|
+
export const JOURNAL_TAIL_BYTES = 4096;
|
|
30
|
+
export const CRITICAL_FLOOR_BYTES = 512;
|
|
31
|
+
export const ID_CAP_FLOOR = 48;
|
|
32
|
+
/**
|
|
33
|
+
* @param {string} runsDir
|
|
34
|
+
* @returns {string}
|
|
35
|
+
*/
|
|
36
|
+
export function campaignsDir(runsDir) {
|
|
37
|
+
return join(runsDir, CAMPAIGN_DIR_NAME);
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* @param {string} runsDir
|
|
41
|
+
* @param {string} campaignId
|
|
42
|
+
* @returns {string}
|
|
43
|
+
*/
|
|
44
|
+
export function campaignDir(runsDir, campaignId) {
|
|
45
|
+
requireId(campaignId, "campaignId");
|
|
46
|
+
return join(campaignsDir(runsDir), campaignId);
|
|
47
|
+
}
|
|
48
|
+
/** @param {string} value @returns {string} */
|
|
49
|
+
export function basenameSafe(value) {
|
|
50
|
+
return value.split(/[\\/]+/u).filter(Boolean).at(-1) ?? "campaign";
|
|
51
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Small numeric and parsing helpers shared by `metrics.mjs`. Nothing here
|
|
3
|
+
* estimates or weights a token count: every value it touches is a number the
|
|
4
|
+
* platform already recorded.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/** @typedef {Record<string, unknown>} JsonObject */
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* @param {unknown} value
|
|
11
|
+
* @returns {JsonObject|null}
|
|
12
|
+
*/
|
|
13
|
+
export function jsonObjectOf(value) {
|
|
14
|
+
return value && typeof value === "object" && !Array.isArray(value) ? /** @type {JsonObject} */ (value) : null;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** @param {unknown} value @returns {number} */
|
|
18
|
+
export function timestampMs(value) {
|
|
19
|
+
return typeof value === "string" ? Date.parse(value) : Number.NaN;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** @param {number} value @returns {number} */
|
|
23
|
+
export function round4(value) {
|
|
24
|
+
return Math.round(value * 10_000) / 10_000;
|
|
25
|
+
}
|
|
@@ -0,0 +1,517 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Metrics projector (TECH-SPEC lean section 6). One pure function over a
|
|
3
|
+
* campaign's own recorded artefacts — every linked run's persisted node
|
|
4
|
+
* snapshots, `events.jsonl`, `usage.jsonl` and `notify.jsonl` — returning
|
|
5
|
+
* exactly the indicators section 6 measures at close. Nothing here estimates
|
|
6
|
+
* a token count or reads a heartbeat: every value comes from a record the
|
|
7
|
+
* platform already wrote for another reason (a node snapshot, a transition,
|
|
8
|
+
* a priced invocation, a delivery receipt).
|
|
9
|
+
*
|
|
10
|
+
* `projectMetrics` takes already-parsed records and never a filesystem path,
|
|
11
|
+
* which keeps every indicator testable without fixtures on disk. Each
|
|
12
|
+
* indicator carries its value, the direction that counts as better, and the
|
|
13
|
+
* number of records it was computed from. An indicator with no supporting
|
|
14
|
+
* record is `null`, never `0`: a missing measurement and a measured zero are
|
|
15
|
+
* different facts.
|
|
16
|
+
*
|
|
17
|
+
* A *logical node* is a contract node id within one run (TECH-SPEC section
|
|
18
|
+
* 6): the same id in two different runs is two logical nodes, because a
|
|
19
|
+
* fresh run re-authors the work rather than resuming it. A *checkpoint* is
|
|
20
|
+
* coarser — the node id alone, deduplicated across every linked run — and is
|
|
21
|
+
* what `linkedRunsPerClosedCheckpoint` divides the run count by: normally the
|
|
22
|
+
* same node closes in the one run that carries it, and the ratio drifts above
|
|
23
|
+
* 1 only when a whole run had to be re-authored after a failure that
|
|
24
|
+
* `resume` could not repair.
|
|
25
|
+
*
|
|
26
|
+
* The second half of this module is the `runner.mjs metrics` command: reading
|
|
27
|
+
* a campaign's linked runs and parsing the command's flags live here, while
|
|
28
|
+
* `metrics-report.mjs` decides how the projection is printed.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
32
|
+
import { join, resolve } from "node:path";
|
|
33
|
+
|
|
34
|
+
import { jsonObjectOf, round4, timestampMs } from "./metrics-evals.mjs";
|
|
35
|
+
import { MAX_ATTEMPTS as NOTIFY_MAX_ATTEMPTS } from "../notify/index.mjs";
|
|
36
|
+
import { renderMetricsJson, renderMetricsReport } from "../report/metrics-report.mjs";
|
|
37
|
+
import { campaignDir } from "./layout.mjs";
|
|
38
|
+
import { readCampaign } from "./record.mjs";
|
|
39
|
+
import { listNodeSnapshots, nodeSnapshotPath } from "../run/node-store.mjs";
|
|
40
|
+
|
|
41
|
+
/** Node statuses that are not terminal: everything else settles a logical node. */
|
|
42
|
+
const OPEN_STATUSES = new Set(["pending", "running"]);
|
|
43
|
+
/** Terminal statuses that count as the node's work having landed. */
|
|
44
|
+
const DONE_STATUSES = new Set(["done", "no-op"]);
|
|
45
|
+
/** Gate review that blocks the node on a failing verdict (TECH-SPEC lean, rule 2). */
|
|
46
|
+
const BLOCKING_REVIEW = "blocking";
|
|
47
|
+
/** A receipt this settled: delivered, no transport bound, or the retry budget spent. */
|
|
48
|
+
const SETTLED_NOTIFY_STATUSES = new Set(["delivered", "no_transport"]);
|
|
49
|
+
/** Target latency for a terminal/attention event to carry a settled receipt (TECH-SPEC section 6). */
|
|
50
|
+
const NOTIFY_TARGET_SEC = 60;
|
|
51
|
+
const SECONDS_PER_HOUR = 3600;
|
|
52
|
+
/** A usage record with no provider-reported cost is `unknown` provenance (`appendUsageRecord`). */
|
|
53
|
+
const UNKNOWN_COST_PROVENANCE = "unknown";
|
|
54
|
+
|
|
55
|
+
/** @typedef {Record<string, unknown>} JsonObject */
|
|
56
|
+
/** @typedef {"down"|"up"|"informative"} Direction */
|
|
57
|
+
/** @typedef {{value: number|null, direction: Direction, count: number}} Indicator */
|
|
58
|
+
/** @typedef {Indicator & {unknownCount: number}} CostIndicator */
|
|
59
|
+
/** @typedef {{value: Record<string, number>|null, direction: Direction, count: number}} GroupedIndicator */
|
|
60
|
+
/** @typedef {{atMs: number, index: number, event: JsonObject}} RunEvent */
|
|
61
|
+
/** @typedef {{runId: string, id: string, status: string, attempt?: number|null, revisions?: number|null, review?: string|null}} RunNode */
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* @typedef {{
|
|
65
|
+
* events?: unknown[],
|
|
66
|
+
* usageRecords?: unknown[],
|
|
67
|
+
* notifications?: unknown[],
|
|
68
|
+
* nodes?: RunNode[],
|
|
69
|
+
* now?: number,
|
|
70
|
+
* }} MetricsInput
|
|
71
|
+
*/
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* @typedef {{
|
|
75
|
+
* nodesDoneRate: Indicator,
|
|
76
|
+
* linkedRunsPerClosedCheckpoint: Indicator,
|
|
77
|
+
* runsPerCampaign: Indicator,
|
|
78
|
+
* wallClockSec: Indicator,
|
|
79
|
+
* usageTokensByKind: GroupedIndicator,
|
|
80
|
+
* usageTokensByKindByRuntime: GroupedIndicator,
|
|
81
|
+
* usageCostUsd: CostIndicator,
|
|
82
|
+
* blockingJudgeFirstPassRate: GroupedIndicator,
|
|
83
|
+
* notifyReceiptRate: Indicator,
|
|
84
|
+
* silentStallRate: Indicator,
|
|
85
|
+
* }} CampaignMetrics
|
|
86
|
+
*/
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Project every section-6 indicator from one campaign's recorded artefacts.
|
|
90
|
+
* Pure and deterministic: identical records yield identical output, rates
|
|
91
|
+
* round to 4 decimals, and durations are seconds.
|
|
92
|
+
*
|
|
93
|
+
* @param {MetricsInput} [input]
|
|
94
|
+
* @returns {CampaignMetrics}
|
|
95
|
+
*/
|
|
96
|
+
export function projectMetrics({ events = [], usageRecords = [], notifications = [], nodes = [], now = Date.now() } = {}) {
|
|
97
|
+
void now;
|
|
98
|
+
const eventList = events.map(jsonObjectOf).filter((event) => event !== null);
|
|
99
|
+
const nodesDone = nodesDoneRateOf(nodes);
|
|
100
|
+
const runs = runsPerCampaignOf(nodes, eventList);
|
|
101
|
+
const closedCheckpoints = closedCheckpointsOf(nodes);
|
|
102
|
+
const span = eventSpanOf(eventList);
|
|
103
|
+
const usage = usageTotalsOf(usageRecords);
|
|
104
|
+
const gates = blockingJudgeFirstPassRateOf(nodes, eventList);
|
|
105
|
+
const notify = notifyReceiptRateOf(notifications);
|
|
106
|
+
const stalls = silentStallRateOf(nodes, eventList);
|
|
107
|
+
return {
|
|
108
|
+
nodesDoneRate: measured("up", nodesDone.terminal, nodesDone.terminal === 0 ? null : nodesDone.done / nodesDone.terminal),
|
|
109
|
+
linkedRunsPerClosedCheckpoint: measured("down", closedCheckpoints, closedCheckpoints === 0 ? null : runs / closedCheckpoints),
|
|
110
|
+
runsPerCampaign: measured("down", runs, runs === 0 ? null : runs),
|
|
111
|
+
wallClockSec: measured("down", span.count, span.seconds),
|
|
112
|
+
usageTokensByKind: grouped("informative", usage.tokenCount, usage.tokensByKind),
|
|
113
|
+
usageTokensByKindByRuntime: grouped("informative", usage.tokenCount, usage.tokensByKindByRuntime),
|
|
114
|
+
usageCostUsd: { ...measured("down", usage.costCount, usage.costCount === 0 ? null : usage.costUsd), unknownCount: usage.unknownCount },
|
|
115
|
+
blockingJudgeFirstPassRate: grouped("up", gates.count, gates.value),
|
|
116
|
+
notifyReceiptRate: measured("up", notify.count, notify.count === 0 ? null : notify.satisfied / notify.count),
|
|
117
|
+
silentStallRate: measured("down", stalls.activeIntervals, stalls.activeHours === 0 ? null : stalls.stalled / stalls.activeHours),
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Wrap one scalar indicator. A count of zero is a missing measurement and
|
|
123
|
+
* yields a null value whatever was computed; a non-finite value is missing too.
|
|
124
|
+
*
|
|
125
|
+
* @param {Direction} direction
|
|
126
|
+
* @param {number} count
|
|
127
|
+
* @param {number|null} value
|
|
128
|
+
* @returns {Indicator}
|
|
129
|
+
*/
|
|
130
|
+
function measured(direction, count, value) {
|
|
131
|
+
const missing = count === 0 || value === null || !Number.isFinite(value);
|
|
132
|
+
return { value: missing ? null : round4(/** @type {number} */ (value)), direction, count };
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Wrap one indicator reported per group (lane or runtime). The empty group map
|
|
137
|
+
* is a missing measurement, not a measured zero.
|
|
138
|
+
*
|
|
139
|
+
* @param {Direction} direction
|
|
140
|
+
* @param {number} count
|
|
141
|
+
* @param {Record<string, number>} value
|
|
142
|
+
* @returns {GroupedIndicator}
|
|
143
|
+
*/
|
|
144
|
+
function grouped(direction, count, value) {
|
|
145
|
+
return { value: count === 0 || Object.keys(value).length === 0 ? null : value, direction, count };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Logical nodes done at any attempt, over logical nodes that reached a
|
|
150
|
+
* terminal state. A node still `pending` or `running` at close is censored:
|
|
151
|
+
* it counts toward neither side (TECH-SPEC section 6 denominators).
|
|
152
|
+
*
|
|
153
|
+
* @param {RunNode[]} nodes
|
|
154
|
+
* @returns {{done: number, terminal: number}}
|
|
155
|
+
*/
|
|
156
|
+
function nodesDoneRateOf(nodes) {
|
|
157
|
+
let done = 0;
|
|
158
|
+
let terminal = 0;
|
|
159
|
+
for (const node of nodes) {
|
|
160
|
+
if (OPEN_STATUSES.has(node.status)) continue;
|
|
161
|
+
terminal += 1;
|
|
162
|
+
if (DONE_STATUSES.has(node.status)) done += 1;
|
|
163
|
+
}
|
|
164
|
+
return { done, terminal };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Distinct checkpoints (node ids, deduplicated across every linked run) that
|
|
169
|
+
* closed at least once.
|
|
170
|
+
*
|
|
171
|
+
* @param {RunNode[]} nodes
|
|
172
|
+
* @returns {number}
|
|
173
|
+
*/
|
|
174
|
+
function closedCheckpointsOf(nodes) {
|
|
175
|
+
const closed = new Set();
|
|
176
|
+
for (const node of nodes) if (DONE_STATUSES.has(node.status)) closed.add(node.id);
|
|
177
|
+
return closed.size;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* @param {RunNode[]} nodes
|
|
182
|
+
* @param {JsonObject[]} events
|
|
183
|
+
* @returns {number}
|
|
184
|
+
*/
|
|
185
|
+
function runsPerCampaignOf(nodes, events) {
|
|
186
|
+
const runIds = new Set();
|
|
187
|
+
for (const node of nodes) runIds.add(node.runId);
|
|
188
|
+
for (const event of events) if (typeof event.runId === "string") runIds.add(event.runId);
|
|
189
|
+
return runIds.size;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Wall-clock span of the recording, in seconds, with the number of timestamped
|
|
194
|
+
* events it was measured from. A single event spans nothing measurable.
|
|
195
|
+
*
|
|
196
|
+
* @param {JsonObject[]} events
|
|
197
|
+
* @returns {{seconds: number|null, count: number}}
|
|
198
|
+
*/
|
|
199
|
+
function eventSpanOf(events) {
|
|
200
|
+
let earliest = Number.POSITIVE_INFINITY;
|
|
201
|
+
let latest = Number.NEGATIVE_INFINITY;
|
|
202
|
+
let count = 0;
|
|
203
|
+
for (const event of events) {
|
|
204
|
+
const atMs = timestampMs(event.at);
|
|
205
|
+
if (!Number.isFinite(atMs)) continue;
|
|
206
|
+
count += 1;
|
|
207
|
+
if (atMs < earliest) earliest = atMs;
|
|
208
|
+
if (atMs > latest) latest = atMs;
|
|
209
|
+
}
|
|
210
|
+
return count < 2 ? { seconds: null, count: 0 } : { seconds: (latest - earliest) / 1000, count };
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Tokens by kind and total cost across every linked run's usage records, both
|
|
215
|
+
* as a campaign total and broken out per runtime (TECH-SPEC section 6). A
|
|
216
|
+
* record contributes exactly what it recorded: uncached input, cache-read
|
|
217
|
+
* input and output tokens are independent totals. Cost sums only records
|
|
218
|
+
* whose provenance is not `unknown` (`appendUsageRecord` sets `unknown`
|
|
219
|
+
* exactly when the provider reported no cost); every other record's
|
|
220
|
+
* invocation is counted separately rather than folded into a measured zero.
|
|
221
|
+
*
|
|
222
|
+
* @param {unknown[]} usageRecords
|
|
223
|
+
* @returns {{tokensByKind: Record<string, number>, tokensByKindByRuntime: Record<string, number>, tokenCount: number, costUsd: number|null, costCount: number, unknownCount: number}}
|
|
224
|
+
*/
|
|
225
|
+
function usageTotalsOf(usageRecords) {
|
|
226
|
+
const tokensByKind = { inputTokens: 0, cacheReadInputTokens: 0, outputTokens: 0 };
|
|
227
|
+
/** @type {Record<string, number>} */
|
|
228
|
+
const tokensByKindByRuntime = {};
|
|
229
|
+
let tokenCount = 0;
|
|
230
|
+
let costUsd = 0;
|
|
231
|
+
let costCount = 0;
|
|
232
|
+
let unknownCount = 0;
|
|
233
|
+
const kinds = /** @type {("inputTokens"|"cacheReadInputTokens"|"outputTokens")[]} */ (["inputTokens", "cacheReadInputTokens", "outputTokens"]);
|
|
234
|
+
for (const raw of usageRecords) {
|
|
235
|
+
const record = jsonObjectOf(raw);
|
|
236
|
+
if (record === null) continue;
|
|
237
|
+
let measuredAny = false;
|
|
238
|
+
const runtime = typeof record.runtimeId === "string" && record.runtimeId !== "" ? record.runtimeId : null;
|
|
239
|
+
for (const kind of kinds) {
|
|
240
|
+
if (typeof record[kind] !== "number" || !Number.isFinite(record[kind])) continue;
|
|
241
|
+
const value = /** @type {number} */ (record[kind]);
|
|
242
|
+
tokensByKind[kind] += value;
|
|
243
|
+
if (runtime !== null) tokensByKindByRuntime[`${runtime}.${kind}`] = (tokensByKindByRuntime[`${runtime}.${kind}`] ?? 0) + value;
|
|
244
|
+
measuredAny = true;
|
|
245
|
+
}
|
|
246
|
+
if (measuredAny) tokenCount += 1;
|
|
247
|
+
if (typeof record.costUsd === "number" && Number.isFinite(record.costUsd) && record.costProvenance !== UNKNOWN_COST_PROVENANCE) {
|
|
248
|
+
costUsd += record.costUsd;
|
|
249
|
+
costCount += 1;
|
|
250
|
+
} else {
|
|
251
|
+
unknownCount += 1;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
return { tokensByKind, tokensByKindByRuntime, tokenCount, costUsd: costCount === 0 ? null : costUsd, costCount, unknownCount };
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* Fraction of blocking-gated checkpoints whose first recorded verdict passed,
|
|
259
|
+
* per lane (the judge runtime that produced it). A node under `advisory` or
|
|
260
|
+
* `none` review never blocks the campaign on a fail, so it is not what this
|
|
261
|
+
* indicator measures (TECH-SPEC section 6, "Blocking judge first-pass rate").
|
|
262
|
+
*
|
|
263
|
+
* @param {RunNode[]} nodes
|
|
264
|
+
* @param {JsonObject[]} events
|
|
265
|
+
* @returns {{value: Record<string, number>, count: number}}
|
|
266
|
+
*/
|
|
267
|
+
function blockingJudgeFirstPassRateOf(nodes, events) {
|
|
268
|
+
/** @type {Map<string, string|null>} */
|
|
269
|
+
const reviewByKey = new Map();
|
|
270
|
+
for (const node of nodes) reviewByKey.set(`${node.runId}:${node.id}`, node.review ?? null);
|
|
271
|
+
const byKey = groupEventsByRunNode(events);
|
|
272
|
+
/** @type {Map<string, {gated: number, passed: number}>} */
|
|
273
|
+
const lanes = new Map();
|
|
274
|
+
let gated = 0;
|
|
275
|
+
for (const [key, entries] of byKey) {
|
|
276
|
+
if (reviewByKey.get(key) !== BLOCKING_REVIEW) continue;
|
|
277
|
+
const first = entries.find(({ event }) => typeof event.verdict === "string");
|
|
278
|
+
if (first === undefined) continue;
|
|
279
|
+
gated += 1;
|
|
280
|
+
const lane = typeof first.event.runtime === "string" && first.event.runtime !== "" ? first.event.runtime : "unknown";
|
|
281
|
+
const tally = lanes.get(lane) ?? { gated: 0, passed: 0 };
|
|
282
|
+
tally.gated += 1;
|
|
283
|
+
if (first.event.verdict === "pass") tally.passed += 1;
|
|
284
|
+
lanes.set(lane, tally);
|
|
285
|
+
}
|
|
286
|
+
/** @type {Record<string, number>} */
|
|
287
|
+
const value = {};
|
|
288
|
+
for (const lane of [...lanes.keys()].sort()) {
|
|
289
|
+
const tally = /** @type {{gated: number, passed: number}} */ (lanes.get(lane));
|
|
290
|
+
value[lane] = round4(tally.passed / tally.gated);
|
|
291
|
+
}
|
|
292
|
+
return { value, count: gated };
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Fraction of notified events (grouped by `dedupeKey`, one per logical
|
|
297
|
+
* terminal/attention transition) that reached a settled receipt — delivered,
|
|
298
|
+
* no transport bound, or failed after the bounded retry budget — within
|
|
299
|
+
* `NOTIFY_TARGET_SEC` of the first attempt (TECH-SPEC section 6).
|
|
300
|
+
*
|
|
301
|
+
* @param {unknown[]} notifications
|
|
302
|
+
* @returns {{satisfied: number, count: number}}
|
|
303
|
+
*/
|
|
304
|
+
function notifyReceiptRateOf(notifications) {
|
|
305
|
+
/** @type {Map<string, JsonObject[]>} */
|
|
306
|
+
const byKey = new Map();
|
|
307
|
+
for (const raw of notifications) {
|
|
308
|
+
const record = jsonObjectOf(raw);
|
|
309
|
+
if (record === null || typeof record.dedupeKey !== "string" || record.dedupeKey === "") continue;
|
|
310
|
+
const list = byKey.get(record.dedupeKey) ?? [];
|
|
311
|
+
list.push(record);
|
|
312
|
+
byKey.set(record.dedupeKey, list);
|
|
313
|
+
}
|
|
314
|
+
let satisfied = 0;
|
|
315
|
+
let count = 0;
|
|
316
|
+
for (const receipts of byKey.values()) {
|
|
317
|
+
const first = receipts.find((receipt) => receipt.attempt === 1);
|
|
318
|
+
if (first === undefined) continue;
|
|
319
|
+
count += 1;
|
|
320
|
+
const firstAtMs = timestampMs(first.at);
|
|
321
|
+
const settled = receipts.find((receipt) => isSettledReceipt(receipt));
|
|
322
|
+
if (settled === undefined) continue;
|
|
323
|
+
const settledAtMs = timestampMs(settled.at);
|
|
324
|
+
if (!Number.isFinite(firstAtMs) || !Number.isFinite(settledAtMs)) continue;
|
|
325
|
+
const deltaSec = (settledAtMs - firstAtMs) / 1000;
|
|
326
|
+
if (deltaSec >= 0 && deltaSec <= NOTIFY_TARGET_SEC) satisfied += 1;
|
|
327
|
+
}
|
|
328
|
+
return { satisfied, count };
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* @param {JsonObject} receipt
|
|
333
|
+
* @returns {boolean}
|
|
334
|
+
*/
|
|
335
|
+
function isSettledReceipt(receipt) {
|
|
336
|
+
if (typeof receipt.status !== "string") return false;
|
|
337
|
+
if (SETTLED_NOTIFY_STATUSES.has(receipt.status)) return true;
|
|
338
|
+
return receipt.status === "failed" && receipt.attempt === NOTIFY_MAX_ATTEMPTS;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* Silent stalls (logical nodes the controller killed for provider silence)
|
|
343
|
+
* per active run-hour. Active time is the sum of every closed `running`
|
|
344
|
+
* interval recorded in `events.jsonl`; a `stalled` status is the controller's
|
|
345
|
+
* own record that a running interval went silent past its timeout, so this
|
|
346
|
+
* needs no heartbeat of its own (TECH-SPEC section 6, hard target zero).
|
|
347
|
+
*
|
|
348
|
+
* @param {RunNode[]} nodes
|
|
349
|
+
* @param {JsonObject[]} events
|
|
350
|
+
* @returns {{stalled: number, activeHours: number, activeIntervals: number}}
|
|
351
|
+
*/
|
|
352
|
+
function silentStallRateOf(nodes, events) {
|
|
353
|
+
const stalled = nodes.filter((node) => node.status === "stalled").length;
|
|
354
|
+
const byKey = groupEventsByRunNode(events);
|
|
355
|
+
let activeSeconds = 0;
|
|
356
|
+
let activeIntervals = 0;
|
|
357
|
+
for (const entries of byKey.values()) {
|
|
358
|
+
for (let index = 0; index < entries.length; index += 1) {
|
|
359
|
+
const event = entries[index].event;
|
|
360
|
+
if (event.to !== "running") continue;
|
|
361
|
+
const next = entries[index + 1];
|
|
362
|
+
if (next === undefined) continue;
|
|
363
|
+
const startMs = entries[index].atMs;
|
|
364
|
+
const endMs = next.atMs;
|
|
365
|
+
if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs < startMs) continue;
|
|
366
|
+
activeSeconds += (endMs - startMs) / 1000;
|
|
367
|
+
activeIntervals += 1;
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
return { stalled, activeHours: activeSeconds / SECONDS_PER_HOUR, activeIntervals };
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/**
|
|
374
|
+
* Group transition events by run and node, ordered by timestamp with the
|
|
375
|
+
* recorded order breaking ties, so a running interval can be paired with the
|
|
376
|
+
* event that ends it.
|
|
377
|
+
*
|
|
378
|
+
* @param {JsonObject[]} events
|
|
379
|
+
* @returns {Map<string, RunEvent[]>}
|
|
380
|
+
*/
|
|
381
|
+
function groupEventsByRunNode(events) {
|
|
382
|
+
/** @type {Map<string, RunEvent[]>} */
|
|
383
|
+
const byKey = new Map();
|
|
384
|
+
events.forEach((event, index) => {
|
|
385
|
+
if (typeof event.node !== "string") return;
|
|
386
|
+
const key = `${typeof event.runId === "string" ? event.runId : ""}:${event.node}`;
|
|
387
|
+
const list = byKey.get(key) ?? [];
|
|
388
|
+
list.push({ atMs: timestampMs(event.at), index, event });
|
|
389
|
+
byKey.set(key, list);
|
|
390
|
+
});
|
|
391
|
+
for (const list of byKey.values()) list.sort((left, right) => orderOf(left) - orderOf(right) || left.index - right.index);
|
|
392
|
+
return byKey;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/** @param {RunEvent} entry @returns {number} */
|
|
396
|
+
function orderOf(entry) {
|
|
397
|
+
return Number.isFinite(entry.atMs) ? entry.atMs : 0;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/** Flags of `runner.mjs metrics`, declared here so the router only names them. */
|
|
401
|
+
/** @type {import("node:util").ParseArgsOptionsConfig} */
|
|
402
|
+
export const METRICS_OPTIONS = { cwd: { type: "string" }, json: { type: "boolean" } };
|
|
403
|
+
|
|
404
|
+
const RUNS_DIR_NAME = ".runs";
|
|
405
|
+
const RUN_EVENTS_FILE = "events.jsonl";
|
|
406
|
+
const USAGE_LOG_FILE = "usage.jsonl";
|
|
407
|
+
const NOTIFY_LOG_FILE = "notify.jsonl";
|
|
408
|
+
|
|
409
|
+
/**
|
|
410
|
+
* @typedef {{
|
|
411
|
+
* campaignId: string,
|
|
412
|
+
* runIds: string[],
|
|
413
|
+
* events: unknown[],
|
|
414
|
+
* usageRecords: unknown[],
|
|
415
|
+
* notifications: unknown[],
|
|
416
|
+
* nodes: RunNode[],
|
|
417
|
+
* }} MetricsSources
|
|
418
|
+
*/
|
|
419
|
+
|
|
420
|
+
/**
|
|
421
|
+
* Read the recorded sources of one campaign: every linked run's persisted
|
|
422
|
+
* node snapshots, transition events (tagged with the run id, since a node id
|
|
423
|
+
* is only unique within one run), `usage.jsonl` and `notify.jsonl`. A missing
|
|
424
|
+
* artefact reads as empty, which the projector reports as a missing
|
|
425
|
+
* measurement and never as a measured zero.
|
|
426
|
+
*
|
|
427
|
+
* @param {string} campaignPath
|
|
428
|
+
* @param {{runsDir?: string}} [options]
|
|
429
|
+
* @returns {MetricsSources}
|
|
430
|
+
*/
|
|
431
|
+
export function readMetricsSources(campaignPath, { runsDir = join(campaignPath, "..", "..") } = {}) {
|
|
432
|
+
const campaign = readCampaign(campaignPath);
|
|
433
|
+
/** @type {unknown[]} */
|
|
434
|
+
const events = [];
|
|
435
|
+
/** @type {unknown[]} */
|
|
436
|
+
const usageRecords = [];
|
|
437
|
+
/** @type {unknown[]} */
|
|
438
|
+
const notifications = [];
|
|
439
|
+
/** @type {RunNode[]} */
|
|
440
|
+
const nodes = [];
|
|
441
|
+
for (const runId of campaign.linkedRunIds) {
|
|
442
|
+
for (const record of readJsonlRecords(join(runsDir, runId, RUN_EVENTS_FILE))) events.push({ ...jsonObjectOf(record), runId });
|
|
443
|
+
for (const record of readJsonlRecords(join(runsDir, runId, USAGE_LOG_FILE))) usageRecords.push(record);
|
|
444
|
+
for (const record of readJsonlRecords(join(runsDir, runId, NOTIFY_LOG_FILE))) notifications.push(record);
|
|
445
|
+
for (const node of readRunNodes(join(runsDir, runId))) nodes.push({ ...node, runId });
|
|
446
|
+
}
|
|
447
|
+
return { campaignId: campaign.id, runIds: [...campaign.linkedRunIds], events, usageRecords, notifications, nodes };
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* Persisted node snapshots of one run, reduced to the fields metrics reads.
|
|
452
|
+
* Reading is tolerant of a run directory with no `nodes/` yet (freshly
|
|
453
|
+
* dispatched) and of a snapshot that fails to parse (never blocks a report on
|
|
454
|
+
* a torn write).
|
|
455
|
+
*
|
|
456
|
+
* @param {string} runDir
|
|
457
|
+
* @returns {Omit<RunNode, "runId">[]}
|
|
458
|
+
*/
|
|
459
|
+
function readRunNodes(runDir) {
|
|
460
|
+
/** @type {Omit<RunNode, "runId">[]} */
|
|
461
|
+
const nodes = [];
|
|
462
|
+
for (const name of listNodeSnapshots(runDir)) {
|
|
463
|
+
let record;
|
|
464
|
+
try {
|
|
465
|
+
record = jsonObjectOf(JSON.parse(readFileSync(nodeSnapshotPath(runDir, name.slice(0, -".json".length)), "utf8")));
|
|
466
|
+
} catch {
|
|
467
|
+
continue;
|
|
468
|
+
}
|
|
469
|
+
if (record === null || typeof record.id !== "string" || typeof record.status !== "string") continue;
|
|
470
|
+
nodes.push({
|
|
471
|
+
id: record.id,
|
|
472
|
+
status: record.status,
|
|
473
|
+
attempt: typeof record.attempt === "number" ? record.attempt : null,
|
|
474
|
+
revisions: typeof record.revisions === "number" ? record.revisions : null,
|
|
475
|
+
review: typeof record.review === "string" ? record.review : null,
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
return nodes;
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
/**
|
|
482
|
+
* `runner.mjs metrics <campaign-id> [--cwd <dir>] [--json]`: project the
|
|
483
|
+
* campaign's recorded artefacts and return what the command prints. Reading
|
|
484
|
+
* only, and never a write: a report of a closed campaign must not touch it.
|
|
485
|
+
*
|
|
486
|
+
* @param {string} campaignId
|
|
487
|
+
* @param {{cwd?: unknown, json?: unknown}} [values]
|
|
488
|
+
* @returns {string}
|
|
489
|
+
*/
|
|
490
|
+
export function renderCampaignMetrics(campaignId, values = {}) {
|
|
491
|
+
const runsDir = join(resolve(typeof values.cwd === "string" && values.cwd !== "" ? values.cwd : process.cwd()), RUNS_DIR_NAME);
|
|
492
|
+
const sources = readMetricsSources(campaignDir(runsDir, campaignId), { runsDir });
|
|
493
|
+
const metrics = projectMetrics(sources);
|
|
494
|
+
return values.json === true ? renderMetricsJson(sources, metrics) : renderMetricsReport(sources, metrics);
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
/**
|
|
498
|
+
* Records of one JSONL artefact. An unterminated final line was never a
|
|
499
|
+
* committed record — the newline is written with the record — so it is skipped
|
|
500
|
+
* rather than parsed.
|
|
501
|
+
*
|
|
502
|
+
* @param {string} path
|
|
503
|
+
* @returns {unknown[]}
|
|
504
|
+
*/
|
|
505
|
+
function readJsonlRecords(path) {
|
|
506
|
+
if (!existsSync(path)) return [];
|
|
507
|
+
const text = readFileSync(path, "utf8");
|
|
508
|
+
const lines = text.split(/\r?\n/u);
|
|
509
|
+
/** @type {unknown[]} */
|
|
510
|
+
const records = [];
|
|
511
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
512
|
+
if (!lines[index].trim()) continue;
|
|
513
|
+
if (index === lines.length - 1 && !text.endsWith("\n")) continue;
|
|
514
|
+
records.push(JSON.parse(lines[index]));
|
|
515
|
+
}
|
|
516
|
+
return records;
|
|
517
|
+
}
|