@feigi/fleet-ctl 3.20.0 → 3.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/ambient-git-vars-mjs-prose.test.mjs +12 -0
- package/scripts/fleet-tick.mjs +65 -3
- package/scripts/fleet-tick.test.mjs +99 -0
- package/scripts/member-outcomes.mjs +21 -12
- package/scripts/member-outcomes.test.mjs +68 -11
- package/scripts/member-record.mjs +43 -13
- package/scripts/member-record.test.mjs +24 -0
- package/scripts/pr-cost.mjs +493 -0
- package/scripts/pr-cost.test.mjs +521 -0
- package/scripts/review-core-booking.test.mjs +35 -0
- package/scripts/review-core-cwd-isolation.test.mjs +5 -5
- package/scripts/review-core-shared-test-run.test.mjs +1 -1
- package/scripts/review-core-testcmd.test.mjs +1 -1
- package/scripts/review-core.mjs +8 -4
- package/scripts/review-host-fixture.mjs +12 -3
- package/scripts/review-in-run-retry.test.mjs +4 -4
- package/scripts/select-dimensions.test.mjs +2 -2
|
@@ -0,0 +1,493 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// $ per merged PR, per implementer cell, and the cost guard over it.
|
|
3
|
+
//
|
|
4
|
+
// pr-cost.mjs [--guard] [--json] [--member-outcomes <tsv>] [--tier-outcomes <tsv>]
|
|
5
|
+
// [--ticket-features <tsv>] [--pricing <json>] [--out <json>]
|
|
6
|
+
//
|
|
7
|
+
// Inputs: the three metrics TSVs (docs/metrics/ by default, relative to the
|
|
8
|
+
// working directory; ticket-features.tsv is written by the router script at
|
|
9
|
+
// dispatch and must exist, else exit 2 — pricing.json and the router-table.json
|
|
10
|
+
// beside this script are optional, though a `--pricing` named but absent is
|
|
11
|
+
// exit 2 too) and ONE `gh pr list --state all`, scoped to PRs created in the
|
|
12
|
+
// window, for merged state, which tier-outcomes.tsv does not record. A PR that
|
|
13
|
+
// list does not return is counted pending. Output: a TSV per cell on stdout
|
|
14
|
+
// (`--json` prints the whole report instead). `--guard` also writes the
|
|
15
|
+
// verdict to `.fleet/cost-guard.json` in the main workspace — where fleet-tick
|
|
16
|
+
// reads it, wherever this runs from; `--out` overrides — and exits on it:
|
|
17
|
+
//
|
|
18
|
+
// 0 ok — no cell tripped
|
|
19
|
+
// 3 at least one cell tripped; `tripped[]` names each
|
|
20
|
+
// 4 no verdict — the baseline cell has fewer than MIN_N merged PRs
|
|
21
|
+
// 2 input error — nothing is written
|
|
22
|
+
//
|
|
23
|
+
// The exit status is the guard's carrier: the verdict never lives only in
|
|
24
|
+
// prose. The router (not in the tree yet; ADR 0016) is to read the file, and
|
|
25
|
+
// fleet-tick's `router` row prints it today; a missing or unparseable file
|
|
26
|
+
// means the default cell only, so deleting it cannot evade the guard.
|
|
27
|
+
//
|
|
28
|
+
// BOOKING. A Pull is a ticket-features.tsv row; its $ is the cost of the
|
|
29
|
+
// member-outcomes.tsv row with the same `session` + `agent`, plus every member
|
|
30
|
+
// that one dispatched (an `agent` path nested under it). A PR's $ is every
|
|
31
|
+
// Pull of its ticket(s) up to the ruling — superseded re-dispatches included —
|
|
32
|
+
// plus every member whose NAME carries the PR (`review-pr-<n>`, `fix-pr-<n>`,
|
|
33
|
+
// the finisher spellings, the review fan-out labels; member-record.mjs's
|
|
34
|
+
// parseMemberName decides), with their own nested members. `merge-bot-*`,
|
|
35
|
+
// `memory` and `__advisor` are booked nowhere: their cost does not vary with
|
|
36
|
+
// the implementer's cell. Only omp rows are read; a blank `cost` books 0 and
|
|
37
|
+
// is counted as unpriced, and a Pull with no member row at all books 0 and is
|
|
38
|
+
// listed as unbooked.
|
|
39
|
+
//
|
|
40
|
+
// A ticket's RULING is its last tier-outcomes.tsv row dated on or after its
|
|
41
|
+
// first Pull in the window; that row names the PR and carries the quality
|
|
42
|
+
// verdict. The PR's CELL is the `chosen_cell` of the last Pull dated on or
|
|
43
|
+
// before the ruling, cross-checked against that Pull's member row: a
|
|
44
|
+
// `subagent_type` other than `fleet-implementer-<cell>`, or an `effort` other
|
|
45
|
+
// than the cell's level, lists the PR under `mismatch` and books it to no cell
|
|
46
|
+
// at all. A PR still OPEN is pending and booked nowhere yet. A Pull with no
|
|
47
|
+
// ruling books to its own `chosen_cell` as unmerged spend — unless its member
|
|
48
|
+
// opened a PR that is still open, which is pending too.
|
|
49
|
+
//
|
|
50
|
+
// PER CELL: `n_pulls`; `n_merged`; `n_pass` (merged PRs passing the quality
|
|
51
|
+
// floor — failure is `minted_false_claim=yes` or `closed_own_ticket=no`);
|
|
52
|
+
// `fail_rate` = failed / merged; `mean_usd` = every $ booked to the cell —
|
|
53
|
+
// merged, unmerged and superseded — / merged, so a cell that abandons work
|
|
54
|
+
// carries that spend; `median_usd` over merged PRs alone, descriptive;
|
|
55
|
+
// `router_usd`, the Pulls' own routing cost, summed. `usd_diff_ci95` is a
|
|
56
|
+
// bootstrap 95% interval on `mean_usd` minus the baseline's — printed to be
|
|
57
|
+
// read, never judged on.
|
|
58
|
+
|
|
59
|
+
import { execFileSync, spawnSync } from "node:child_process";
|
|
60
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
61
|
+
import { dirname, join } from "node:path";
|
|
62
|
+
import { fileURLToPath } from "node:url";
|
|
63
|
+
import { makeDie, defineFlags, isDigits } from "./arg.mjs";
|
|
64
|
+
import { gitEnv, workspaceDirFromGitCommonDir } from "./git-env.mjs";
|
|
65
|
+
import { isCLI } from "./is-cli.mjs";
|
|
66
|
+
import { CELL } from "./ledger-grammar.mjs";
|
|
67
|
+
import { parseMemberName } from "./member-record.mjs";
|
|
68
|
+
import { parseTsv as parseMemberTsv } from "./member-outcomes.mjs";
|
|
69
|
+
import { parseTierOutcomes } from "./tier-outcomes.mjs";
|
|
70
|
+
|
|
71
|
+
const NAME = "pr-cost";
|
|
72
|
+
const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
|
|
73
|
+
|
|
74
|
+
// The guard's window opens here and never rolls: it is reset only by editing
|
|
75
|
+
// this constant in a reviewed change.
|
|
76
|
+
export const WINDOW_START = "2026-10-03";
|
|
77
|
+
export const BASELINE_CELL = "slow-high";
|
|
78
|
+
// Stage 1 is the model axis at fixed `high`.
|
|
79
|
+
export const STAGE_1 = ["slow-high", "task-high", "smol-high"];
|
|
80
|
+
export const MIN_N = 20;
|
|
81
|
+
// A cell trips when its fail rate is at least this many points above the
|
|
82
|
+
// baseline's, or its mean $ is at least the baseline's.
|
|
83
|
+
export const TRIP_POINTS = 15;
|
|
84
|
+
export const EXIT = { ok: 0, tripped: 3, none: 4 };
|
|
85
|
+
export const GH_PR_LIMIT = 1000;
|
|
86
|
+
|
|
87
|
+
export const FEATURE_COLUMNS = [
|
|
88
|
+
"run_date", "session", "agent", "ticket", "policy_cell", "chosen_cell", "exploration_draw",
|
|
89
|
+
"sizing_src", "sizing_pre", "router_usd", "brief_chars", "criteria", "comments", "age_days",
|
|
90
|
+
"paths", "test_paths", "xrefs", "kind",
|
|
91
|
+
];
|
|
92
|
+
|
|
93
|
+
// ticket-features.tsv: one bare header line naming the columns, then rows.
|
|
94
|
+
// Read by header name; a row whose width differs from the header's is refused.
|
|
95
|
+
export function parseFeatures(text) {
|
|
96
|
+
const lines = String(text ?? "").split("\n").filter((l) => l.trim() && !l.startsWith("#"));
|
|
97
|
+
if (lines.length === 0) throw new Error("ticket-features.tsv has no header line");
|
|
98
|
+
const header = lines[0].split("\t");
|
|
99
|
+
const missing = FEATURE_COLUMNS.filter((c) => !header.includes(c));
|
|
100
|
+
if (missing.length) throw new Error(`ticket-features.tsv header lacks ${missing.join(", ")}`);
|
|
101
|
+
return lines.slice(1).map((l, i) => {
|
|
102
|
+
const cells = l.split("\t");
|
|
103
|
+
if (cells.length !== header.length) {
|
|
104
|
+
throw new Error(`ticket-features.tsv row ${i + 1}: ${cells.length} fields, expected ${header.length} — ${l.slice(0, 60)}`);
|
|
105
|
+
}
|
|
106
|
+
const row = Object.fromEntries(header.map((c, j) => [c, cells[j]]));
|
|
107
|
+
if (!isDigits(row.ticket)) throw new Error(`ticket-features.tsv row ${i + 1}: ticket ${JSON.stringify(row.ticket)} is not a number`);
|
|
108
|
+
if (!CELL.test(row.chosen_cell)) throw new Error(`ticket-features.tsv row ${i + 1}: chosen_cell ${JSON.stringify(row.chosen_cell)} is not a cell`);
|
|
109
|
+
return row;
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const EXCLUDED_MEMBER = /^(?:merge-?bot|memory$|__advisor$)/i;
|
|
114
|
+
// omp nests a member's id as `<parent>/<parent>.<child>`: the child is the last segment after either separator.
|
|
115
|
+
const lastSegment = (agent) => String(agent).split(/[/.]/).at(-1);
|
|
116
|
+
const usd = (r) => (r.cost === "" || r.cost === undefined ? 0 : Number(r.cost));
|
|
117
|
+
const isMerged = (pr) => pr?.state === "MERGED";
|
|
118
|
+
const failed = (tier) => tier.minted_false_claim === "yes" || tier.closed_own_ticket === "no";
|
|
119
|
+
const round = (x, d = 4) => (x === null || !Number.isFinite(x) ? null : Number(x.toFixed(d)));
|
|
120
|
+
|
|
121
|
+
function median(xs) {
|
|
122
|
+
if (xs.length === 0) return null;
|
|
123
|
+
const s = [...xs].sort((a, b) => a - b);
|
|
124
|
+
const m = s.length >> 1;
|
|
125
|
+
return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Deterministic, so a re-run prints the same interval.
|
|
129
|
+
function mulberry32(seed) {
|
|
130
|
+
let a = seed >>> 0;
|
|
131
|
+
return () => {
|
|
132
|
+
a = (a + 0x6d2b79f5) >>> 0;
|
|
133
|
+
let t = a;
|
|
134
|
+
t = Math.imul(t ^ (t >>> 15), t | 1);
|
|
135
|
+
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
|
136
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Each unit is one booked group: `{ usd, merged }`. mean_usd is Σusd / Σmerged.
|
|
141
|
+
export function bootstrapDiffCI(cellUnits, baseUnits, { iterations = 2000, seed = 1 } = {}) {
|
|
142
|
+
if (!cellUnits.length || !baseUnits.length) return null;
|
|
143
|
+
const rand = mulberry32(seed);
|
|
144
|
+
const resample = (units) => {
|
|
145
|
+
let u = 0, m = 0;
|
|
146
|
+
for (let i = 0; i < units.length; i++) {
|
|
147
|
+
const x = units[Math.floor(rand() * units.length)];
|
|
148
|
+
u += x.usd; m += x.merged;
|
|
149
|
+
}
|
|
150
|
+
return m ? u / m : null;
|
|
151
|
+
};
|
|
152
|
+
const diffs = [];
|
|
153
|
+
for (let i = 0; i < iterations; i++) {
|
|
154
|
+
const a = resample(cellUnits), b = resample(baseUnits);
|
|
155
|
+
if (a !== null && b !== null) diffs.push(a - b);
|
|
156
|
+
}
|
|
157
|
+
if (diffs.length < iterations / 2) return null;
|
|
158
|
+
diffs.sort((x, y) => x - y);
|
|
159
|
+
return [round(diffs[Math.floor(0.025 * (diffs.length - 1))], 2), round(diffs[Math.ceil(0.975 * (diffs.length - 1))], 2)];
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// The trip rule, in integer counts so a 3-of-20 difference is exactly 15
|
|
163
|
+
// points rather than 14.999… in floating point.
|
|
164
|
+
export function trips(cell, base) {
|
|
165
|
+
const failC = cell.n - cell.n_pass, failB = base.n - base.n_pass;
|
|
166
|
+
const failRise = (failC * base.n - failB * cell.n) * 100 >= TRIP_POINTS * cell.n * base.n;
|
|
167
|
+
return failRise || cell.mean_usd >= base.mean_usd;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export function computeReport({ members, tiers, features, prs, routerTable = null, pricing = null, windowStart = WINDOW_START }) {
|
|
171
|
+
const omp = members.filter((r) => r.harness === "omp");
|
|
172
|
+
const memberKey = (session, agent) => `${session}\0${agent}`;
|
|
173
|
+
const prState = new Map(prs.map((p) => [String(p.number), p]));
|
|
174
|
+
|
|
175
|
+
// Direct bookings, keyed by the member row they name.
|
|
176
|
+
const pulls = features.filter((f) => f.run_date >= windowStart);
|
|
177
|
+
const pullByKey = new Map(pulls.map((p) => [memberKey(p.session, p.agent), p]));
|
|
178
|
+
const prMemberKeys = new Map();
|
|
179
|
+
for (const r of omp) {
|
|
180
|
+
if (EXCLUDED_MEMBER.test(lastSegment(r.member))) continue;
|
|
181
|
+
if (pullByKey.has(memberKey(r.session, r.agent))) continue;
|
|
182
|
+
const { pr } = parseMemberName(r.member);
|
|
183
|
+
if (pr) prMemberKeys.set(memberKey(r.session, r.agent), pr);
|
|
184
|
+
}
|
|
185
|
+
// Every omp row books to its nearest booked ancestor (itself included).
|
|
186
|
+
const pullUsd = new Map(), prMemberUsd = new Map();
|
|
187
|
+
const memberOfPull = new Map();
|
|
188
|
+
const booked = []; // [row, pull | null, pr | null]
|
|
189
|
+
for (const r of omp) {
|
|
190
|
+
if (EXCLUDED_MEMBER.test(lastSegment(r.member))) continue;
|
|
191
|
+
const segs = String(r.agent).split("/");
|
|
192
|
+
for (let i = segs.length; i > 0; i--) {
|
|
193
|
+
const k = memberKey(r.session, segs.slice(0, i).join("/"));
|
|
194
|
+
const pull = pullByKey.get(k);
|
|
195
|
+
const pr = prMemberKeys.get(k);
|
|
196
|
+
if (!pull && !pr) continue;
|
|
197
|
+
booked.push([r, pull ?? null, pull ? null : pr]);
|
|
198
|
+
if (pull) {
|
|
199
|
+
pullUsd.set(pull, (pullUsd.get(pull) ?? 0) + usd(r));
|
|
200
|
+
if (i === segs.length) memberOfPull.set(pull, r);
|
|
201
|
+
} else prMemberUsd.set(pr, (prMemberUsd.get(pr) ?? 0) + usd(r));
|
|
202
|
+
break;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// Rulings: per ticket, the last tier-outcomes row on or after its first Pull.
|
|
207
|
+
const pullsByTicket = new Map();
|
|
208
|
+
for (const p of pulls) {
|
|
209
|
+
if (!pullsByTicket.has(p.ticket)) pullsByTicket.set(p.ticket, []);
|
|
210
|
+
pullsByTicket.get(p.ticket).push(p);
|
|
211
|
+
}
|
|
212
|
+
const groups = new Map(); // pr -> { pr, tier, pulls[] }
|
|
213
|
+
const unruled = [];
|
|
214
|
+
for (const [ticket, ps] of pullsByTicket) {
|
|
215
|
+
const first = ps[0].run_date;
|
|
216
|
+
const ruling = tiers.filter((t) => t.ticket.split("+").includes(ticket) && t.run_date >= first).at(-1);
|
|
217
|
+
if (!ruling) { unruled.push(...ps); continue; }
|
|
218
|
+
const g = groups.get(ruling.pr) ?? { pr: ruling.pr, tier: ruling, pulls: [] };
|
|
219
|
+
if (ruling.run_date > g.tier.run_date) g.tier = ruling;
|
|
220
|
+
for (const p of ps) (p.run_date <= ruling.run_date ? g.pulls : unruled).push(p);
|
|
221
|
+
groups.set(ruling.pr, g);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
const cells = new Map();
|
|
225
|
+
const cellOf = (c) => {
|
|
226
|
+
if (!cells.has(c)) {
|
|
227
|
+
cells.set(c, { cell: c, n_pulls: 0, n_merged: 0, n_pass: 0, usd: 0, router_usd: 0, merged_usd: [], units: [] });
|
|
228
|
+
}
|
|
229
|
+
return cells.get(c);
|
|
230
|
+
};
|
|
231
|
+
const routerUsd = (p) => (p.router_usd === "" ? 0 : Number(p.router_usd));
|
|
232
|
+
const mismatch = [], pending = [];
|
|
233
|
+
|
|
234
|
+
for (const g of groups.values()) {
|
|
235
|
+
if (g.pulls.length === 0) continue;
|
|
236
|
+
const carrier = g.pulls.at(-1);
|
|
237
|
+
const cell = carrier.chosen_cell;
|
|
238
|
+
const m = memberOfPull.get(carrier);
|
|
239
|
+
const level = cell.split("-").slice(1).join("-");
|
|
240
|
+
if (m && (m.subagentType !== `fleet-implementer-${cell}` || m.effort !== level)) {
|
|
241
|
+
mismatch.push({ pr: g.pr, cell, subagent_type: m.subagentType, effort: m.effort });
|
|
242
|
+
continue;
|
|
243
|
+
}
|
|
244
|
+
const state = prState.get(g.pr);
|
|
245
|
+
if (!state || state.state === "OPEN") { pending.push(g.pr); continue; }
|
|
246
|
+
const total = g.pulls.reduce((s, p) => s + (pullUsd.get(p) ?? 0), 0) + (prMemberUsd.get(g.pr) ?? 0);
|
|
247
|
+
const c = cellOf(cell);
|
|
248
|
+
for (const p of g.pulls) { cellOf(p.chosen_cell).n_pulls++; c.router_usd += routerUsd(p); }
|
|
249
|
+
c.usd += total;
|
|
250
|
+
const merged = isMerged(state) ? 1 : 0;
|
|
251
|
+
c.units.push({ usd: total, merged });
|
|
252
|
+
if (merged) {
|
|
253
|
+
c.n_merged++;
|
|
254
|
+
c.merged_usd.push(total);
|
|
255
|
+
if (!failed(g.tier)) c.n_pass++;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
for (const p of unruled) {
|
|
259
|
+
const opened = memberOfPull.get(p)?.pr;
|
|
260
|
+
if (opened && prState.get(opened)?.state === "OPEN") { pending.push(opened); continue; }
|
|
261
|
+
const c = cellOf(p.chosen_cell);
|
|
262
|
+
c.n_pulls++;
|
|
263
|
+
c.router_usd += routerUsd(p);
|
|
264
|
+
const spent = pullUsd.get(p) ?? 0;
|
|
265
|
+
c.usd += spent;
|
|
266
|
+
c.units.push({ usd: spent, merged: 0 });
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
const base = cells.get(BASELINE_CELL) ?? null;
|
|
270
|
+
const meanUsd = (c) => (c.n_merged ? c.usd / c.n_merged : null);
|
|
271
|
+
const figures = [...cells.values()].sort((a, b) => a.cell.localeCompare(b.cell)).map((c) => ({
|
|
272
|
+
cell: c.cell,
|
|
273
|
+
n_pulls: c.n_pulls,
|
|
274
|
+
n_merged: c.n_merged,
|
|
275
|
+
n_pass: c.n_pass,
|
|
276
|
+
fail_rate: c.n_merged ? round((c.n_merged - c.n_pass) / c.n_merged) : null,
|
|
277
|
+
mean_usd: round(meanUsd(c), 2),
|
|
278
|
+
median_usd: round(median(c.merged_usd), 2),
|
|
279
|
+
router_usd: round(c.router_usd, 4),
|
|
280
|
+
usd_diff_ci95: c.cell === BASELINE_CELL || !base ? null : bootstrapDiffCI(c.units, base.units),
|
|
281
|
+
}));
|
|
282
|
+
|
|
283
|
+
// The guard.
|
|
284
|
+
const byCell = new Map(figures.map((f) => [f.cell, f]));
|
|
285
|
+
const b = byCell.get(BASELINE_CELL) ?? { n_merged: 0, n_pass: 0, mean_usd: null, fail_rate: null };
|
|
286
|
+
const baseline = { cell: BASELINE_CELL, n: b.n_merged, mean_usd: b.mean_usd, fail_rate: b.fail_rate };
|
|
287
|
+
const tripped = [];
|
|
288
|
+
if (baseline.n >= MIN_N) {
|
|
289
|
+
for (const f of figures) {
|
|
290
|
+
if (f.cell === BASELINE_CELL || f.n_merged < MIN_N) continue;
|
|
291
|
+
// The $ leg takes the unrounded means: the cent-rounded figures are for
|
|
292
|
+
// display, and a trip is sticky, so a cell cheaper by under half a cent
|
|
293
|
+
// must not trip on rounding alone.
|
|
294
|
+
const c = cells.get(f.cell);
|
|
295
|
+
if (trips({ n: f.n_merged, n_pass: f.n_pass, mean_usd: meanUsd(c) }, { n: b.n_merged, n_pass: b.n_pass, mean_usd: meanUsd(base) })) tripped.push(f.cell);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
const verdict = baseline.n < MIN_N ? "none" : tripped.length ? "tripped" : "ok";
|
|
299
|
+
// Retire: every non-default stage-1 cell tripped before any stratum adopted one.
|
|
300
|
+
const adopted = Object.values(routerTable?.rows ?? {}).some((c) => c !== BASELINE_CELL);
|
|
301
|
+
const retire = !adopted && STAGE_1.filter((c) => c !== BASELINE_CELL).every((c) => tripped.includes(c));
|
|
302
|
+
// The member rows this report books: every windowed Pull's, and those of
|
|
303
|
+
// PR-named members whose PR a windowed ticket was ruled on.
|
|
304
|
+
const inReport = booked.filter(([, pull, pr]) => pull || groups.has(pr)).map(([r]) => r);
|
|
305
|
+
|
|
306
|
+
return {
|
|
307
|
+
window_start: windowStart,
|
|
308
|
+
baseline,
|
|
309
|
+
cells: figures,
|
|
310
|
+
tripped,
|
|
311
|
+
verdict,
|
|
312
|
+
retire,
|
|
313
|
+
mismatch,
|
|
314
|
+
pending: [...new Set(pending)].sort((x, y) => Number(x) - Number(y)),
|
|
315
|
+
unpriced: inReport.filter((r) => r.cost === "").length,
|
|
316
|
+
// A Pull whose member row never reached member-outcomes.tsv books $0 into
|
|
317
|
+
// its cell's mean, which reads as a cheap run unless it is listed here.
|
|
318
|
+
unbooked_pulls: pulls.filter((p) => !pullUsd.has(p)).map((p) => p.agent),
|
|
319
|
+
ab: abReport(groups, pulls, prState, pullUsd, prMemberUsd),
|
|
320
|
+
cross_check: crossCheck(inReport, pricing),
|
|
321
|
+
};
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
// The reasoning test: arm A on an even ticket, B on an odd one. B has run only
|
|
325
|
+
// if some odd-ticket Pull carries a `sizing_pre`.
|
|
326
|
+
export function abReport(groups, pulls, prState, pullUsd, prMemberUsd) {
|
|
327
|
+
const arm = (ticket) => (Number(ticket) % 2 === 0 ? "A" : "B");
|
|
328
|
+
const bPulls = pulls.filter((p) => arm(p.ticket) === "B");
|
|
329
|
+
if (!bPulls.some((p) => p.sizing_pre !== "")) {
|
|
330
|
+
return { status: "not-run", reason: "B not run — the table has no row a better classifier could change" };
|
|
331
|
+
}
|
|
332
|
+
const arms = { A: { merged: 0, failed: 0, usd: 0 }, B: { merged: 0, failed: 0, usd: 0 } };
|
|
333
|
+
for (const g of groups.values()) {
|
|
334
|
+
if (!g.pulls.length || !isMerged(prState.get(g.pr))) continue;
|
|
335
|
+
const a = arms[arm(g.pulls.at(-1).ticket)];
|
|
336
|
+
a.merged++;
|
|
337
|
+
if (failed(g.tier)) a.failed++;
|
|
338
|
+
a.usd += g.pulls.reduce((s, p) => s + (pullUsd.get(p) ?? 0) + (p.router_usd === "" ? 0 : Number(p.router_usd)), 0)
|
|
339
|
+
+ (prMemberUsd.get(g.pr) ?? 0);
|
|
340
|
+
}
|
|
341
|
+
const disagree = bPulls.filter((p) => p.policy_cell !== p.chosen_cell).length;
|
|
342
|
+
const out = {
|
|
343
|
+
A: { n_merged: arms.A.merged, fail_rate: arms.A.merged ? round(arms.A.failed / arms.A.merged) : null, mean_usd: arms.A.merged ? round(arms.A.usd / arms.A.merged, 2) : null },
|
|
344
|
+
B: { n_merged: arms.B.merged, fail_rate: arms.B.merged ? round(arms.B.failed / arms.B.merged) : null, mean_usd: arms.B.merged ? round(arms.B.usd / arms.B.merged, 2) : null },
|
|
345
|
+
disagreement: bPulls.length ? round(disagree / bPulls.length) : null,
|
|
346
|
+
};
|
|
347
|
+
// B can only show an effect through the Pulls where it chose differently.
|
|
348
|
+
if (arms.A.merged < MIN_N || arms.B.merged < MIN_N) return { status: "insufficient", ...out };
|
|
349
|
+
if (disagree < MIN_N) return { status: "underpowered", ...out };
|
|
350
|
+
const bWins = out.B.mean_usd < out.A.mean_usd && arms.B.failed * arms.A.merged <= arms.A.failed * arms.B.merged;
|
|
351
|
+
return { status: bWins ? "B-wins" : "B-off", ...out };
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
// How far each booked row's recorded `cost` sits from its token columns priced
|
|
355
|
+
// at pricing.json's rates — a ratio, never a $.
|
|
356
|
+
export function crossCheck(rows, pricing) {
|
|
357
|
+
if (!pricing) return null;
|
|
358
|
+
let recorded = 0, expected = 0, n = 0, skipped = 0;
|
|
359
|
+
for (const r of rows) {
|
|
360
|
+
const p = pricing.models?.[r.model];
|
|
361
|
+
const tok = (k) => Number(r[k] || 0);
|
|
362
|
+
const w1h = r.tokensCacheWrite1h === "" ? null : Number(r.tokensCacheWrite1h);
|
|
363
|
+
const parts = [
|
|
364
|
+
[tok("tokensIn"), p?.input], [tok("tokensOut"), p?.output], [tok("tokensCacheRead"), p?.cache_read],
|
|
365
|
+
[w1h, p?.cache_write_1h], [w1h === null ? null : tok("tokensCacheCreate") - w1h, p?.cache_write_5m],
|
|
366
|
+
];
|
|
367
|
+
if (!p || r.cost === "" || parts.some(([t, price]) => t === null || (t > 0 && typeof price !== "number"))) { skipped++; continue; }
|
|
368
|
+
recorded += Number(r.cost);
|
|
369
|
+
expected += parts.reduce((s, [t, price]) => s + (t > 0 ? t * price : 0), 0) / 1e6;
|
|
370
|
+
n++;
|
|
371
|
+
}
|
|
372
|
+
return { rows: n, skipped, ratio: expected > 0 ? round(recorded / expected) : null };
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
const TSV_COLUMNS = ["cell", "n_pulls", "n_merged", "n_pass", "fail_rate", "mean_usd", "median_usd", "router_usd", "usd_diff_ci95", "guard"];
|
|
376
|
+
|
|
377
|
+
export function formatReport(report) {
|
|
378
|
+
const guardOf = (f) => {
|
|
379
|
+
if (f.cell === BASELINE_CELL) return "baseline";
|
|
380
|
+
if (report.tripped.includes(f.cell)) return "tripped";
|
|
381
|
+
return f.n_merged >= MIN_N && report.verdict !== "none" ? "ok" : `n<${MIN_N}`;
|
|
382
|
+
};
|
|
383
|
+
const cell = (v) => (v === null || v === undefined ? "" : Array.isArray(v) ? `[${v.join(", ")}]` : String(v));
|
|
384
|
+
const lines = [TSV_COLUMNS.join("\t")];
|
|
385
|
+
for (const f of report.cells) lines.push(TSV_COLUMNS.map((c) => (c === "guard" ? guardOf(f) : cell(f[c]))).join("\t"));
|
|
386
|
+
const b = report.baseline;
|
|
387
|
+
lines.push(`# verdict: ${report.verdict} (baseline ${b.cell} n=${b.n}/${MIN_N}${report.tripped.length ? `; tripped ${report.tripped.join(" ")}` : ""})${report.retire ? " — retire: every non-default stage-1 cell tripped" : ""}`);
|
|
388
|
+
lines.push(`# window_start ${report.window_start}; pending PRs: ${report.pending.length ? report.pending.join(" ") : "none"}; unpriced member rows: ${report.unpriced}`);
|
|
389
|
+
if (report.mismatch.length) {
|
|
390
|
+
lines.push(`# mismatch (excluded from every cell): ${report.mismatch.map((m) => `PR#${m.pr} ${m.cell} vs ${m.subagent_type || "(none)"}/${m.effort || "(none)"}`).join("; ")}`);
|
|
391
|
+
}
|
|
392
|
+
if (report.unbooked_pulls.length) {
|
|
393
|
+
lines.push(`# unbooked Pulls (no member row, so $0 in the mean): ${report.unbooked_pulls.join(" ")}`);
|
|
394
|
+
}
|
|
395
|
+
const ab = report.ab;
|
|
396
|
+
lines.push(ab.status === "not-run"
|
|
397
|
+
? `# A/B: ${ab.reason}`
|
|
398
|
+
: `# A/B: ${ab.status} — A n=${ab.A.n_merged} $${cell(ab.A.mean_usd)} fail ${cell(ab.A.fail_rate)}; B n=${ab.B.n_merged} $${cell(ab.B.mean_usd)} fail ${cell(ab.B.fail_rate)}; disagreement ${cell(ab.disagreement)}`);
|
|
399
|
+
const x = report.cross_check;
|
|
400
|
+
lines.push(x ? `# cross-check: recorded cost / pricing.json = ${x.ratio ?? "n/a"} over ${x.rows} rows; ${x.skipped} skipped` : "# cross-check: no pricing.json");
|
|
401
|
+
return lines.join("\n") + "\n";
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
export function guardFile(report, computedAt) {
|
|
405
|
+
return {
|
|
406
|
+
computed_at: computedAt,
|
|
407
|
+
window_start: report.window_start,
|
|
408
|
+
baseline: report.baseline,
|
|
409
|
+
cells: report.cells.map((f) => ({ cell: f.cell, n: f.n_merged, mean_usd: f.mean_usd, fail_rate: f.fail_rate })),
|
|
410
|
+
tripped: report.tripped,
|
|
411
|
+
verdict: report.verdict,
|
|
412
|
+
retire: report.retire,
|
|
413
|
+
// The n the baseline and every judged cell need, so a reader prints it
|
|
414
|
+
// rather than restating it.
|
|
415
|
+
min_n: MIN_N,
|
|
416
|
+
};
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// ---------------------------------------------------------------------------
|
|
420
|
+
// CLI
|
|
421
|
+
// ---------------------------------------------------------------------------
|
|
422
|
+
|
|
423
|
+
// `.fleet/cost-guard.json` in the main workspace — the git common dir's parent,
|
|
424
|
+
// where fleet-tick reads it — not in the cwd, which from a linked worktree is a
|
|
425
|
+
// different directory the tick never looks in. The cwd when git cannot answer.
|
|
426
|
+
function defaultGuardPath() {
|
|
427
|
+
const r = spawnSync("git", ["rev-parse", "--git-common-dir"], { encoding: "utf8", env: gitEnv() });
|
|
428
|
+
return join(workspaceDirFromGitCommonDir(r.stdout) ?? ".", ".fleet", "cost-guard.json");
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
function main() {
|
|
432
|
+
const die = makeDie(NAME);
|
|
433
|
+
const { arg, has, sweep, stray } = defineFlags(die, {
|
|
434
|
+
flags: {
|
|
435
|
+
guard: "bool", json: "bool", "member-outcomes": "value", "tier-outcomes": "value",
|
|
436
|
+
"ticket-features": "value", pricing: "value", out: "value",
|
|
437
|
+
},
|
|
438
|
+
});
|
|
439
|
+
sweep();
|
|
440
|
+
stray();
|
|
441
|
+
const read = (path, ifMissing = "") => {
|
|
442
|
+
try { return readFileSync(path, "utf8"); }
|
|
443
|
+
catch (e) { die(`cannot read ${path}: ${e.code ?? e.message}${e.code === "ENOENT" ? ifMissing : ""}`); }
|
|
444
|
+
};
|
|
445
|
+
const parse = (what, fn) => {
|
|
446
|
+
try { return fn(); }
|
|
447
|
+
catch (e) { die(`${what}: ${e.message}`); }
|
|
448
|
+
};
|
|
449
|
+
const membersPath = arg("member-outcomes") ?? "docs/metrics/member-outcomes.tsv";
|
|
450
|
+
const tiersPath = arg("tier-outcomes") ?? "docs/metrics/tier-outcomes.tsv";
|
|
451
|
+
const featuresPath = arg("ticket-features") ?? "docs/metrics/ticket-features.tsv";
|
|
452
|
+
const pricingArg = arg("pricing");
|
|
453
|
+
const pricingPath = pricingArg ?? "docs/metrics/pricing.json";
|
|
454
|
+
const members = parse(membersPath, () => parseMemberTsv(read(membersPath)));
|
|
455
|
+
const tiers = parse(tiersPath, () => parseTierOutcomes(read(tiersPath)));
|
|
456
|
+
const features = parse(featuresPath, () => parseFeatures(read(featuresPath, " — the router script writes it at dispatch, so there is nothing to price before it has dispatched")));
|
|
457
|
+
// The default pricing.json is optional; one named on the command line is not.
|
|
458
|
+
const pricing = pricingArg === null && !existsSync(pricingPath) ? null : parse(pricingPath, () => JSON.parse(read(pricingPath)));
|
|
459
|
+
const tablePath = join(SCRIPT_DIR, "router-table.json");
|
|
460
|
+
const routerTable = existsSync(tablePath) ? parse(tablePath, () => JSON.parse(read(tablePath))) : null;
|
|
461
|
+
|
|
462
|
+
let out;
|
|
463
|
+
try {
|
|
464
|
+
out = execFileSync("gh", ["pr", "list", "--state", "all", "--search", `created:>=${WINDOW_START}`,
|
|
465
|
+
"--limit", String(GH_PR_LIMIT), "--json", "number,state,mergedAt"], {
|
|
466
|
+
encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], maxBuffer: 64 * 1024 * 1024, env: gitEnv(),
|
|
467
|
+
});
|
|
468
|
+
} catch (e) {
|
|
469
|
+
die(`gh pr list failed: ${e.code ?? (e.signal ? `killed by ${e.signal}` : `exit ${e.status}`)}`);
|
|
470
|
+
}
|
|
471
|
+
const prs = parse("gh pr list", () => JSON.parse(out));
|
|
472
|
+
if (!Array.isArray(prs) || prs.some((p) => !p || typeof p.number !== "number" || typeof p.state !== "string")) {
|
|
473
|
+
die("gh pr list did not return {number,state} rows");
|
|
474
|
+
}
|
|
475
|
+
// Merged state for a PR past the cap would read as missing — pending —
|
|
476
|
+
// silently, so a full page is refused rather than trusted.
|
|
477
|
+
if (prs.length >= GH_PR_LIMIT) die(`gh pr list returned ${prs.length} PRs, its cap — merged state may be truncated`);
|
|
478
|
+
|
|
479
|
+
const report = computeReport({ members, tiers, features, prs, routerTable, pricing });
|
|
480
|
+
process.stdout.write(has("json") ? JSON.stringify(report, null, 2) + "\n" : formatReport(report));
|
|
481
|
+
if (!has("guard")) return;
|
|
482
|
+
const file = arg("out") ?? defaultGuardPath();
|
|
483
|
+
try {
|
|
484
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
485
|
+
writeFileSync(`${file}.tmp`, JSON.stringify(guardFile(report, new Date().toISOString()), null, 2) + "\n");
|
|
486
|
+
renameSync(`${file}.tmp`, file);
|
|
487
|
+
} catch (e) {
|
|
488
|
+
die(`cannot write ${file}: ${e.code ?? e.message}`);
|
|
489
|
+
}
|
|
490
|
+
process.exitCode = EXIT[report.verdict];
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
if (isCLI(import.meta.url)) main();
|