@feigi/fleet-ctl 3.20.0 → 3.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/ambient-git-vars-mjs-prose.test.mjs +12 -0
- package/scripts/fleet-tick.mjs +65 -3
- package/scripts/fleet-tick.test.mjs +99 -0
- package/scripts/member-outcomes.mjs +21 -12
- package/scripts/member-outcomes.test.mjs +68 -11
- package/scripts/member-record.mjs +43 -13
- package/scripts/member-record.test.mjs +24 -0
- package/scripts/pr-cost.mjs +493 -0
- package/scripts/pr-cost.test.mjs +521 -0
- package/scripts/review-core-booking.test.mjs +35 -0
- package/scripts/review-core-cwd-isolation.test.mjs +5 -5
- package/scripts/review-core-shared-test-run.test.mjs +1 -1
- package/scripts/review-core-testcmd.test.mjs +1 -1
- package/scripts/review-core.mjs +8 -4
- package/scripts/review-host-fixture.mjs +12 -3
- package/scripts/review-in-run-retry.test.mjs +4 -4
- package/scripts/select-dimensions.test.mjs +2 -2
package/package.json
CHANGED
|
@@ -255,6 +255,18 @@ const COVERED_MJS = {
|
|
|
255
255
|
// cache reader child gets the same env but unsets both itself, so no
|
|
256
256
|
// fixture can tell its env apart.
|
|
257
257
|
"recipe-prove.mjs": 1,
|
|
258
|
+
// TWO scrubbed children. `defaultGuardPath()`'s `rev-parse --git-common-dir`
|
|
259
|
+
// names the workspace the guard file is written into: an ambient GIT_DIR
|
|
260
|
+
// would answer for ANOTHER repository, and the verdict would land in its
|
|
261
|
+
// `.fleet/` at exit 0 while fleet-tick reads this one's. Measured in
|
|
262
|
+
// pr-cost.test.mjs, "an ambient GIT_DIR naming another repository cannot move
|
|
263
|
+
// the guard file out of the repository pr-cost runs in". The `gh pr list`
|
|
264
|
+
// child is the second: gh's remote resolution follows GIT_DIR too (measured
|
|
265
|
+
// for shortlist.mjs's `probeState()` above), so merged state could be read
|
|
266
|
+
// for another repository's PRs; the gh stub in pr-cost.test.mjs, "an ambient
|
|
267
|
+
// GIT_DIR does not reach the gh child that reads merged state", records
|
|
268
|
+
// that the child sees no GIT_DIR.
|
|
269
|
+
"pr-cost.mjs": 2,
|
|
258
270
|
};
|
|
259
271
|
|
|
260
272
|
// The two-name-only exemption list #1599's second design question answers
|
package/scripts/fleet-tick.mjs
CHANGED
|
@@ -45,6 +45,12 @@
|
|
|
45
45
|
// Anything but `clean` — dirty, unknown, no baseline — prints one
|
|
46
46
|
// MAIN-CHECKOUT-* line and holds every dispatching row until the
|
|
47
47
|
// maintainer clears it; it never refuses the tick.
|
|
48
|
+
// guard `.fleet/cost-guard.json` beside the shortlist, pr-cost.mjs
|
|
49
|
+
// --guard's verdict on the implementer cells. Printed as the
|
|
50
|
+
// `router` row and never acted on here: a missing or unreadable
|
|
51
|
+
// file reads DEFAULT-ONLY, the way the router (not in the tree
|
|
52
|
+
// yet; ADR 0016) is to read it. pr-cost.mjs writes the main
|
|
53
|
+
// workspace's file by default, the one read here.
|
|
48
54
|
//
|
|
49
55
|
// The pure half below is `reconcile()` over the counts `deriveRun()` reads off
|
|
50
56
|
// the ledger; main() does the I/O. Split so the guard table and the reading are
|
|
@@ -61,7 +67,7 @@ import { parseMember, parseToken } from "./ledger-grammar.mjs";
|
|
|
61
67
|
// reads what they leave: the slots this tick's own dispatches fill.
|
|
62
68
|
export function reconcile(s) {
|
|
63
69
|
const rev = reviewers(s);
|
|
64
|
-
return [...mainCheckoutHold(s, [implementers(s, rev.left), ...rev.rows, mergeBot(s)]), ...shortlistRows(s)];
|
|
70
|
+
return [...mainCheckoutHold(s, [implementers(s, rev.left), ...rev.rows, mergeBot(s)]), ...shortlistRows(s), ...routerRows(s)];
|
|
65
71
|
}
|
|
66
72
|
|
|
67
73
|
// #2210: a main checkout changed since the run's baseline — or one the tick
|
|
@@ -254,6 +260,59 @@ function shortlistRows(s) {
|
|
|
254
260
|
}];
|
|
255
261
|
}
|
|
256
262
|
|
|
263
|
+
// The cost guard's verdict, as pr-cost.mjs --guard wrote it: OK, DEFAULT-ONLY
|
|
264
|
+
// (a cell tripped, or no readable guard file — the router then runs the
|
|
265
|
+
// default cell alone), or NO VERDICT (the baseline is short of its n). The
|
|
266
|
+
// controller has nothing to do about any of them inside a run, so the row
|
|
267
|
+
// never acts; it is here so the verdict is printed by code, not recalled.
|
|
268
|
+
// Every verdict carries the guard's own `computed_at`: a refresh that failed
|
|
269
|
+
// leaves the old file on disk, and only its age says the OK is not current.
|
|
270
|
+
// No `router` state at all (a caller that read no guard) prints no row.
|
|
271
|
+
export function routerRows(s) {
|
|
272
|
+
const g = s.router;
|
|
273
|
+
if (g === undefined) return [];
|
|
274
|
+
const row = (action, detail) => [{ role: "router", action, acts: false, detail }];
|
|
275
|
+
const run = "run pr-cost.mjs --guard";
|
|
276
|
+
if (g.status === "missing") return row("DEFAULT-ONLY", `cost-guard.json missing — ${run}`);
|
|
277
|
+
if (g.status !== "ok") return row("DEFAULT-ONLY", `cost-guard.json unreadable (${g.why}) — ${run}`);
|
|
278
|
+
const { baseline: b, cells, tripped, verdict, min_n: minN, computed_at: computed } = g.guard;
|
|
279
|
+
const age = `guard computed ${computed}`;
|
|
280
|
+
if (verdict === "none") return row("NO VERDICT", `baseline n=${b.n}/${minN}; ${age}`);
|
|
281
|
+
const pct = (x) => (x === null ? "n/a" : `${Math.round(x * 100)}%`);
|
|
282
|
+
const usd = (x) => (x === null ? "n/a" : `$${x.toFixed(2)}`);
|
|
283
|
+
const vs = (c) => `${c.cell} ${usd(c.mean_usd)} vs ${usd(b.mean_usd)}, fail ${pct(c.fail_rate)} vs ${pct(b.fail_rate)}, n=${c.n}/${b.n}`;
|
|
284
|
+
if (verdict === "tripped") {
|
|
285
|
+
const retired = g.guard.retire ? "; every non-default stage-1 cell tripped — the router retires" : "";
|
|
286
|
+
return row("DEFAULT-ONLY", `cost guard: ${cells.filter((c) => tripped.includes(c.cell)).map(vs).join("; ")}${retired}; ${age}`);
|
|
287
|
+
}
|
|
288
|
+
const others = cells.filter((c) => c.cell !== b.cell);
|
|
289
|
+
return row("OK", `${others.length ? others.map(vs).join("; ") : `baseline ${b.cell} ${usd(b.mean_usd)}, n=${b.n}`}; ${age}`);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
// pr-cost.mjs's guard file, read as the router is to read it: anything but a
|
|
293
|
+
// well-formed, self-consistent verdict is not one.
|
|
294
|
+
export function readCostGuard(path) {
|
|
295
|
+
if (path === null) return { status: "missing" };
|
|
296
|
+
let text;
|
|
297
|
+
try {
|
|
298
|
+
text = readFileSync(path, "utf8");
|
|
299
|
+
} catch (e) {
|
|
300
|
+
return e.code === "ENOENT" ? { status: "missing" } : { status: "unreadable", why: e.code ?? e.message };
|
|
301
|
+
}
|
|
302
|
+
let guard;
|
|
303
|
+
try { guard = JSON.parse(text); } catch { return { status: "unreadable", why: "not JSON" }; }
|
|
304
|
+
const cellShaped = (c) => c && typeof c.cell === "string" && Number.isInteger(c.n)
|
|
305
|
+
&& (c.mean_usd === null || typeof c.mean_usd === "number") && (c.fail_rate === null || typeof c.fail_rate === "number");
|
|
306
|
+
const ok = guard && ["ok", "tripped", "none"].includes(guard.verdict) && typeof guard.computed_at === "string"
|
|
307
|
+
&& cellShaped(guard.baseline) && Array.isArray(guard.cells) && guard.cells.every(cellShaped) && Number.isInteger(guard.min_n)
|
|
308
|
+
&& Array.isArray(guard.tripped) && guard.tripped.every((c) => typeof c === "string")
|
|
309
|
+
// The verdict is "tripped" exactly when a cell is named, and only a reported cell can be named.
|
|
310
|
+
&& (guard.verdict === "tripped") === (guard.tripped.length > 0)
|
|
311
|
+
&& guard.tripped.every((c) => guard.cells.some((x) => x.cell === c))
|
|
312
|
+
&& (guard.retire === undefined || typeof guard.retire === "boolean");
|
|
313
|
+
return ok ? { status: "ok", guard } : { status: "unreadable", why: "not a guard verdict" };
|
|
314
|
+
}
|
|
315
|
+
|
|
257
316
|
export function formatLines(rows) {
|
|
258
317
|
const w = Math.max(...rows.map((r) => r.role.length));
|
|
259
318
|
return rows.map((r) => r.actual === undefined
|
|
@@ -1086,7 +1145,10 @@ function claimed() {
|
|
|
1086
1145
|
|
|
1087
1146
|
function main() {
|
|
1088
1147
|
const { fold, state: path, ...caps } = options();
|
|
1089
|
-
const
|
|
1148
|
+
const listPath = shortlistPath();
|
|
1149
|
+
const current = readShortlist(listPath);
|
|
1150
|
+
// Beside the shortlist, off the same git probe rather than a second one.
|
|
1151
|
+
const router = readCostGuard(listPath === null ? null : join(dirname(listPath), "cost-guard.json"));
|
|
1090
1152
|
|
|
1091
1153
|
// The PRIOR run's liveness, before anything that can refuse — #1597. The gh
|
|
1092
1154
|
// read and the ledger read below both exit 2 on failure, and a stall
|
|
@@ -1142,7 +1204,7 @@ function main() {
|
|
|
1142
1204
|
const mainCheckout = checkMainCheckout();
|
|
1143
1205
|
|
|
1144
1206
|
const rows = reconcile({
|
|
1145
|
-
...caps, ...run, heads, supply, shortlistStatus: current.status, refresh: fresh, mainCheckout,
|
|
1207
|
+
...caps, ...run, heads, supply, shortlistStatus: current.status, refresh: fresh, mainCheckout, router,
|
|
1146
1208
|
});
|
|
1147
1209
|
const said = describe(mainCheckout, run.live);
|
|
1148
1210
|
const lines = [...(said === null ? [] : [said]), ...formatLines(rows)];
|
|
@@ -278,6 +278,93 @@ test("no refresh, no shortlist row", () => {
|
|
|
278
278
|
assert.deepEqual(reconcile(state()).map((r) => r.role), ["implementers", "reviewers", "merge-bot"]);
|
|
279
279
|
});
|
|
280
280
|
|
|
281
|
+
// ---------------------------------------------------------------------------
|
|
282
|
+
// The router row: pr-cost.mjs --guard's verdict, read off a guard file the way
|
|
283
|
+
// the tick reads it. One case per state, each against a fixture file.
|
|
284
|
+
|
|
285
|
+
import { routerRows, readCostGuard } from "./fleet-tick.mjs";
|
|
286
|
+
|
|
287
|
+
const BASE = { cell: "slow-high", n: 31, mean_usd: 31, fail_rate: 0.29 };
|
|
288
|
+
const guardJson = (over = {}) => JSON.stringify({
|
|
289
|
+
computed_at: "2026-10-03T00:00:00.000Z", window_start: "2026-10-03", baseline: BASE,
|
|
290
|
+
cells: [{ cell: "slow-high", n: 31, mean_usd: 31, fail_rate: 0.29 }, { cell: "task-high", n: 24, mean_usd: 12.3, fail_rate: 0.25 }],
|
|
291
|
+
tripped: [], verdict: "ok", retire: false, min_n: 20, ...over,
|
|
292
|
+
});
|
|
293
|
+
function routerRowOf(text) {
|
|
294
|
+
const dir = mkdtempSync(join(tmpdir(), "fleet-tick-guard-"));
|
|
295
|
+
try {
|
|
296
|
+
const path = join(dir, "cost-guard.json");
|
|
297
|
+
if (text !== null) writeFileSync(path, text);
|
|
298
|
+
const rows = reconcile(state({ router: readCostGuard(path) }));
|
|
299
|
+
assert.equal(actionable(rows), false, "no router state is work for the controller");
|
|
300
|
+
const r = rows.filter((x) => x.role === "router");
|
|
301
|
+
assert.equal(r.length, 1);
|
|
302
|
+
assert.equal(rows.at(-1), r[0], "the router row prints last");
|
|
303
|
+
return { ...r[0], line: formatLines(rows).at(-1) };
|
|
304
|
+
} finally {
|
|
305
|
+
rmSync(dir, { recursive: true, force: true });
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
test("router: a guard with no cell tripped is OK, naming each cell against the baseline", () => {
|
|
310
|
+
const r = routerRowOf(guardJson());
|
|
311
|
+
assert.equal(r.action, "OK");
|
|
312
|
+
assert.equal(r.acts, false);
|
|
313
|
+
assert.equal(r.line, "router → OK (task-high $12.30 vs $31.00, fail 25% vs 29%, n=24/31; guard computed 2026-10-03T00:00:00.000Z)");
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
test("router: the row carries the guard's own computed_at, so a stale OK reads as old", () => {
|
|
317
|
+
const r = routerRowOf(guardJson({ computed_at: "2026-09-01T08:00:00.000Z" }));
|
|
318
|
+
assert.equal(r.action, "OK");
|
|
319
|
+
assert.match(r.detail, /; guard computed 2026-09-01T08:00:00\.000Z$/);
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
test("router: a tripped cell is DEFAULT-ONLY, naming the cost guard and the cell it tripped", () => {
|
|
323
|
+
const r = routerRowOf(guardJson({
|
|
324
|
+
verdict: "tripped", tripped: ["smol-high"],
|
|
325
|
+
cells: [{ ...BASE }, { cell: "smol-high", n: 21, mean_usd: 9.5, fail_rate: 0.45 }, { cell: "task-high", n: 24, mean_usd: 12.3, fail_rate: 0.25 }],
|
|
326
|
+
}));
|
|
327
|
+
assert.equal(r.action, "DEFAULT-ONLY");
|
|
328
|
+
assert.equal(r.detail, "cost guard: smol-high $9.50 vs $31.00, fail 45% vs 29%, n=21/31; guard computed 2026-10-03T00:00:00.000Z");
|
|
329
|
+
const retired = routerRowOf(guardJson({ verdict: "tripped", tripped: ["task-high"], retire: true }));
|
|
330
|
+
assert.match(retired.detail, /^cost guard: task-high .*; every non-default stage-1 cell tripped — the router retires; guard computed /);
|
|
331
|
+
});
|
|
332
|
+
|
|
333
|
+
test("router: a baseline short of its n is NO VERDICT, with the count and the n it needs", () => {
|
|
334
|
+
const r = routerRowOf(guardJson({ verdict: "none", baseline: { ...BASE, n: 7 } }));
|
|
335
|
+
assert.equal(r.action, "NO VERDICT");
|
|
336
|
+
assert.equal(r.line, "router → NO VERDICT (baseline n=7/20; guard computed 2026-10-03T00:00:00.000Z)");
|
|
337
|
+
});
|
|
338
|
+
|
|
339
|
+
test("router: no guard file is DEFAULT-ONLY, naming the command that writes one", () => {
|
|
340
|
+
const r = routerRowOf(null);
|
|
341
|
+
assert.equal(r.line, "router → DEFAULT-ONLY (cost-guard.json missing — run pr-cost.mjs --guard)");
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
test("router: a guard file that is not a verdict reads DEFAULT-ONLY too — deleting or breaking it cannot evade the guard", () => {
|
|
345
|
+
const cell = { cell: "task-high", n: 24, mean_usd: 1, fail_rate: 0 };
|
|
346
|
+
for (const [text, why] of [["{", "not JSON"], [guardJson({ verdict: "maybe" }), "not a guard verdict"],
|
|
347
|
+
[guardJson({ tripped: "smol-high" }), "not a guard verdict"], [guardJson({ min_n: undefined }), "not a guard verdict"],
|
|
348
|
+
[guardJson({ computed_at: undefined }), "not a guard verdict"],
|
|
349
|
+
[guardJson({ cells: [{ ...cell, n: "24" }] }), "not a guard verdict"],
|
|
350
|
+
[guardJson({ cells: [{ ...cell, mean_usd: "12" }] }), "not a guard verdict"],
|
|
351
|
+
[guardJson({ cells: [{ ...cell, fail_rate: "0.2" }] }), "not a guard verdict"],
|
|
352
|
+
[guardJson({ baseline: { ...BASE, mean_usd: "31" } }), "not a guard verdict"],
|
|
353
|
+
// Self-contradicting files: a verdict that disagrees with tripped[], a tripped cell the file never reports, a non-boolean retire.
|
|
354
|
+
[guardJson({ verdict: "ok", tripped: ["task-high"] }), "not a guard verdict"],
|
|
355
|
+
[guardJson({ verdict: "tripped", tripped: [] }), "not a guard verdict"],
|
|
356
|
+
[guardJson({ verdict: "tripped", tripped: ["ghost"] }), "not a guard verdict"],
|
|
357
|
+
[guardJson({ retire: "yes" }), "not a guard verdict"]]) {
|
|
358
|
+
const r = routerRowOf(text);
|
|
359
|
+
assert.equal(r.action, "DEFAULT-ONLY", text);
|
|
360
|
+
assert.equal(r.detail, `cost-guard.json unreadable (${why}) — run pr-cost.mjs --guard`, text);
|
|
361
|
+
}
|
|
362
|
+
});
|
|
363
|
+
|
|
364
|
+
test("router: with no guard state read at all, no router row", () => {
|
|
365
|
+
assert.deepEqual(routerRows(state()), []);
|
|
366
|
+
});
|
|
367
|
+
|
|
281
368
|
test("formatLines prints role, actual/target and the ACTION on one line each", () => {
|
|
282
369
|
assert.deepEqual(
|
|
283
370
|
formatLines(reconcile(state({
|
|
@@ -1320,6 +1407,18 @@ test("CLI: runCli with keep: true hands back a fixture dir that still exists", (
|
|
|
1320
1407
|
assert.equal(existsSync(runCli([]).dir), false, "without keep the dir is gone on return");
|
|
1321
1408
|
});
|
|
1322
1409
|
|
|
1410
|
+
test("CLI: the router row reads the run's .fleet/cost-guard.json, and prints DEFAULT-ONLY without one", () => {
|
|
1411
|
+
const none = runCli([], { shortlist: shortlistText([]) });
|
|
1412
|
+
assert.equal(none.status, 0, none.stderr);
|
|
1413
|
+
assert.deepEqual(lineOf(none, "router"), ["router → DEFAULT-ONLY (cost-guard.json missing — run pr-cost.mjs --guard)"]);
|
|
1414
|
+
const ok = runCli([], {
|
|
1415
|
+
shortlist: shortlistText([]),
|
|
1416
|
+
beforeRun: (repo) => writeFileSync(join(repo, ".fleet", "cost-guard.json"), guardJson({ verdict: "none", baseline: { ...BASE, n: 12 } })),
|
|
1417
|
+
});
|
|
1418
|
+
assert.equal(ok.status, 0, ok.stderr);
|
|
1419
|
+
assert.deepEqual(lineOf(ok, "router"), ["router → NO VERDICT (baseline n=12/20; guard computed 2026-10-03T00:00:00.000Z)"]);
|
|
1420
|
+
});
|
|
1421
|
+
|
|
1323
1422
|
test("CLI: the #1692 shape, read off the ledger — two live implementers at cap 2 pull nothing", () => {
|
|
1324
1423
|
const r = runCli([], {
|
|
1325
1424
|
ledger: { rows: ["#412 impl-412 · tier-ok=impl-412:fleet-implementer", "#415 impl-415 · tier-ok=impl-415:fleet-implementer"], dispatched: ["impl-412", "impl-415"] },
|
|
@@ -77,24 +77,30 @@ export function rowsForSession(sessionDir, stats = {}) {
|
|
|
77
77
|
tokensCacheCreate: r.tokens_cache_create, tokensOut: r.tokens_out,
|
|
78
78
|
wallS: r.wall_s, turns: r.turns, agent: r.agent, torn: false,
|
|
79
79
|
subagentType: r.subagent_type,
|
|
80
|
+
tokensIn: r.tokens_in, tokensCacheRead: r.tokens_cache_read,
|
|
81
|
+
tokensCacheWrite1h: r.tokens_cache_write_1h ?? "", cost: r.cost == null ? "" : Number(r.cost.toFixed(6)),
|
|
80
82
|
}));
|
|
81
83
|
}
|
|
82
84
|
|
|
83
85
|
// APPENDED TO, never inserted into: every read-out in the file's header and
|
|
84
|
-
// in
|
|
85
|
-
// silently repoints every `$n` a reader already wrote down.
|
|
86
|
-
//
|
|
86
|
+
// in the design specs indexes by position, so a column added anywhere but the
|
|
87
|
+
// end silently repoints every `$n` a reader already wrote down.
|
|
88
|
+
// `subagent_type` therefore follows `harness`, and the four priced columns
|
|
89
|
+
// follow it. `cost` is the provider's own per-turn `usage.cost.total`, summed
|
|
90
|
+
// to the micro-dollar — the $ of record; `tokens_cache_write_1h` and `cost`
|
|
91
|
+
// are blank where the transcript recorded no figure, never 0.
|
|
87
92
|
export const COLUMNS = [
|
|
88
93
|
"session", "run_date", "role", "member", "model", "effort", "ticket", "pr",
|
|
89
94
|
"tokens_cache_create", "tokens_out", "wall_s", "turns", "agent", "harness",
|
|
90
|
-
"subagent_type",
|
|
95
|
+
"subagent_type", "tokens_in", "tokens_cache_read", "tokens_cache_write_1h", "cost",
|
|
91
96
|
];
|
|
92
97
|
|
|
93
98
|
// Row objects use camelCase; the file uses snake_case. One map, one direction
|
|
94
99
|
// each, so a rename cannot silently drop a column.
|
|
95
100
|
const FIELD = {
|
|
96
101
|
tokens_cache_create: "tokensCacheCreate", tokens_out: "tokensOut", wall_s: "wallS",
|
|
97
|
-
subagent_type: "subagentType",
|
|
102
|
+
subagent_type: "subagentType", tokens_in: "tokensIn", tokens_cache_read: "tokensCacheRead",
|
|
103
|
+
tokens_cache_write_1h: "tokensCacheWrite1h",
|
|
98
104
|
};
|
|
99
105
|
const field = (c) => FIELD[c] ?? c;
|
|
100
106
|
// Keyed on the transcript's path-relative stem, not the member name: `member`
|
|
@@ -118,13 +124,16 @@ export function formatTsv(rows) {
|
|
|
118
124
|
}
|
|
119
125
|
|
|
120
126
|
// REFUSES a row whose cell count is not exactly COLUMNS.length rather than
|
|
121
|
-
// padding it. Padding looked harmless and was not:
|
|
122
|
-
// so a short row parsed to `agent: ""` — a key
|
|
123
|
-
// produce, which means mergeRows can never REPLACE
|
|
124
|
-
// were measured, all previously exit 0: a
|
|
125
|
-
// permanent phantom that re-scraping could
|
|
126
|
-
// three data rows; and adding one column
|
|
127
|
-
// onto one key per session, 2,702 rows
|
|
127
|
+
// padding it. Padding looked harmless and was not: when this was written
|
|
128
|
+
// `agent` was the LAST column, so a short row parsed to `agent: ""` — a key
|
|
129
|
+
// rowsForSession can never produce, which means mergeRows can never REPLACE
|
|
130
|
+
// it. Three ways to reach that were measured, all previously exit 0: a
|
|
131
|
+
// transcript torn mid-write became a permanent phantom that re-scraping could
|
|
132
|
+
// not heal; git conflict markers became three data rows; and adding one column
|
|
133
|
+
// ahead of `agent` collapsed the corpus onto one key per session, 2,702 rows
|
|
134
|
+
// to 156. `agent` is no longer last — the priced columns follow it — and
|
|
135
|
+
// there a padded "" would read as "no figure was recorded" for a member whose
|
|
136
|
+
// transcript carried one, so a short row stays refused for that too.
|
|
128
137
|
export function parseTsv(text) {
|
|
129
138
|
return String(text ?? "").split("\n")
|
|
130
139
|
.filter((l) => l.trim() && !l.startsWith("#"))
|
|
@@ -137,6 +137,37 @@ test("a generated word pair that merely starts with a family word yields blanks
|
|
|
137
137
|
assert.deepEqual(parseMemberName("Impl1341/Impl1341.CwdProbe"), { ticket: "", pr: "" });
|
|
138
138
|
});
|
|
139
139
|
|
|
140
|
+
test("a review fan-out label books its PR, in the label's own spelling and in the stem omp writes for it", () => {
|
|
141
|
+
// omp names the member by deleting every character outside [A-Za-z0-9_-]
|
|
142
|
+
// from the label and appending -<n> from a label's second dispatch on.
|
|
143
|
+
for (const [label, stem] of [
|
|
144
|
+
["review:correctness:pr2132", "reviewcorrectnesspr2132"],
|
|
145
|
+
["review:silent-failure:pr2132", "reviewsilent-failurepr2132"],
|
|
146
|
+
["verify:comments:pr2132", "verifycommentspr2132"],
|
|
147
|
+
["snapshot:pr2132", "snapshotpr2132"],
|
|
148
|
+
["test-run:pr2132", "test-runpr2132"],
|
|
149
|
+
]) {
|
|
150
|
+
assert.deepEqual(parseMemberName(label), { ticket: "", pr: "2132" }, label);
|
|
151
|
+
assert.deepEqual(parseMemberName(stem), { ticket: "", pr: "2132" }, stem);
|
|
152
|
+
assert.deepEqual(parseMemberName(`${stem}-7`), { ticket: "", pr: "2132" }, `${stem}-7`);
|
|
153
|
+
// Nested under the reviewer that dispatched it: the last segment is the member.
|
|
154
|
+
assert.deepEqual(parseMemberName(`review-pr-2132/${stem}`), { ticket: "", pr: "2132" }, `nested ${stem}`);
|
|
155
|
+
}
|
|
156
|
+
// A dimension key that itself spells `pr` still yields the trailing number.
|
|
157
|
+
assert.deepEqual(parseMemberName("reviewprpr12"), { ticket: "", pr: "12" });
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
test("a fan-out stem that carries no PR, or a name that only resembles one, books nothing", () => {
|
|
161
|
+
// Measured stems from before the label carried its PR.
|
|
162
|
+
for (const name of ["verifycorrectness-26", "reviewtests", "snapshot-8", "snapshot", "test-run",
|
|
163
|
+
// Generated or human names: PascalCase never matches, and a bare number is not `pr<n>`.
|
|
164
|
+
"ReviewCorrectness", "VerifyAndRepair", "Verify685", "ReviewTestsPr12", "review:correctness", "verify::pr", "reviewpr",
|
|
165
|
+
// Look-alikes of a fan-out stem: a prefix before the family word, a key-less `verify`, a hyphen before `pr`.
|
|
166
|
+
"xreviewfoopr12", "xsnapshotpr12", "verifypr12", "snapshot-pr12", "test-run-pr12", "verify-pr12", "reviewers-sprint-pr3"]) {
|
|
167
|
+
assert.deepEqual(parseMemberName(name), { ticket: "", pr: "" }, name);
|
|
168
|
+
}
|
|
169
|
+
});
|
|
170
|
+
|
|
140
171
|
// ---------------------------------------------------------------------------
|
|
141
172
|
// omp fixtures — shaped like real ~/.omp/agent/sessions/**/*.jsonl lines
|
|
142
173
|
// ---------------------------------------------------------------------------
|
|
@@ -271,25 +302,37 @@ test("`subagent_type` is what the DISPATCH named, and blank when it named nothin
|
|
|
271
302
|
});
|
|
272
303
|
|
|
273
304
|
test("a row whose cell count is wrong is REFUSED, never padded", () => {
|
|
274
|
-
// Padding put "" in the
|
|
275
|
-
//
|
|
305
|
+
// Padding put "" in the trailing columns — `agent` was the LAST one when
|
|
306
|
+
// this was written, a key rowsForSession can never produce, so mergeRows
|
|
276
307
|
// could never replace it. Three measured routes there, all exit 0: a torn
|
|
277
308
|
// last line became a permanent phantom that re-scraping could not heal, git
|
|
278
309
|
// conflict markers became three data rows, and adding one column ahead of
|
|
279
|
-
// `agent` collapsed 2,702 rows to 156.
|
|
310
|
+
// `agent` collapsed 2,702 rows to 156. The columns that trail `agent` now
|
|
311
|
+
// are the priced ones, where a padded blank reads as "no figure recorded".
|
|
280
312
|
//
|
|
281
|
-
// The
|
|
282
|
-
//
|
|
283
|
-
//
|
|
284
|
-
//
|
|
313
|
+
// The 15-field spelling is the PREVIOUS schema, from before the four priced
|
|
314
|
+
// columns: it must be refused rather than padded with blank prices, which
|
|
315
|
+
// would read as "no figure was recorded" for members whose transcript
|
|
316
|
+
// carried one. Re-scraping is the migration. The 14-field one before it is
|
|
317
|
+
// refused the same way.
|
|
285
318
|
//
|
|
286
319
|
// Mutation this must survive: restoring `cells[i] ?? ""`.
|
|
287
320
|
const short = ["s1", "2026-08-25", "memory"].join("\t");
|
|
288
|
-
assert.throws(() => parseTsv(short), /malformed row: 3 fields, expected
|
|
321
|
+
assert.throws(() => parseTsv(short), /malformed row: 3 fields, expected 19/);
|
|
289
322
|
assert.throws(() => parseTsv("<<<<<<< HEAD"), /malformed row/);
|
|
290
|
-
|
|
323
|
+
const full = formatTsv([row()]).replace(/\n$/, "").split("\t");
|
|
324
|
+
assert.equal(full.length, 19);
|
|
325
|
+
assert.throws(() => parseTsv(full.slice(0, 15).join("\t")), /15 fields, expected 19/);
|
|
326
|
+
assert.throws(() => parseTsv(full.slice(0, 14).join("\t")), /14 fields, expected 19/);
|
|
291
327
|
// A long row is refused too — that is the schema-drift direction.
|
|
292
|
-
assert.throws(() => parseTsv(
|
|
328
|
+
assert.throws(() => parseTsv(full.join("\t") + "\textra"), /20 fields/);
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
test("the four priced columns follow subagent_type, and a scraped member fills them", () => {
|
|
332
|
+
assert.deepEqual(COLUMNS.slice(COLUMNS.indexOf("subagent_type")),
|
|
333
|
+
["subagent_type", "tokens_in", "tokens_cache_read", "tokens_cache_write_1h", "cost"]);
|
|
334
|
+
const [parsed] = parseTsv(formatTsv([row({ tokensIn: 12, tokensCacheRead: 3400, tokensCacheWrite1h: 900, cost: 0.123457 })]));
|
|
335
|
+
assert.deepEqual([parsed.tokensIn, parsed.tokensCacheRead, parsed.tokensCacheWrite1h, parsed.cost], ["12", "3400", "900", "0.123457"]);
|
|
293
336
|
});
|
|
294
337
|
|
|
295
338
|
const row = (o = {}) => ({
|
|
@@ -300,7 +343,8 @@ const row = (o = {}) => ({
|
|
|
300
343
|
// only `member` still get distinct transcript ids, and fixtures that share
|
|
301
344
|
// the default member (untouched) still key as the SAME agent.
|
|
302
345
|
agent: o.member ?? "impl-580", harness: "omp",
|
|
303
|
-
subagentType: "fleet-implementer",
|
|
346
|
+
subagentType: "fleet-implementer",
|
|
347
|
+
tokensIn: 0, tokensCacheRead: 0, tokensCacheWrite1h: 0, cost: 0.01, ...o,
|
|
304
348
|
});
|
|
305
349
|
|
|
306
350
|
test("re-scraping a session REPLACES its rows rather than appending duplicates", () => {
|
|
@@ -442,6 +486,19 @@ test("two separate sessions scraped in two runs merge into one TSV", () => {
|
|
|
442
486
|
assert.equal(rows.find((r) => r.member === "Solo").subagentType, "fleet-implementer-alt");
|
|
443
487
|
});
|
|
444
488
|
|
|
489
|
+
test("a scrape writes the priced columns off the transcript's usage, the 1h share off its cttl", () => {
|
|
490
|
+
const dir = fixture([["impl-580", [sessionEvt("/x"), thinkingEvt("high"),
|
|
491
|
+
assistantEvt("claude-opus-5", { input: 7, cacheRead: 500, cacheWrite: 300, cttl: { ephemeral1h: 300 }, cost: { total: 0.25 } }),
|
|
492
|
+
assistantEvt("claude-opus-5", { input: 3, cacheRead: 800, cacheWrite: 40, cttl: { ephemeral5m: 40 }, cost: { total: 0.0500004 } }, "2026-08-25T10:05:00.000Z"),
|
|
493
|
+
]]]);
|
|
494
|
+
const out = join(tempDir("mo-out-"), "member-outcomes.tsv");
|
|
495
|
+
const r = spawnSync(process.execPath, [CLI, dir, "--file", out], { encoding: "utf8" });
|
|
496
|
+
assert.equal(r.status, 0, r.stderr);
|
|
497
|
+
const [row] = parseTsv(readFileSync(out, "utf8"));
|
|
498
|
+
assert.deepEqual([row.tokensIn, row.tokensCacheRead, row.tokensCacheCreate, row.tokensCacheWrite1h, row.cost],
|
|
499
|
+
["10", "1300", "340", "300", "0.3"]);
|
|
500
|
+
});
|
|
501
|
+
|
|
445
502
|
test("a run writes rows, and a second run over the same session changes nothing", () => {
|
|
446
503
|
const dir = fixture([["impl-580", [sessionEvt("/x"), thinkingEvt("xhigh"), assistantEvt("claude-opus-5")]]]);
|
|
447
504
|
const out = join(tempDir("mo-out-"), "member-outcomes.tsv");
|
|
@@ -7,11 +7,11 @@
|
|
|
7
7
|
//
|
|
8
8
|
// The record: harness, session, role, agent, model, thinking,
|
|
9
9
|
// subagent_type, tokens_in, tokens_cache_create, tokens_cache_read,
|
|
10
|
-
// tokens_out, cost, wall_s, turns, ticket, pr.
|
|
11
|
-
// reader produced the row — structural,
|
|
12
|
-
// lives under, before any byte is
|
|
13
|
-
//
|
|
14
|
-
// module
|
|
10
|
+
// tokens_cache_write_1h, tokens_out, cost, wall_s, turns, ticket, pr.
|
|
11
|
+
// `harness` is set by whichever reader produced the row — structural,
|
|
12
|
+
// decided by which root the transcript lives under, before any byte is
|
|
13
|
+
// parsed. `cost` is real (`usage.cost.total`, the provider's own per-turn
|
|
14
|
+
// figure) — this module prices nothing itself.
|
|
15
15
|
//
|
|
16
16
|
// `thinking` is the harness-written level, blank when a hole is visible.
|
|
17
17
|
// `subagent_type` is what the member was DISPATCHED AS — the agent DEFINITION the dispatch named, never the
|
|
@@ -137,10 +137,11 @@ export function normalizeModel(raw) {
|
|
|
137
137
|
// pr:""}`, losing its join key into tier-outcomes.tsv exactly the way an
|
|
138
138
|
// unmatched finisher spelling once did.
|
|
139
139
|
//
|
|
140
|
-
// Every pattern below
|
|
141
|
-
//
|
|
142
|
-
//
|
|
143
|
-
//
|
|
140
|
+
// Every pattern below but the review fan-out one (specialistPr, after the
|
|
141
|
+
// function) is written against the lower-kebab spelling, so the name is first
|
|
142
|
+
// rewritten by `canonicalMemberName` (compute-spend.mjs), the same rewrite
|
|
143
|
+
// classifyRole applies: `Impl327` and `FixPr774` book the ticket and PR their
|
|
144
|
+
// kebab forms do instead of blanks.
|
|
144
145
|
export function parseMemberName(name) {
|
|
145
146
|
const s = canonicalMemberName(name).replace(/-(?:[a-z]|v\d+)$/, "");
|
|
146
147
|
let m = /^(?:fix|review|finish(?:er)?|resolve)-pr-(\d+)(?:-\d+)?$/.exec(s);
|
|
@@ -149,7 +150,28 @@ export function parseMemberName(name) {
|
|
|
149
150
|
if (m) return { ticket: "", pr: m[1] };
|
|
150
151
|
m = /^impl-(\d+)$/.exec(s);
|
|
151
152
|
if (m) return { ticket: m[1], pr: "" };
|
|
152
|
-
return { ticket: "", pr:
|
|
153
|
+
return { ticket: "", pr: specialistPr(name) };
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// A review fan-out dispatch is labelled `review:<key>:pr<n>`,
|
|
157
|
+
// `verify:<key>:pr<n>`, `snapshot:pr<n>` or `test-run:pr<n>` (review-core.mjs),
|
|
158
|
+
// and books to PR <n>. The label is not what reaches this function off disk:
|
|
159
|
+
// omp derives the member id from it by deleting every character outside
|
|
160
|
+
// `[A-Za-z0-9_-]` and appending `-<n>` from the second dispatch of one label
|
|
161
|
+
// on (measured: `verify:silent-failure` lands as `verifysilent-failure-6`), so
|
|
162
|
+
// `review:correctness:pr12` is the stem `reviewcorrectnesspr12`, or
|
|
163
|
+
// `reviewcorrectnesspr12-3`. Both spellings are matched by applying that same
|
|
164
|
+
// deletion first. Only the LAST path segment is read — a member nested under
|
|
165
|
+
// a reviewer keeps its own id there — and only lowercase, the case the label
|
|
166
|
+
// templates write, so a generated PascalCase name never books a PR. A stem
|
|
167
|
+
// from before the label carried its PR (`verifycorrectness-26`) has no
|
|
168
|
+
// `pr<n>` and stays blank. The key never ends in a hyphen — a label's `<key>:`
|
|
169
|
+
// loses its colon, never leaving one — so a typed `verify-pr12` or
|
|
170
|
+
// `review-sprint-pr3` is not a fan-out stem and stays blank.
|
|
171
|
+
const SPECIALIST_STEM_RE = /^(?:(?:review|verify)[a-z0-9_-]*?[a-z0-9_]|snapshot|test-run)pr(\d+)(?:-\d+)?$/;
|
|
172
|
+
function specialistPr(name) {
|
|
173
|
+
const stem = String(name ?? "").split(/[/.]/).at(-1).replace(/[^A-Za-z0-9_-]+/g, "");
|
|
174
|
+
return SPECIALIST_STEM_RE.exec(stem)?.[1] ?? "";
|
|
153
175
|
}
|
|
154
176
|
|
|
155
177
|
// ---------------------------------------------------------------------------
|
|
@@ -328,8 +350,8 @@ function createdPrNumbers(text) {
|
|
|
328
350
|
export function foldOmpTranscript(jsonlText, filePath) {
|
|
329
351
|
let model = null, thinking = null, task = null, resolvedModelIdentity = null, agent = null;
|
|
330
352
|
let firstTs = null, lastTs = null;
|
|
331
|
-
let input = 0, cacheWrite = 0, cacheRead = 0, output = 0, cost = 0, turns = 0;
|
|
332
|
-
let sawCost = false;
|
|
353
|
+
let input = 0, cacheWrite = 0, cacheWrite1h = 0, cacheRead = 0, output = 0, cost = 0, turns = 0;
|
|
354
|
+
let sawCost = false, unsplit = false;
|
|
333
355
|
let malformedNonLastLines = 0;
|
|
334
356
|
const entries = [];
|
|
335
357
|
const createCallIds = new Set();
|
|
@@ -369,6 +391,8 @@ export function foldOmpTranscript(jsonlText, filePath) {
|
|
|
369
391
|
cacheWrite += cw;
|
|
370
392
|
cacheRead += Number(u.cacheRead ?? 0);
|
|
371
393
|
output += Number(u.output ?? 0);
|
|
394
|
+
if (u.cttl && typeof u.cttl === "object") cacheWrite1h += Number(u.cttl.ephemeral1h ?? 0);
|
|
395
|
+
else if (cw > 0) unsplit = true;
|
|
372
396
|
if (u.cost && typeof u.cost.total === "number") { cost += u.cost.total; sawCost = true; }
|
|
373
397
|
turns++;
|
|
374
398
|
entries.push({
|
|
@@ -391,6 +415,12 @@ export function foldOmpTranscript(jsonlText, filePath) {
|
|
|
391
415
|
return {
|
|
392
416
|
model, thinking, task, resolvedModelIdentity, agent,
|
|
393
417
|
input, cacheWrite, cacheRead, output,
|
|
418
|
+
// The 1h-TTL share of `cacheWrite`, off each turn's `usage.cttl`
|
|
419
|
+
// (`{ephemeral1h}` or `{ephemeral5m}`, present only on a turn that wrote
|
|
420
|
+
// cache). A real 0 when nothing was written, and unknown — `null` — when
|
|
421
|
+
// any turn that wrote cache recorded no `cttl`: the split was never
|
|
422
|
+
// recorded for that share, and a partial sum would pass for the whole.
|
|
423
|
+
cacheWrite1h: unsplit ? null : cacheWrite1h,
|
|
394
424
|
cost: sawCost ? cost : null,
|
|
395
425
|
turns,
|
|
396
426
|
wallS: Number.isFinite(span) ? Math.round(span) : 0,
|
|
@@ -497,7 +527,7 @@ export function ompMemberRecord(folded, agentStem, spawnDepth = 0) {
|
|
|
497
527
|
// carries no `session_init` line to read one from.
|
|
498
528
|
subagent_type: folded.agent ?? "",
|
|
499
529
|
tokens_in: folded.input, tokens_cache_create: folded.cacheWrite,
|
|
500
|
-
tokens_cache_read: folded.cacheRead, tokens_out: folded.output,
|
|
530
|
+
tokens_cache_read: folded.cacheRead, tokens_cache_write_1h: folded.cacheWrite1h, tokens_out: folded.output,
|
|
501
531
|
cost: folded.cost,
|
|
502
532
|
wall_s: folded.wallS, turns: folded.turns,
|
|
503
533
|
ticket, pr,
|
|
@@ -88,6 +88,30 @@ test("readOmpMember: cost, tokens and thinking come off real usage/thinking_leve
|
|
|
88
88
|
assert.equal(rec.turns, 1);
|
|
89
89
|
});
|
|
90
90
|
|
|
91
|
+
test("readOmpMember: tokens_cache_write_1h sums each turn's usage.cttl.ephemeral1h, and is unknown when cache was written with no TTL split", () => {
|
|
92
|
+
const turn = (cacheWrite, cttl, ts) => assistantEvt("claude-opus-5", { input: 1, output: 1, cacheRead: 0, cacheWrite, ...(cttl ? { cttl } : {}), cost: { total: 0.01 } }, ts);
|
|
93
|
+
const rec = (...turns) => readOmpMember([sessionEvt("/x"), thinkingEvt("high"), ...turns].join("\n"), "/fake/path.jsonl", "impl-1");
|
|
94
|
+
// Measured shapes: `{ephemeral1h:n}` and `{ephemeral5m:n}`, only on a turn that wrote cache.
|
|
95
|
+
const mixed = rec(turn(300, { ephemeral1h: 300 }), turn(0, null, "2026-09-08T15:13:00.000Z"),
|
|
96
|
+
turn(50, { ephemeral5m: 50 }, "2026-09-08T15:14:00.000Z"), turn(20, { ephemeral1h: 20 }, "2026-09-08T15:15:00.000Z"));
|
|
97
|
+
assert.equal(mixed.tokens_cache_create, 370);
|
|
98
|
+
assert.equal(mixed.tokens_cache_write_1h, 320);
|
|
99
|
+
assert.equal(rec(turn(0, null)).tokens_cache_write_1h, 0, "nothing written is a real zero");
|
|
100
|
+
assert.equal(rec(turn(40, null)).tokens_cache_write_1h, null, "cache written with no recorded split is unknown, never zero");
|
|
101
|
+
assert.equal(rec(turn(300, { ephemeral1h: 300 }), turn(700, null, "2026-09-08T15:13:00.000Z")).tokens_cache_write_1h, null,
|
|
102
|
+
"one cache-writing turn with no recorded split makes the whole figure unknown, never the recorded share alone");
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
test("readOmpMember: cost is null, never 0, when no turn carries usage.cost.total", () => {
|
|
106
|
+
const rec = (...turns) => readOmpMember([sessionEvt("/x"), thinkingEvt("high"), ...turns].join("\n"), "/fake/path.jsonl", "impl-1");
|
|
107
|
+
const noCost = assistantEvt("claude-opus-5", { input: 1, output: 1, cacheRead: 0, cacheWrite: 0 });
|
|
108
|
+
const priced = assistantEvt("claude-opus-5", { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, cost: { total: 0.25 } }, "2026-09-08T15:13:00.000Z");
|
|
109
|
+
assert.equal(rec(noCost).cost, null, "no figure recorded is blank, so pr-cost counts it unpriced instead of booking a free member");
|
|
110
|
+
assert.equal(rec(noCost, noCost).cost, null);
|
|
111
|
+
assert.equal(rec(noCost).turns, 1, "the turn itself still counts");
|
|
112
|
+
assert.equal(rec(priced).cost, 0.25);
|
|
113
|
+
});
|
|
114
|
+
|
|
91
115
|
test("foldOmpTranscript: resolvedModelIdentity comes off session_init, present before any assistant turn (#1345)", () => {
|
|
92
116
|
// Measured shape: always provider-prefixed (`anthropic/claude-opus-5`),
|
|
93
117
|
// written at DISPATCH — before the member's first assistant turn, which
|