@feigi/fleet-ctl 3.92.1 → 3.93.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@feigi/fleet-ctl",
3
- "version": "3.92.1",
3
+ "version": "3.93.0",
4
4
  "description": "Agent fleet: run-team controller, merge bot, PR reviewer, and the ticket pipeline they share",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -76,9 +76,9 @@
76
76
  import { readFileSync } from "node:fs";
77
77
  import { makeDie, defineFlags } from "./arg.mjs";
78
78
  import { isCLI } from "./is-cli.mjs";
79
- import { CELL, POLICY_CELL } from "./ledger-grammar.mjs";
79
+ import { POLICY_CELL } from "./ledger-grammar.mjs";
80
80
  import { formatTsv as formatMemberTsv, parseTsv as parseMemberTsv } from "./member-outcomes.mjs";
81
- import { parseFeatures } from "./pr-cost.mjs";
81
+ import { admissibleRow, parseFeatures } from "./pull-attribution.mjs";
82
82
  import { DATE, rulingFor, rulingsByTicket } from "./tier-outcomes.mjs";
83
83
 
84
84
  const NAME = "cell-readout";
@@ -86,7 +86,6 @@ export const GATE = Object.freeze({ comparisons: 10, runDates: 5 });
86
86
  export const STOP = Object.freeze({ verdicts: 10, failRate: 0.8 });
87
87
 
88
88
  const key = (session, agent) => `${session}\0${agent}`;
89
- const levelOf = (cell) => CELL.exec(cell)?.[2] ?? null;
90
89
 
91
90
  /**
92
91
  * `features` keyed by session+agent. A repeat naming the same `chosen_cell`
@@ -106,10 +105,7 @@ function pullsOf(features) {
106
105
  /** The join's member row for a Pull when the Pull is admissible, else null. */
107
106
  function admissibleMember(pull, members) {
108
107
  const m = members.get(key(pull.session, pull.agent));
109
- if (!m) return null;
110
- const level = levelOf(pull.chosen_cell);
111
- if (level === null || m.subagentType !== `fleet-implementer-${pull.chosen_cell}` || m.effort !== level || !m.model) return null;
112
- return m;
108
+ return m && admissibleRow(m, pull.chosen_cell) ? m : null;
113
109
  }
114
110
 
115
111
  // One member row per session+agent. A key repeated with different fields is
@@ -102,7 +102,8 @@ export function rowsForSession(sessionDir, stats = {}, { verifyMain = verifyMain
102
102
  // position-indexed read-outs name columns ahead of them. `subagent_type`
103
103
  // follows `harness`. `cost` is the provider's own per-turn `usage.cost.total`,
104
104
  // summed to the micro-dollar — the $ of record; `tokens_cache_write_1h` and
105
- // `cost` are blank where the transcript recorded no figure, never 0.
105
+ // `cost` are blank where the transcript recorded no figure, never 0, and
106
+ // `cost` is also blank when any turn that reported tokens recorded none.
106
107
  export const COLUMNS = [
107
108
  "session", "run_date", "role", "member", "model", "effort", "ticket", "pr",
108
109
  "tokens_cache_create", "tokens_out", "wall_s", "turns", "agent", "harness",
@@ -140,7 +141,7 @@ export function withReports(rows, scratch) {
140
141
  // crash the session never exited and wrote no yield sidecar;
141
142
  // hollow none of the above: the member stopped on its own.
142
143
  const IMPLEMENTER_LEAF = /^impl-\d+(?:-[a-z0-9]+)?$/;
143
- const isImplementer = (member) => IMPLEMENTER_LEAF.test(member.split("/").pop());
144
+ export const isImplementer = (member) => IMPLEMENTER_LEAF.test(member.split("/").pop());
144
145
 
145
146
  export function endingOf({ member, pr, signals = null, earlierPrs = new Set(), verifyMain = verifyMainWithGit }) {
146
147
  if (!isImplementer(member)) return "";
@@ -359,7 +359,7 @@ export function foldOmpTranscript(jsonlText, filePath) {
359
359
  let model = null, thinking = null, task = null, resolvedModelIdentity = null, agent = null;
360
360
  let firstTs = null, lastTs = null;
361
361
  let input = 0, cacheWrite = 0, cacheWrite1h = 0, cacheRead = 0, output = 0, cost = 0, turns = 0;
362
- let sawCost = false, unsplit = false;
362
+ let sawCost = false, unsplit = false, unpriced = false;
363
363
  let malformedNonLastLines = 0;
364
364
  // What the transcript's END says, read for the `ending` column: the last
365
365
  // assistant turn's stopReason, whether a model fell back, whether the
@@ -406,13 +406,15 @@ export function foldOmpTranscript(jsonlText, filePath) {
406
406
  const u = m.usage;
407
407
  if (typeof m.model === "string" && m.model) model = m.model;
408
408
  const cw = Number(u.cacheWrite ?? 0);
409
- input += Number(u.input ?? 0);
409
+ const tin = Number(u.input ?? 0), tread = Number(u.cacheRead ?? 0), tout = Number(u.output ?? 0);
410
+ input += tin;
410
411
  cacheWrite += cw;
411
- cacheRead += Number(u.cacheRead ?? 0);
412
- output += Number(u.output ?? 0);
412
+ cacheRead += tread;
413
+ output += tout;
413
414
  if (u.cttl && typeof u.cttl === "object") cacheWrite1h += Number(u.cttl.ephemeral1h ?? 0);
414
415
  else if (cw > 0) unsplit = true;
415
416
  if (u.cost && typeof u.cost.total === "number") { cost += u.cost.total; sawCost = true; }
417
+ else if (tin > 0 || cw > 0 || tread > 0 || tout > 0) unpriced = true;
416
418
  turns++;
417
419
  entries.push({
418
420
  kind: "assistant", cacheWrite: cw,
@@ -441,7 +443,10 @@ export function foldOmpTranscript(jsonlText, filePath) {
441
443
  // any turn that wrote cache recorded no `cttl`: the split was never
442
444
  // recorded for that share, and a partial sum would pass for the whole.
443
445
  cacheWrite1h: unsplit ? null : cacheWrite1h,
444
- cost: sawCost ? cost : null,
446
+ // Unknown — `null` — on the same rule: when no turn carried a figure, or
447
+ // when any turn that reported tokens carried none (or a non-number), so
448
+ // a partial sum never passes for the member's whole cost.
449
+ cost: sawCost && !unpriced ? cost : null,
445
450
  turns,
446
451
  wallS: Number.isFinite(span) ? Math.round(span) : 0,
447
452
  entries,
@@ -42,7 +42,7 @@ import { makeDie, defineFlags } from "./arg.mjs";
42
42
  import { gitEnv } from "./git-env.mjs";
43
43
  import { FleetDirUnresolvable, fleetFile } from "./fleet-dir.mjs";
44
44
  import { isCLI } from "./is-cli.mjs";
45
- import { FEATURE_COLUMNS, parseFeatures } from "./pr-cost.mjs";
45
+ import { FEATURE_COLUMNS, parseFeatures } from "./pull-attribution.mjs";
46
46
 
47
47
  const NAME = "metrics-refresh";
48
48
  const PR_COST = fileURLToPath(new URL("./pr-cost.mjs", import.meta.url));
@@ -25,55 +25,42 @@
25
25
  // fleet-tick's `router` row prints it today; a missing or unparseable file
26
26
  // means the default cell only, so deleting it cannot evade the guard.
27
27
  //
28
- // BOOKING. A Pull is a ticket-features.tsv row; its $ is the cost of the
29
- // member-outcomes.tsv row with the same `session` + `agent`, plus every member
30
- // that one dispatched (an `agent` path nested under it). A PR's $ is every
31
- // Pull of its ticket(s) up to the ruling — superseded re-dispatches included —
32
- // plus every member whose NAME carries the PR (`review-pr-<n>`, `fix-pr-<n>`,
33
- // the finisher spellings, the review fan-out labels; member-record.mjs's
34
- // parseMemberName decides), with their own nested members. `merge-bot-*`,
35
- // `memory` and `__advisor` are booked nowhere: their cost does not vary with
36
- // the implementer's cell. Only omp rows are read; a blank `cost` books 0 and
37
- // is counted as unpriced, and a Pull with no member row at all books 0 and is
38
- // listed as unbooked.
28
+ // BOOKING is pull-attribution.mjs's, the one the router's fit reads too: each
29
+ // Pull in the window is a unit booked as routed to its `chosen_cell`, with
30
+ // every implementer under it, its PR's review, fix and finisher members and
31
+ // their nested members, and its merge through its Ruling — merged when that
32
+ // PR's state is MERGED. A Pull is counted, or excluded (no Admissible row),
33
+ // pending (its PR still open, or not in the list), left out (never merged, no
34
+ // `fixed` bail, and every ending the ticket's or neither's) or unpriced (a
35
+ // booked row with a blank `cost`, or a Ruling whose PR none of its
36
+ // implementers carries); only counted Pulls enter a cell's figures. A `cost`
37
+ // that is neither blank nor a number is an input error (exit 2), as is a
38
+ // ticket-features `run_date` that is neither blank nor YYYY-MM-DD, or a ruling
39
+ // of a windowed Pull's ticket whose run_date is not YYYY-MM-DD or whose verdict
40
+ // is not yes or no.
39
41
  //
40
- // A ticket's RULING is its last tier-outcomes.tsv row in file order dated on or
41
- // after its first Pull in the window, skipping rows with both verdict columns
42
- // blank, which were never ruled; that row names the PR and carries the quality
43
- // verdict. A PR ruling several tickets takes its verdict from the later of
44
- // their rulings in file order, whatever their dates. A ruling whose run_date is
45
- // not YYYY-MM-DD is an input error (exit 2), as is one whose verdict is not
46
- // yes or no. The PR's CELL is the `chosen_cell` of the last Pull booked to it
47
- // — tickets in the order of their first Pull, each ticket's Pulls in file
48
- // order, those dated on or before the ticket's ruling — cross-checked against
49
- // that Pull's member row: a `subagent_type` other than
50
- // `fleet-implementer-<cell>`, or an `effort` other than the cell's level,
51
- // lists the PR under `mismatch` and books it to no cell at all. A PR still
52
- // OPEN is pending and booked nowhere yet. A Pull with no ruling books to its
53
- // own `chosen_cell` as unmerged spend — unless its member opened a PR that is
54
- // still open, which is pending too.
55
- //
56
- // PER CELL: `n_pulls`; `n_merged`; `n_pass` (merged PRs passing the quality
57
- // floor — failure is `minted_false_claim=yes` or `closed_own_ticket=no`);
58
- // `fail_rate` = failed / merged; `mean_usd` = every $ booked to the cell —
59
- // merged, unmerged and superseded — / merged, so a cell that abandons work
60
- // carries that spend; `median_usd` over merged PRs alone, descriptive;
61
- // `router_usd`, the Pulls' own routing cost, summed. `usd_diff_ci95` is a
62
- // bootstrap 95% interval on `mean_usd` minus the baseline's — printed to be
63
- // read, never judged on.
42
+ // PER CELL, over its counted Pulls: `n_pulls`; `n_merged`; `n_claims` (merged
43
+ // PRs plus every `bail:fixed` or `bail:fixed-refuted` ending); `n_failed`
44
+ // (merged PRs ruled `minted_false_claim=yes` or `closed_own_ticket=no`, plus
45
+ // every `bail:fixed-refuted`); `fail_rate` = failed / claims; `mean_usd` = the
46
+ // Pulls' spend — merged or not — / merged, so a cell that abandons work carries
47
+ // that spend; `median_usd` over the merged Pulls alone, descriptive;
48
+ // `router_usd`, the Pulls' own routing cost, summed and never in their spend.
49
+ // `usd_diff_ci95` is a bootstrap 95% interval on `mean_usd` minus the
50
+ // baseline's — printed to be read, never judged on. Then the cell's
51
+ // `excluded`, `left_out` and `unpriced` Pull counts.
64
52
 
65
53
  import { execFileSync } from "node:child_process";
66
54
  import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
67
55
  import { dirname, join } from "node:path";
68
56
  import { fileURLToPath } from "node:url";
69
- import { makeDie, defineFlags, isDigits } from "./arg.mjs";
57
+ import { makeDie, defineFlags } from "./arg.mjs";
70
58
  import { gitEnv } from "./git-env.mjs";
71
59
  import { fleetFile, FleetDirUnresolvable } from "./fleet-dir.mjs";
72
60
  import { isCLI } from "./is-cli.mjs";
73
- import { CELL } from "./ledger-grammar.mjs";
74
- import { parseMemberName } from "./member-record.mjs";
75
61
  import { parseTsv as parseMemberTsv } from "./member-outcomes.mjs";
76
- import { RulingError, parseTierOutcomes, rulingFor, rulingsByTicket } from "./tier-outcomes.mjs";
62
+ import { AttributionError, attribute, parseFeatures, tally } from "./pull-attribution.mjs";
63
+ import { RulingError, parseTierOutcomes } from "./tier-outcomes.mjs";
77
64
 
78
65
  const NAME = "pr-cost";
79
66
  const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
@@ -91,38 +78,7 @@ export const TRIP_POINTS = 15;
91
78
  export const EXIT = { ok: 0, tripped: 3, none: 4 };
92
79
  export const GH_PR_LIMIT = 1000;
93
80
 
94
- export const FEATURE_COLUMNS = [
95
- "run_date", "session", "agent", "ticket", "policy_cell", "chosen_cell", "exploration_draw",
96
- "sizing_src", "sizing_pre", "router_usd", "brief_chars", "criteria", "comments", "age_days",
97
- "paths", "test_paths", "xrefs", "kind",
98
- ];
99
-
100
- // ticket-features.tsv: one bare header line naming the columns, then rows.
101
- // Read by header name; a row whose width differs from the header's is refused.
102
- export function parseFeatures(text) {
103
- const lines = String(text ?? "").split("\n").filter((l) => l.trim() && !l.startsWith("#"));
104
- if (lines.length === 0) throw new Error("ticket-features.tsv has no header line");
105
- const header = lines[0].split("\t");
106
- const missing = FEATURE_COLUMNS.filter((c) => !header.includes(c));
107
- if (missing.length) throw new Error(`ticket-features.tsv header lacks ${missing.join(", ")}`);
108
- return lines.slice(1).map((l, i) => {
109
- const cells = l.split("\t");
110
- if (cells.length !== header.length) {
111
- throw new Error(`ticket-features.tsv row ${i + 1}: ${cells.length} fields, expected ${header.length} — ${l.slice(0, 60)}`);
112
- }
113
- const row = Object.fromEntries(header.map((c, j) => [c, cells[j]]));
114
- if (!isDigits(row.ticket)) throw new Error(`ticket-features.tsv row ${i + 1}: ticket ${JSON.stringify(row.ticket)} is not a number`);
115
- if (!CELL.test(row.chosen_cell)) throw new Error(`ticket-features.tsv row ${i + 1}: chosen_cell ${JSON.stringify(row.chosen_cell)} is not a cell`);
116
- return row;
117
- });
118
- }
119
-
120
- export const EXCLUDED_MEMBER = /^(?:merge-?bot|memory$|__advisor$)/i;
121
- // omp nests a member's id as `<parent>/<parent>.<child>`: the child is the last segment after either separator.
122
- export const lastSegment = (agent) => String(agent).split(/[/.]/).at(-1);
123
- const usd = (r) => (r.cost === "" || r.cost === undefined ? 0 : Number(r.cost));
124
- const isMerged = (pr) => pr?.state === "MERGED";
125
- const failed = (tier) => tier.minted_false_claim === "yes" || tier.closed_own_ticket === "no";
81
+ const routerUsd = (pull) => (pull.router_usd === "" ? 0 : Number(pull.router_usd));
126
82
  const round = (x, d = 4) => (x === null || !Number.isFinite(x) ? null : Number(x.toFixed(d)));
127
83
 
128
84
  function median(xs) {
@@ -167,7 +123,8 @@ export function bootstrapDiffCI(cellUnits, baseUnits, { iterations = 2000, seed
167
123
  }
168
124
 
169
125
  // The trip rule, in integer counts so a 3-of-20 difference is exactly 15
170
- // points rather than 14.999… in floating point.
126
+ // points rather than 14.999… in floating point. `n` is a cell's claims and
127
+ // `n_pass` the claims among them that held.
171
128
  export function trips(cell, base) {
172
129
  const failC = cell.n - cell.n_pass, failB = base.n - base.n_pass;
173
130
  const failRise = (failC * base.n - failB * cell.n) * 100 >= TRIP_POINTS * cell.n * base.n;
@@ -175,142 +132,46 @@ export function trips(cell, base) {
175
132
  }
176
133
 
177
134
  export function computeReport({ members, tiers, features, prs, routerTable = null, pricing = null, windowStart = WINDOW_START }) {
178
- const omp = members.filter((r) => r.harness === "omp");
179
- const memberKey = (session, agent) => `${session}\0${agent}`;
180
- const prState = new Map(prs.map((p) => [String(p.number), p]));
181
-
182
- // Direct bookings, keyed by the member row they name.
183
- const pulls = features.filter((f) => f.run_date >= windowStart);
184
- const pullByKey = new Map(pulls.map((p) => [memberKey(p.session, p.agent), p]));
185
- const prMemberKeys = new Map();
186
- for (const r of omp) {
187
- if (EXCLUDED_MEMBER.test(lastSegment(r.member))) continue;
188
- if (pullByKey.has(memberKey(r.session, r.agent))) continue;
189
- const { pr } = parseMemberName(r.member);
190
- if (pr) prMemberKeys.set(memberKey(r.session, r.agent), pr);
191
- }
192
- // Every omp row books to its nearest booked ancestor (itself included).
193
- const pullUsd = new Map(), prMemberUsd = new Map();
194
- const memberOfPull = new Map();
195
- const booked = []; // [row, pull | null, pr | null]
196
- for (const r of omp) {
197
- if (EXCLUDED_MEMBER.test(lastSegment(r.member))) continue;
198
- const segs = String(r.agent).split("/");
199
- for (let i = segs.length; i > 0; i--) {
200
- const k = memberKey(r.session, segs.slice(0, i).join("/"));
201
- const pull = pullByKey.get(k);
202
- const pr = prMemberKeys.get(k);
203
- if (!pull && !pr) continue;
204
- booked.push([r, pull ?? null, pull ? null : pr]);
205
- if (pull) {
206
- pullUsd.set(pull, (pullUsd.get(pull) ?? 0) + usd(r));
207
- if (i === segs.length) memberOfPull.set(pull, r);
208
- } else prMemberUsd.set(pr, (prMemberUsd.get(pr) ?? 0) + usd(r));
209
- break;
210
- }
211
- }
212
-
213
- // Rulings: per ticket, `rulingFor` of the ticket's first Pull — the last
214
- // ruling row in file order dated on or after it, both-blank rows skipped.
215
- const rulings = rulingsByTicket(tiers);
216
- const fileOrder = new Map(tiers.map((t, i) => [t, i]));
217
- const pullsByTicket = new Map();
218
- for (const p of pulls) {
219
- if (!pullsByTicket.has(p.ticket)) pullsByTicket.set(p.ticket, []);
220
- pullsByTicket.get(p.ticket).push(p);
221
- }
222
- const groups = new Map(); // pr -> { pr, tier, pulls[] }
223
- const unruled = [];
224
- for (const [ticket, ps] of pullsByTicket) {
225
- const ruling = rulingFor(rulings, ticket, ps[0].run_date);
226
- if (!ruling) { unruled.push(...ps); continue; }
227
- const g = groups.get(ruling.pr) ?? { pr: ruling.pr, tier: ruling, pulls: [] };
228
- if (fileOrder.get(ruling) > fileOrder.get(g.tier)) g.tier = ruling;
229
- for (const p of ps) (p.run_date <= ruling.run_date ? g.pulls : unruled).push(p);
230
- groups.set(ruling.pr, g);
231
- }
232
-
233
- const cells = new Map();
234
- const cellOf = (c) => {
235
- if (!cells.has(c)) {
236
- cells.set(c, { cell: c, n_pulls: 0, n_merged: 0, n_pass: 0, usd: 0, router_usd: 0, merged_usd: [], units: [] });
237
- }
238
- return cells.get(c);
239
- };
240
- const routerUsd = (p) => (p.router_usd === "" ? 0 : Number(p.router_usd));
241
- const mismatch = [], pending = [];
242
-
243
- for (const g of groups.values()) {
244
- if (g.pulls.length === 0) continue;
245
- const carrier = g.pulls.at(-1);
246
- const cell = carrier.chosen_cell;
247
- const m = memberOfPull.get(carrier);
248
- const level = cell.split("-").slice(1).join("-");
249
- if (m && (m.subagentType !== `fleet-implementer-${cell}` || m.effort !== level)) {
250
- mismatch.push({ pr: g.pr, cell, subagent_type: m.subagentType, effort: m.effort });
251
- continue;
252
- }
253
- const state = prState.get(g.pr);
254
- if (!state || state.state === "OPEN") { pending.push(g.pr); continue; }
255
- const total = g.pulls.reduce((s, p) => s + (pullUsd.get(p) ?? 0), 0) + (prMemberUsd.get(g.pr) ?? 0);
256
- const c = cellOf(cell);
257
- for (const p of g.pulls) { cellOf(p.chosen_cell).n_pulls++; c.router_usd += routerUsd(p); }
258
- c.usd += total;
259
- const merged = isMerged(state) ? 1 : 0;
260
- c.units.push({ usd: total, merged });
261
- if (merged) {
262
- c.n_merged++;
263
- c.merged_usd.push(total);
264
- if (!failed(g.tier)) c.n_pass++;
265
- }
266
- }
267
- for (const p of unruled) {
268
- const opened = memberOfPull.get(p)?.pr;
269
- if (opened && prState.get(opened)?.state === "OPEN") { pending.push(opened); continue; }
270
- const c = cellOf(p.chosen_cell);
271
- c.n_pulls++;
272
- c.router_usd += routerUsd(p);
273
- const spent = pullUsd.get(p) ?? 0;
274
- c.usd += spent;
275
- c.units.push({ usd: spent, merged: 0 });
276
- }
277
-
278
- const base = cells.get(BASELINE_CELL) ?? null;
279
- const meanUsd = (c) => (c.n_merged ? c.usd / c.n_merged : null);
280
- const figures = [...cells.values()].sort((a, b) => a.cell.localeCompare(b.cell)).map((c) => ({
281
- cell: c.cell,
282
- n_pulls: c.n_pulls,
283
- n_merged: c.n_merged,
284
- n_pass: c.n_pass,
285
- fail_rate: c.n_merged ? round((c.n_merged - c.n_pass) / c.n_merged) : null,
286
- mean_usd: round(meanUsd(c), 2),
287
- median_usd: round(median(c.merged_usd), 2),
288
- router_usd: round(c.router_usd, 4),
289
- usd_diff_ci95: c.cell === BASELINE_CELL || !base ? null : bootstrapDiffCI(c.units, base.units),
135
+ const units = attribute({ features, members, tiers, prs, windowStart });
136
+ const counted = units.filter((u) => u.status === "counted");
137
+ const tallies = new Map(tally(units).map((t) => [t.cell, t]));
138
+ const countedIn = (cell) => counted.filter((u) => u.cell === cell);
139
+ const bootUnits = (cell) => countedIn(cell).map((u) => ({ usd: u.spend, merged: u.merged ? 1 : 0 }));
140
+ const meanUsd = (t) => (t.merged ? t.spend / t.merged : null);
141
+ const base = tallies.get(BASELINE_CELL) ?? null;
142
+ const figures = [...tallies.values()].map((t) => ({
143
+ cell: t.cell,
144
+ n_pulls: t.n,
145
+ n_merged: t.merged,
146
+ n_claims: t.claims,
147
+ n_failed: t.failed,
148
+ fail_rate: t.claims ? round(t.failed / t.claims) : null,
149
+ mean_usd: round(meanUsd(t), 2),
150
+ median_usd: round(median(countedIn(t.cell).filter((u) => u.merged).map((u) => u.spend)), 2),
151
+ router_usd: round(countedIn(t.cell).reduce((s, u) => s + routerUsd(u.pull), 0), 4),
152
+ usd_diff_ci95: t.cell === BASELINE_CELL || !base ? null : bootstrapDiffCI(bootUnits(t.cell), bootUnits(BASELINE_CELL)),
153
+ excluded: t.excluded,
154
+ left_out: t.left_out,
155
+ unpriced: t.unpriced,
290
156
  }));
291
157
 
292
158
  // The guard.
293
- const byCell = new Map(figures.map((f) => [f.cell, f]));
294
- const b = byCell.get(BASELINE_CELL) ?? { n_merged: 0, n_pass: 0, mean_usd: null, fail_rate: null };
159
+ const b = figures.find((f) => f.cell === BASELINE_CELL) ?? { n_merged: 0, mean_usd: null, fail_rate: null };
295
160
  const baseline = { cell: BASELINE_CELL, n: b.n_merged, mean_usd: b.mean_usd, fail_rate: b.fail_rate };
161
+ // The $ leg takes the unrounded means: the cent-rounded figures are for
162
+ // display, and a trip is sticky, so a cell cheaper by under half a cent
163
+ // must not trip on rounding alone. The fail leg reads the claims.
164
+ const quality = (t) => ({ n: t.claims, n_pass: t.claims - t.failed, mean_usd: meanUsd(t) });
296
165
  const tripped = [];
297
166
  if (baseline.n >= MIN_N) {
298
- for (const f of figures) {
299
- if (f.cell === BASELINE_CELL || f.n_merged < MIN_N) continue;
300
- // The $ leg takes the unrounded means: the cent-rounded figures are for
301
- // display, and a trip is sticky, so a cell cheaper by under half a cent
302
- // must not trip on rounding alone.
303
- const c = cells.get(f.cell);
304
- if (trips({ n: f.n_merged, n_pass: f.n_pass, mean_usd: meanUsd(c) }, { n: b.n_merged, n_pass: b.n_pass, mean_usd: meanUsd(base) })) tripped.push(f.cell);
167
+ for (const t of tallies.values()) {
168
+ if (t.cell !== BASELINE_CELL && t.merged >= MIN_N && trips(quality(t), quality(base))) tripped.push(t.cell);
305
169
  }
306
170
  }
307
171
  const verdict = baseline.n < MIN_N ? "none" : tripped.length ? "tripped" : "ok";
308
172
  // Retire: every non-default stage-1 cell tripped before any stratum adopted one.
309
173
  const adopted = Object.values(routerTable?.rows ?? {}).some((c) => c !== BASELINE_CELL);
310
174
  const retire = !adopted && STAGE_1.filter((c) => c !== BASELINE_CELL).every((c) => tripped.includes(c));
311
- // The member rows this report books: every windowed Pull's, and those of
312
- // PR-named members whose PR a windowed ticket was ruled on.
313
- const inReport = booked.filter(([, pull, pr]) => pull || groups.has(pr)).map(([r]) => r);
314
175
 
315
176
  return {
316
177
  window_start: windowStart,
@@ -319,44 +180,40 @@ export function computeReport({ members, tiers, features, prs, routerTable = nul
319
180
  tripped,
320
181
  verdict,
321
182
  retire,
322
- mismatch,
323
- pending: [...new Set(pending)].sort((x, y) => Number(x) - Number(y)),
324
- unpriced: inReport.filter((r) => r.cost === "").length,
325
- // A Pull whose member row never reached member-outcomes.tsv books $0 into
326
- // its cell's mean, which reads as a cheap run unless it is listed here.
327
- unbooked_pulls: pulls.filter((p) => !pullUsd.has(p)).map((p) => p.agent),
328
- ab: abReport(groups, pulls, prState, pullUsd, prMemberUsd),
329
- cross_check: crossCheck(inReport, pricing),
183
+ pending: [...new Set(units.filter((u) => u.status === "pending").map((u) => u.pendingPr))].sort((x, y) => Number(x) - Number(y)),
184
+ ab: abReport(units),
185
+ cross_check: crossCheck(units.flatMap((u) => u.rows), pricing),
330
186
  };
331
187
  }
332
188
 
333
189
  // The reasoning test: arm A on an even ticket, B on an odd one. B has run only
334
- // if some odd-ticket Pull carries a `sizing_pre`.
335
- export function abReport(groups, pulls, prState, pullUsd, prMemberUsd) {
190
+ // if some odd-ticket Pull carries a `sizing_pre`. An arm's $ is its counted
191
+ // Pulls' spend plus their own `router_usd`, since the arms compare policies,
192
+ // not cells.
193
+ export function abReport(units) {
336
194
  const arm = (ticket) => (Number(ticket) % 2 === 0 ? "A" : "B");
337
- const bPulls = pulls.filter((p) => arm(p.ticket) === "B");
195
+ const bPulls = units.map((u) => u.pull).filter((p) => arm(p.ticket) === "B");
338
196
  if (!bPulls.some((p) => p.sizing_pre !== "")) {
339
197
  return { status: "not-run", reason: "B not run — the table has no row a better classifier could change" };
340
198
  }
341
- const arms = { A: { merged: 0, failed: 0, usd: 0 }, B: { merged: 0, failed: 0, usd: 0 } };
342
- for (const g of groups.values()) {
343
- if (!g.pulls.length || !isMerged(prState.get(g.pr))) continue;
344
- const a = arms[arm(g.pulls.at(-1).ticket)];
345
- a.merged++;
346
- if (failed(g.tier)) a.failed++;
347
- a.usd += g.pulls.reduce((s, p) => s + (pullUsd.get(p) ?? 0) + (p.router_usd === "" ? 0 : Number(p.router_usd)), 0)
348
- + (prMemberUsd.get(g.pr) ?? 0);
199
+ const arms = { A: { merged: 0, claims: 0, failed: 0, usd: 0 }, B: { merged: 0, claims: 0, failed: 0, usd: 0 } };
200
+ for (const u of units) {
201
+ if (u.status !== "counted") continue;
202
+ const a = arms[arm(u.pull.ticket)];
203
+ if (u.merged) a.merged++;
204
+ a.claims += u.claims;
205
+ a.failed += u.falseClaims;
206
+ a.usd += u.spend + routerUsd(u.pull);
349
207
  }
350
208
  const disagree = bPulls.filter((p) => p.policy_cell !== p.chosen_cell).length;
351
- const out = {
352
- A: { n_merged: arms.A.merged, fail_rate: arms.A.merged ? round(arms.A.failed / arms.A.merged) : null, mean_usd: arms.A.merged ? round(arms.A.usd / arms.A.merged, 2) : null },
353
- B: { n_merged: arms.B.merged, fail_rate: arms.B.merged ? round(arms.B.failed / arms.B.merged) : null, mean_usd: arms.B.merged ? round(arms.B.usd / arms.B.merged, 2) : null },
354
- disagreement: bPulls.length ? round(disagree / bPulls.length) : null,
355
- };
209
+ const side = (a) => ({
210
+ n_merged: a.merged, fail_rate: a.claims ? round(a.failed / a.claims) : null, mean_usd: a.merged ? round(a.usd / a.merged, 2) : null,
211
+ });
212
+ const out = { A: side(arms.A), B: side(arms.B), disagreement: bPulls.length ? round(disagree / bPulls.length) : null };
356
213
  // B can only show an effect through the Pulls where it chose differently.
357
214
  if (arms.A.merged < MIN_N || arms.B.merged < MIN_N) return { status: "insufficient", ...out };
358
215
  if (disagree < MIN_N) return { status: "underpowered", ...out };
359
- const bWins = out.B.mean_usd < out.A.mean_usd && arms.B.failed * arms.A.merged <= arms.A.failed * arms.B.merged;
216
+ const bWins = out.B.mean_usd < out.A.mean_usd && arms.B.failed * arms.A.claims <= arms.A.failed * arms.B.claims;
360
217
  return { status: bWins ? "B-wins" : "B-off", ...out };
361
218
  }
362
219
 
@@ -381,7 +238,10 @@ export function crossCheck(rows, pricing) {
381
238
  return { rows: n, skipped, ratio: expected > 0 ? round(recorded / expected) : null };
382
239
  }
383
240
 
384
- const TSV_COLUMNS = ["cell", "n_pulls", "n_merged", "n_pass", "fail_rate", "mean_usd", "median_usd", "router_usd", "usd_diff_ci95", "guard"];
241
+ const TSV_COLUMNS = [
242
+ "cell", "n_pulls", "n_merged", "n_claims", "n_failed", "fail_rate", "mean_usd", "median_usd", "router_usd", "usd_diff_ci95",
243
+ "excluded", "left_out", "unpriced", "guard",
244
+ ];
385
245
 
386
246
  export function formatReport(report) {
387
247
  const guardOf = (f) => {
@@ -394,13 +254,7 @@ export function formatReport(report) {
394
254
  for (const f of report.cells) lines.push(TSV_COLUMNS.map((c) => (c === "guard" ? guardOf(f) : cell(f[c]))).join("\t"));
395
255
  const b = report.baseline;
396
256
  lines.push(`# verdict: ${report.verdict} (baseline ${b.cell} n=${b.n}/${MIN_N}${report.tripped.length ? `; tripped ${report.tripped.join(" ")}` : ""})${report.retire ? " — retire: every non-default stage-1 cell tripped" : ""}`);
397
- lines.push(`# window_start ${report.window_start}; pending PRs: ${report.pending.length ? report.pending.join(" ") : "none"}; unpriced member rows: ${report.unpriced}`);
398
- if (report.mismatch.length) {
399
- lines.push(`# mismatch (excluded from every cell): ${report.mismatch.map((m) => `PR#${m.pr} ${m.cell} vs ${m.subagent_type || "(none)"}/${m.effort || "(none)"}`).join("; ")}`);
400
- }
401
- if (report.unbooked_pulls.length) {
402
- lines.push(`# unbooked Pulls (no member row, so $0 in the mean): ${report.unbooked_pulls.join(" ")}`);
403
- }
257
+ lines.push(`# window_start ${report.window_start}; pending PRs: ${report.pending.length ? report.pending.join(" ") : "none"}`);
404
258
  const ab = report.ab;
405
259
  lines.push(ab.status === "not-run"
406
260
  ? `# A/B: ${ab.reason}`
@@ -493,7 +347,11 @@ function main() {
493
347
 
494
348
  let report;
495
349
  try { report = computeReport({ members, tiers, features, prs, routerTable, pricing }); }
496
- catch (e) { if (e instanceof RulingError) die(`${tiersPath}: ${e.message}`); throw e; }
350
+ catch (e) {
351
+ if (e instanceof RulingError) die(`${tiersPath}: ${e.message}`);
352
+ if (e instanceof AttributionError) die(e.message);
353
+ throw e;
354
+ }
497
355
  process.stdout.write(has("json") ? JSON.stringify(report, null, 2) + "\n" : formatReport(report));
498
356
  if (!has("guard")) return;
499
357
  const file = arg("out") ?? defaultGuardPath();
@@ -0,0 +1,260 @@
1
+ // The one booking of member rows, merges and claims to Pulls, read by both the
2
+ // cost guard (pr-cost.mjs) and the router's fit (ticket-router.mjs), so the two
3
+ // print the same figures for a cell. Neither CLI imports the other; both import
4
+ // this.
5
+ //
6
+ // THE UNIT is the Pull, a ticket-features.tsv row, booked AS ROUTED to its
7
+ // `chosen_cell`: its merge, its Ruling and all its spend. A ticket Pulled twice
8
+ // is two units. Nothing splits across cells.
9
+ //
10
+ // JOINS. Only omp member rows are read, and a member whose name's last segment
11
+ // starts with merge-bot (or mergebot), or is `memory` or `__advisor`, is booked
12
+ // nowhere: its cost does not vary with the cell.
13
+ // - An implementer row (member-outcomes' implementer leaf) joins the Pull of
14
+ // its `ticket` in its own session. A session with no Pull of that ticket
15
+ // (a claim resumed in a later run) joins the ticket's latest earlier Pull.
16
+ // - A review, fix or finisher row — any row whose name carries a PR — joins
17
+ // the Pull whose implementer rows carry that PR in `pr`.
18
+ // - A nested member books with its nearest booked ancestor.
19
+ //
20
+ // RULING. A Pull's candidate Ruling is `rulingFor` of its ticket and date. One
21
+ // PR's merge counts once: among the Pulls whose candidate names it, the Pull
22
+ // whose implementer rows carry the PR takes it, with the verdict of the latest
23
+ // such ruling row in file order. When none carries it, the last of them in
24
+ // file order takes it and is unpriced, since its PR's review and fix spend
25
+ // cannot be booked. The other candidates stay unruled.
26
+ //
27
+ // MERGED: the Pull holds a Ruling and, when PR states are given, its PR is
28
+ // MERGED. With PR states, a ruled PR that is OPEN or not in the list, or an
29
+ // unruled Pull whose implementer opened a PR still OPEN, is pending.
30
+ //
31
+ // STATUS, first that applies:
32
+ // excluded no Admissible row among the Pull's implementer rows;
33
+ // pending see above (only with PR states);
34
+ // left-out holds no claim (never merged, no `fixed` bail), and every
35
+ // implementer ending is the ticket's or neither's;
36
+ // unpriced a booked row's `cost` is blank, or the Pull took its Ruling
37
+ // without carrying its PR;
38
+ // counted every other Pull: the only ones in a cell's figures.
39
+ //
40
+ // ENDINGS come from member-outcomes' `ending` column. The cell's: `hollow`,
41
+ // `budget`, `bail:fixed-refuted`, `unknown`. The ticket's: every other
42
+ // `bail:<cause>`. Neither's: `provider`, `crash`, `abort`.
43
+ //
44
+ // CLAIMS. A merged Pull is one claim, false when its Ruling fails the floor
45
+ // (`minted_false_claim=yes` or `closed_own_ticket=no`). Each `bail:fixed` or
46
+ // `bail:fixed-refuted` ending is one claim, false when refuted. A cell's fail
47
+ // rate is its counted Pulls' false claims over their claims.
48
+ //
49
+ // SPEND is the recorded `cost` of every row booked to the Pull. A `cost` that
50
+ // is neither blank nor a number is refused, naming the row.
51
+
52
+ import { isDigits } from "./arg.mjs";
53
+ import { CELL } from "./ledger-grammar.mjs";
54
+ import { parseMemberName } from "./member-record.mjs";
55
+ import { isImplementer } from "./member-outcomes.mjs";
56
+ import { DATE, rulingFor, rulingsByTicket } from "./tier-outcomes.mjs";
57
+
58
+ // The ticket-features row, one per Pull, in this order.
59
+ export const FEATURE_COLUMNS = Object.freeze([
60
+ "run_date", "session", "agent", "ticket", "policy_cell", "chosen_cell", "exploration_draw",
61
+ "sizing_src", "sizing_pre", "router_usd", "brief_chars", "criteria", "comments", "age_days",
62
+ "paths", "test_paths", "xrefs", "kind",
63
+ ]);
64
+
65
+ // ticket-features.tsv: one bare header line naming the columns, then rows.
66
+ // Read by header name; a row whose width differs from the header's is refused.
67
+ export function parseFeatures(text) {
68
+ const lines = String(text ?? "").split("\n").filter((l) => l.trim() && !l.startsWith("#"));
69
+ if (lines.length === 0) throw new Error("ticket-features.tsv has no header line");
70
+ const header = lines[0].split("\t");
71
+ const missing = FEATURE_COLUMNS.filter((c) => !header.includes(c));
72
+ if (missing.length) throw new Error(`ticket-features.tsv header lacks ${missing.join(", ")}`);
73
+ return lines.slice(1).map((l, i) => {
74
+ const cells = l.split("\t");
75
+ if (cells.length !== header.length) {
76
+ throw new Error(`ticket-features.tsv row ${i + 1}: ${cells.length} fields, expected ${header.length} — ${l.slice(0, 60)}`);
77
+ }
78
+ const row = Object.fromEntries(header.map((c, j) => [c, cells[j]]));
79
+ if (!isDigits(row.ticket)) throw new Error(`ticket-features.tsv row ${i + 1}: ticket ${JSON.stringify(row.ticket)} is not a number`);
80
+ if (!CELL.test(row.chosen_cell)) throw new Error(`ticket-features.tsv row ${i + 1}: chosen_cell ${JSON.stringify(row.chosen_cell)} is not a cell`);
81
+ return row;
82
+ });
83
+ }
84
+
85
+ export const EXCLUDED_MEMBER = /^(?:merge-?bot|memory$|__advisor$)/i;
86
+ // omp nests a member's id as `<parent>/<parent>.<child>`: the child is the last segment after either separator.
87
+ export const lastSegment = (agent) => String(agent).split(/[/.]/).at(-1);
88
+
89
+ // An Admissible row for `cell`: dispatched as the cell's own definition, run at
90
+ // the cell's level, with a resolved model on record.
91
+ export function admissibleRow(row, cell) {
92
+ const level = CELL.exec(cell)?.[2] ?? null;
93
+ return level !== null && row.subagentType === `fleet-implementer-${cell}` && row.effort === level && !!row.model;
94
+ }
95
+
96
+ // A bad input row, as opposed to a bug: a caller that reports bad input catches this class alone.
97
+ export class AttributionError extends Error {}
98
+
99
+ /**
100
+ * The features rows with a YYYY-MM-DD `run_date`, the rows that can be placed
101
+ * against the window and a ruling. A blank `run_date` (route writes one for a
102
+ * session id with no date) is no Pull and is left out. Any other date is a
103
+ * hand-edit that cannot be placed and is refused: dropping it would shrink
104
+ * every figure unseen.
105
+ */
106
+ function datedRows(features) {
107
+ return features.filter((r) => {
108
+ if (DATE.test(r.run_date)) return true;
109
+ if (r.run_date !== "") throw new AttributionError(`ticket #${r.ticket}: a ticket-features row's run_date is '${r.run_date}', expected YYYY-MM-DD or blank`);
110
+ return false;
111
+ });
112
+ }
113
+
114
+ const CELL_ENDINGS = new Set(["hollow", "budget", "bail:fixed-refuted", "unknown"]);
115
+ const NEITHER_ENDINGS = new Set(["provider", "crash", "abort"]);
116
+ const ticketsEnding = (e) => e.startsWith("bail:") && !CELL_ENDINGS.has(e);
117
+ const failsFloor = (ruling) => ruling.minted_false_claim === "yes" || ruling.closed_own_ticket === "no";
118
+ // An omp session id opens with its ISO start time, so two of them sort in time order.
119
+ const STARTED = /^\d{4}-\d{2}-\d{2}T/;
120
+ const startedBefore = (pull, row) => (STARTED.test(pull.session) && STARTED.test(row.session) ? pull.session < row.session : pull.run_date <= row.run_date);
121
+
122
+ function costOf(row) {
123
+ const c = String(row.cost ?? "");
124
+ if (c === "") return null;
125
+ const n = Number(c);
126
+ if (c.trim() === "" || !Number.isFinite(n)) throw new AttributionError(`member-outcomes row session ${row.session} agent ${row.agent}: cost '${c}' is not a number`);
127
+ return n;
128
+ }
129
+
130
+ /**
131
+ * One unit per ticket-features row dated on or after `windowStart` (every dated
132
+ * row when null), in file order: `{ pull, cell, stratum, status, merged, pr,
133
+ * ruling, pendingPr, claims, falseClaims, rows, spend }`. `prs` is `gh pr list`'s
134
+ * `{ number, state }` rows, or null when PR state is unknown (the fit). `stratum`
135
+ * maps a features row to its stratum. Throws `AttributionError` on a features
136
+ * row it cannot date or a booked `cost` it cannot read, and `RulingError` as
137
+ * `rulingFor` does.
138
+ */
139
+ export function attribute({ features, members, tiers, prs = null, windowStart = null, stratum = () => null }) {
140
+ const key = (session, agent) => `${session}\0${agent}`;
141
+ const pulls = datedRows(features);
142
+ const bySessionTicket = new Map(), byTicket = new Map();
143
+ for (const p of pulls) {
144
+ const k = key(p.session, p.ticket);
145
+ (bySessionTicket.get(k) ?? bySessionTicket.set(k, []).get(k)).push(p);
146
+ (byTicket.get(p.ticket) ?? byTicket.set(p.ticket, []).get(p.ticket)).push(p);
147
+ }
148
+
149
+ const omp = members.filter((r) => r.harness === "omp" && !EXCLUDED_MEMBER.test(lastSegment(r.member)));
150
+ // Implementer rows to their Pull, and a PR-named member's key to its PR.
151
+ const implPull = new Map(), prOfKey = new Map();
152
+ const impls = new Map(pulls.map((p) => [p, []]));
153
+ for (const r of omp) {
154
+ if (isImplementer(r.member)) {
155
+ if (!r.ticket) continue;
156
+ const own = bySessionTicket.get(key(r.session, r.ticket));
157
+ const pull = own ? (own.find((p) => p.agent === r.agent) ?? own.at(-1)) : byTicket.get(r.ticket)?.findLast((p) => startedBefore(p, r));
158
+ if (!pull) continue;
159
+ implPull.set(key(r.session, r.agent), pull);
160
+ impls.get(pull).push(r);
161
+ } else {
162
+ const { pr } = parseMemberName(r.member);
163
+ if (pr) prOfKey.set(key(r.session, r.agent), pr);
164
+ }
165
+ }
166
+ // Each PR to the last Pull, in file order, whose implementer rows carry it.
167
+ const carrier = new Map();
168
+ for (const p of pulls) for (const r of impls.get(p)) if (r.pr) carrier.set(r.pr, p);
169
+
170
+ // Every row books to its nearest booked ancestor, itself included.
171
+ const rows = new Map(pulls.map((p) => [p, []]));
172
+ for (const r of omp) {
173
+ const segs = String(r.agent).split("/");
174
+ for (let i = segs.length; i > 0; i--) {
175
+ const k = key(r.session, segs.slice(0, i).join("/"));
176
+ const pull = implPull.get(k) ?? carrier.get(prOfKey.get(k));
177
+ if (pull) rows.get(pull).push(r);
178
+ if (pull || prOfKey.has(k)) break;
179
+ }
180
+ }
181
+
182
+ // Rulings: each PR's merge to one Pull.
183
+ const inWindow = pulls.filter((p) => windowStart === null || p.run_date >= windowStart);
184
+ const rulings = rulingsByTicket(tiers);
185
+ const fileOrder = new Map(tiers.map((t, i) => [t, i]));
186
+ const candidates = new Map(); // pr -> [{ pull, ruling }]
187
+ for (const p of inWindow) {
188
+ const ruling = rulingFor(rulings, p.ticket, p.run_date);
189
+ if (ruling) (candidates.get(ruling.pr) ?? candidates.set(ruling.pr, []).get(ruling.pr)).push({ pull: p, ruling });
190
+ }
191
+ const ruled = new Map(); // pull -> { pr, ruling, carried }
192
+ for (const [pr, cands] of candidates) {
193
+ const ruling = cands.reduce((a, c) => (fileOrder.get(c.ruling) > fileOrder.get(a) ? c.ruling : a), cands[0].ruling);
194
+ const held = cands.find((c) => c.pull === carrier.get(pr));
195
+ ruled.set(held?.pull ?? cands.at(-1).pull, { pr, ruling, carried: !!held });
196
+ }
197
+
198
+ const state = prs && new Map(prs.map((p) => [String(p.number), p.state]));
199
+ return inWindow.map((pull) => {
200
+ const own = impls.get(pull);
201
+ const booked = rows.get(pull);
202
+ const r = ruled.get(pull) ?? null;
203
+ let merged = r !== null;
204
+ let pendingPr = null;
205
+ if (state) {
206
+ if (r) {
207
+ const s = state.get(r.pr);
208
+ if (s === undefined || s === "OPEN") pendingPr = r.pr;
209
+ merged = s === "MERGED";
210
+ } else pendingPr = own.find((m) => m.pr && state.get(m.pr) === "OPEN")?.pr ?? null;
211
+ }
212
+ let claims = 0, falseClaims = 0;
213
+ if (merged) {
214
+ claims++;
215
+ if (failsFloor(r.ruling)) falseClaims++;
216
+ }
217
+ for (const m of own) {
218
+ if (m.ending === "bail:fixed" || m.ending === "bail:fixed-refuted") claims++;
219
+ if (m.ending === "bail:fixed-refuted") falseClaims++;
220
+ }
221
+ let spend = 0, blank = false;
222
+ for (const m of booked) {
223
+ const c = costOf(m);
224
+ if (c === null) blank = true;
225
+ else spend += c;
226
+ }
227
+ let status = "counted";
228
+ if (!own.some((m) => admissibleRow(m, pull.chosen_cell))) status = "excluded";
229
+ else if (pendingPr !== null) status = "pending";
230
+ else if (claims === 0 && own.every((m) => ticketsEnding(m.ending) || NEITHER_ENDINGS.has(m.ending))) status = "left-out";
231
+ else if (blank || (r && !r.carried)) status = "unpriced";
232
+ return {
233
+ pull, cell: pull.chosen_cell, stratum: stratum(pull), status, merged, pr: r?.pr ?? null, ruling: r?.ruling ?? null,
234
+ pendingPr, claims, falseClaims, rows: booked, spend: blank ? null : spend,
235
+ };
236
+ });
237
+ }
238
+
239
+ /**
240
+ * Per cell, in cell-name order: `{ cell, n, merged, claims, failed, spend,
241
+ * excluded, left_out, unpriced, pending }`. `n`, `merged`, `claims`, `failed`
242
+ * and `spend` are over counted Pulls alone; the last four count the Pulls of
243
+ * each other status.
244
+ */
245
+ export function tally(units) {
246
+ const cells = new Map();
247
+ for (const u of units) {
248
+ const c = cells.get(u.cell) ?? cells.set(u.cell, {
249
+ cell: u.cell, n: 0, merged: 0, claims: 0, failed: 0, spend: 0, excluded: 0, left_out: 0, unpriced: 0, pending: 0,
250
+ }).get(u.cell);
251
+ if (u.status === "counted") {
252
+ c.n++;
253
+ if (u.merged) c.merged++;
254
+ c.claims += u.claims;
255
+ c.failed += u.falseClaims;
256
+ c.spend += u.spend;
257
+ } else c[u.status.replace("-", "_")]++;
258
+ }
259
+ return [...cells.values()].sort((a, b) => (a.cell < b.cell ? -1 : a.cell > b.cell ? 1 : 0));
260
+ }
@@ -47,19 +47,13 @@ import { makeDie, defineFlags } from "./arg.mjs";
47
47
  import { CELL, POLICY_CELL, drawCell, parseMember } from "./ledger-grammar.mjs";
48
48
  import { sessionFromReceipt } from "./phase0-receipt.mjs";
49
49
  import { parseTsv as parseMemberTsv } from "./member-outcomes.mjs";
50
- import { DATE, lastPullByTicket, parseTierOutcomes, rulingFor, rulingsByTicket } from "./tier-outcomes.mjs";
50
+ import { AttributionError, FEATURE_COLUMNS, attribute, parseFeatures, tally } from "./pull-attribution.mjs";
51
+ import { RulingError, parseTierOutcomes } from "./tier-outcomes.mjs";
51
52
 
52
53
  const NAME = "ticket-router";
53
54
  const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
54
55
  export const DEFAULT_TABLE = join(SCRIPT_DIR, "router-table.json");
55
56
 
56
- // The ticket-features row, one per Pull, in this order.
57
- export const COLUMNS = Object.freeze([
58
- "run_date", "session", "agent", "ticket", "policy_cell", "chosen_cell",
59
- "exploration_draw", "sizing_src", "sizing_pre", "router_usd", "brief_chars",
60
- "criteria", "comments", "age_days", "paths", "test_paths", "xrefs", "kind",
61
- ]);
62
-
63
57
  export const STRATA = Object.freeze(["light", "heavy", "unknown"]);
64
58
  // The free classifier's thresholds: heavy past the median brief length of
65
59
  // the tickets whose sizing verdict is known, or past five criteria.
@@ -323,140 +317,52 @@ export function route({ session, ticket, arm, implRow, chainHead = false, issue,
323
317
  }
324
318
 
325
319
  export function formatFeatureRow(row) {
326
- return COLUMNS.map((c) => String(row[c] ?? "").replace(/[\t\r\n]/g, " ")).join("\t");
327
- }
328
-
329
- /** Data rows of a ticket-features TSV as objects; throws on a missing header or a short row. */
330
- export function parseFeatures(text) {
331
- const lines = String(text ?? "").split("\n").filter((l) => l.trim() && !l.startsWith("#"));
332
- if (lines.length === 0) throw new Error("no header line");
333
- if (lines[0] !== COLUMNS.join("\t")) throw new Error(`header is not the ${COLUMNS.length} ticket-features columns`);
334
- return lines.slice(1).map((l) => {
335
- const cells = l.split("\t");
336
- if (cells.length !== COLUMNS.length) throw new Error(`malformed row: ${cells.length} fields, expected ${COLUMNS.length} — ${l.slice(0, 60)}`);
337
- return Object.fromEntries(COLUMNS.map((c, i) => [c, cells[i]]));
338
- });
320
+ return FEATURE_COLUMNS.map((c) => String(row[c] ?? "").replace(/[\t\r\n]/g, " ")).join("\t");
339
321
  }
340
322
 
341
323
  // ---------------------------------------------------------------------------
342
324
  // fit
343
325
  // ---------------------------------------------------------------------------
344
326
 
345
- // Members whose cost does not vary with the implementer cell.
346
- const UNBOOKED = (name) => name.startsWith("merge-bot-") || name === "memory" || name === "__advisor";
347
327
  const round4 = (x) => Math.round(x * 1e4) / 1e4;
348
328
 
349
329
  /**
350
- * The verdict on a ticket's Pull dated `pullDate`, `{ pr, fail }`, or null: the
351
- * ruling `rulingFor` picks. A verdict is a ruled PR; the floor fails on a
352
- * minted false claim or an unclosed ticket.
353
- */
354
- function verdictOf(rulings, ticket, pullDate) {
355
- const r = rulingFor(rulings, ticket, pullDate);
356
- return r && { pr: String(r.pr ?? ""), fail: r.minted_false_claim === "yes" || r.closed_own_ticket === "no" };
357
- }
358
-
359
- /**
360
- * The features rows with a YYYY-MM-DD `run_date`, the rows that can be placed
361
- * against the window, the cut and a ruling. A blank `run_date` (route writes one
362
- * for a session id with no date) is no input row and is left out. Any other
363
- * date is a hand-edit that cannot be placed and is refused, as `rulingFor`
364
- * refuses a ruling's: dropping it would shrink the fit unseen.
330
+ * The fit's Pulls: pull-attribution.mjs's units over the window — booked as
331
+ * routed, each to its own row's cell and stratum — restricted to rows the free
332
+ * classifier routed plus exploration rows: a row a live B classifier routed is
333
+ * the A/B's test set, not the fit's. The fit sees no PR state, so a Pull with
334
+ * a Ruling is merged. A fit reads everything in the commit it is given: no row
335
+ * is cut by date, because `--check` re-reads that same commit rather than
336
+ * re-cutting the working tree's files.
365
337
  */
366
- function datedRows(features) {
367
- return features.filter((r) => {
368
- if (DATE.test(r.run_date)) return true;
369
- if (r.run_date !== "") throw new Error(`ticket #${r.ticket}: a ticket-features row's run_date is '${r.run_date}', expected YYYY-MM-DD or blank`);
370
- return false;
371
- });
338
+ function fitPulls({ features, members, verdicts, window }) {
339
+ return attribute({ features, members, tiers: verdicts, windowStart: window || null, stratum: ruleStratum })
340
+ .filter((u) => u.pull.exploration_draw !== "" || u.pull.sizing_src === "rule");
372
341
  }
373
342
 
374
- /**
375
- * Per-ticket input rows: the ticket's features rows inside the window,
376
- * attributed to its LAST row's (in file order) cell and stratum, restricted to rows the
377
- * free classifier routed plus exploration rows — a row a live B classifier
378
- * routed is the A/B's test set, not the fit's. Its verdict rules its last
379
- * input row. A row with a blank `run_date` (route writes one for a session id
380
- * with no date) is no input row: it cannot be placed against the window or
381
- * its ruling, and a window bound drops it anyway. A `run_date` that is
382
- * neither blank nor YYYY-MM-DD is refused (see `datedRows`).
383
- * A fit reads everything in the commit it is given: no row is cut by date,
384
- * because `--check` re-reads that same commit rather than re-cutting the
385
- * working tree's files.
386
- */
387
- function fitTickets({ features, members, verdicts, window }) {
388
- const inRange = datedRows(features).filter((r) => !window || r.run_date >= window);
389
- const byTicket = new Map();
390
- for (const r of inRange) {
391
- if (!byTicket.has(r.ticket)) byTicket.set(r.ticket, []);
392
- byTicket.get(r.ticket).push(r);
393
- }
394
- // `rulingFor` refuses a ruling whose date is not YYYY-MM-DD when its ticket
395
- // is an input ticket. An input ticket is one the fit reads — its last
396
- // in-window row is an exploration row or has `sizing_src` `rule`, which the
397
- // skip below keeps — not every ticket with an in-window features row.
398
- // `fit --due` also refuses for a ticket the skip drops: `mergedSince` calls
399
- // `verdictOf` on every ticket with an in-window row the fitted commit does
400
- // not hold, with no such skip.
401
- const rulings = rulingsByTicket(verdicts);
402
- const lastPulls = lastPullByTicket(inRange);
403
- const out = [];
404
- for (const [ticket, rows] of byTicket) {
405
- const last = lastPulls.get(ticket);
406
- if (last.exploration_draw === "" && last.sizing_src !== "rule") continue;
407
- const verdict = verdictOf(rulings, ticket, last.run_date);
408
- const keys = new Set(rows.map((r) => `${r.session}\0${r.agent}`));
409
- let cost = 0;
410
- let costKnown = true;
411
- let booked = 0;
412
- let fixRounds = 0;
413
- for (const mr of members) {
414
- const name = String(mr.member ?? mr.agent ?? "");
415
- if (UNBOOKED(name)) continue;
416
- if (!(keys.has(`${mr.session}\0${mr.agent}`) || mr.ticket === ticket || (verdict && verdict.pr && mr.pr === verdict.pr))) continue;
417
- booked++;
418
- // `cost` is a priced member-outcomes column; a row without it makes
419
- // the ticket's $ unknown, so the fit adopts nothing off a TSV that lacks the column.
420
- if (mr.cost === undefined || mr.cost === "" || !Number.isFinite(Number(mr.cost))) costKnown = false;
421
- else cost += Number(mr.cost);
422
- if (parseMember(name)?.family === "fix-pr") fixRounds++;
423
- }
424
- // A ticket with no member row on record has an unknown cost, not a free one.
425
- if (booked === 0) costKnown = false;
426
- for (const r of rows) if (r.router_usd !== "") cost += Number(r.router_usd) || 0;
427
- out.push({ ticket, cell: last.chosen_cell, stratum: ruleStratum(last), verdict, cost, costKnown, fixRounds });
428
- }
429
- return out;
430
- }
431
-
432
- function estimate(tickets) {
433
- const est = {};
434
- for (const t of tickets) {
435
- for (const s of [t.stratum, "*"]) {
436
- est[s] ??= {};
437
- const e = (est[s][t.cell] ??= { n: 0, merged: 0, fails: 0, cost: 0, costKnown: true, fix: 0 });
438
- e.n++;
439
- e.cost += t.cost;
440
- e.costKnown &&= t.costKnown;
441
- if (t.verdict) {
442
- e.merged++;
443
- if (t.verdict.fail) e.fails++;
444
- e.fix += t.fixRounds;
445
- }
446
- }
447
- }
343
+ // Per stratum and `*`, per cell: `n` counted Pulls, `merged`, $ per merged PR,
344
+ // false claims over claims, fix rounds per merged PR, and the cell's excluded,
345
+ // left-out and unpriced Pull counts.
346
+ function estimate(units) {
448
347
  const out = {};
449
- for (const s of Object.keys(est).sort()) {
348
+ for (const s of units.length ? [...new Set([...units.map((u) => u.stratum), "*"])].sort() : []) {
349
+ const mine = s === "*" ? units : units.filter((u) => u.stratum === s);
450
350
  out[s] = {};
451
- for (const c of Object.keys(est[s]).sort()) {
452
- const e = est[s][c];
453
- out[s][c] = {
454
- n: e.n,
455
- merged: e.merged,
456
- usd_per_merged: e.merged && e.costKnown ? round4(e.cost / e.merged) : null,
457
- fail_rate: e.merged ? round4(e.fails / e.merged) : null,
458
- fix_rounds: e.merged ? round4(e.fix / e.merged) : null,
351
+ for (const t of tally(mine)) {
352
+ const fix = mine.filter((u) => u.cell === t.cell && u.status === "counted" && u.merged)
353
+ .reduce((n, u) => n + u.rows.filter((r) => parseMember(r.member)?.family === "fix-pr").length, 0);
354
+ out[s][t.cell] = {
355
+ n: t.n,
356
+ merged: t.merged,
357
+ claims: t.claims,
358
+ failed: t.failed,
359
+ usd_per_merged: t.merged ? round4(t.spend / t.merged) : null,
360
+ fail_rate: t.claims ? round4(t.failed / t.claims) : null,
361
+ fix_rounds: t.merged ? round4(fix / t.merged) : null,
459
362
  review_findings: null,
363
+ excluded: t.excluded,
364
+ left_out: t.left_out,
365
+ unpriced: t.unpriced,
460
366
  };
461
367
  }
462
368
  }
@@ -487,8 +393,8 @@ function adopt(est, cells, tripped) {
487
393
  */
488
394
  export function fitTable({ prior, features, members, verdicts, guard, commit = null }) {
489
395
  const window = guard.window_start ?? prior.window_start ?? null;
490
- const tickets = fitTickets({ features, members, verdicts, window });
491
- const estimates = estimate(tickets);
396
+ const units = fitPulls({ features, members, verdicts, window });
397
+ const estimates = estimate(units);
492
398
  const tripped = new Set(guard.tripped);
493
399
 
494
400
  let { stage, cells } = prior;
@@ -508,7 +414,7 @@ export function fitTable({ prior, features, members, verdicts, guard, commit = n
508
414
  return {
509
415
  window_start: window,
510
416
  fitted_commit: commit,
511
- n_rows: tickets.length,
417
+ n_rows: units.length,
512
418
  stage,
513
419
  cells,
514
420
  burn_in: !cells.every(atN),
@@ -520,16 +426,22 @@ export function fitTable({ prior, features, members, verdicts, guard, commit = n
520
426
  }
521
427
 
522
428
  /**
523
- * Merged PRs (tickets with a verdict on their last such row in file order)
524
- * among the YYYY-MM-DD-dated features rows on or after `window_start` that the
525
- * fitted commit's features file (`fitted`, parsed rows; empty before any fit)
526
- * does not hold, matched by `session` and `agent`.
429
+ * Merged PRs since the fit: pull-attribution.mjs's counted, merged Pulls on or
430
+ * after `window_start` whose features row the fitted commit's features file
431
+ * (`fitted`, parsed rows; empty before any fit) does not hold, matched by
432
+ * `session` and `agent`.
527
433
  */
528
- export function mergedSince({ table, features, verdicts, fitted }) {
529
- const rulings = rulingsByTicket(verdicts);
434
+ export function mergedSince({ table, features, members, verdicts, fitted }) {
530
435
  const read = new Set(fitted.map((r) => `${r.session}\0${r.agent}`));
531
- const after = datedRows(features).filter((r) => (!table.window_start || r.run_date >= table.window_start) && !read.has(`${r.session}\0${r.agent}`));
532
- return [...lastPullByTicket(after)].filter(([ticket, p]) => verdictOf(rulings, ticket, p.run_date)).length;
436
+ return attribute({ features, members, tiers: verdicts, windowStart: table.window_start || null })
437
+ .filter((u) => u.status === "counted" && u.merged && !read.has(`${u.pull.session}\0${u.pull.agent}`)).length;
438
+ }
439
+
440
+ // Each cell's Pull counts over every stratum, as the fit prints them.
441
+ function cellCounts(estimates) {
442
+ return Object.entries(estimates["*"] ?? {})
443
+ .map(([c, e]) => `${c} n=${e.n} merged=${e.merged} failed=${e.failed} excluded=${e.excluded} left_out=${e.left_out} unpriced=${e.unpriced}`)
444
+ .join(", ") || "none";
533
445
  }
534
446
 
535
447
  // ---------------------------------------------------------------------------
@@ -630,7 +542,7 @@ function main() {
630
542
  // the rule, and neither this append nor the drain checks it.
631
543
  try {
632
544
  mkdirSync(dirname(pending), { recursive: true });
633
- const header = statSync(pending, { throwIfNoEntry: false })?.size > 0 ? "" : `${COLUMNS.join("\t")}\n`;
545
+ const header = statSync(pending, { throwIfNoEntry: false })?.size > 0 ? "" : `${FEATURE_COLUMNS.join("\t")}\n`;
634
546
  appendFileSync(pending, `${header}${formatFeatureRow(row)}\n`);
635
547
  } catch (e) {
636
548
  process.stderr.write(`${NAME}: WARNING could not append to ${pending}: ${e.message} — this Pull's ticket-features row is not recorded\n`);
@@ -664,13 +576,15 @@ function main() {
664
576
  }
665
577
  return sha;
666
578
  };
667
- // The join throws on a features row it cannot place and on a ruling of an input ticket it cannot place.
668
- // For `fit` and `--check` an input ticket is one `fitTickets` reads (its last in-window row is an
669
- // exploration row or has `sizing_src` `rule`), not every ticket with an in-window features row. `fit --due` also
670
- // covers a ticket `fitTickets` skips, because `mergedSince` calls `verdictOf` on every ticket with an in-window
671
- // row the fitted commit does not hold, with no such skip.
579
+ // The join throws on a features row it cannot date, a booked member row whose `cost` is not a number, and a
580
+ // ruling of a windowed Pull's ticket it cannot place — for `fit` and `--check` too on a Pull the fit then skips
581
+ // as a live B classifier's, since every windowed Pull is booked before that skip. Anything else it throws is a
582
+ // bug, never bad input, so it surfaces with its stack.
672
583
  const joined = (fit) => {
673
- try { return fit(); } catch (e) { die(`cannot join --features ${need("features")} with --verdicts ${need("verdicts")}: ${e.message}`); }
584
+ try { return fit(); } catch (e) {
585
+ if (!(e instanceof RulingError || e instanceof AttributionError)) throw e;
586
+ die(`cannot join --features ${need("features")}, --members ${need("members")} and --verdicts ${need("verdicts")}: ${e.message}`);
587
+ }
674
588
  };
675
589
 
676
590
  if (mode === "check") {
@@ -685,7 +599,7 @@ function main() {
685
599
  process.exitCode = 1;
686
600
  return;
687
601
  }
688
- process.stdout.write(`${NAME}: ${tablePath} matches a re-fit of ${refit.n_rows} tickets at ${short(fitted)}\n`);
602
+ process.stdout.write(`${NAME}: ${tablePath} matches a re-fit of ${refit.n_rows} Pulls at ${short(fitted)}\n`);
689
603
  return;
690
604
  }
691
605
 
@@ -699,14 +613,14 @@ function main() {
699
613
  if (F.has("due")) {
700
614
  const fitted = fittedCommit();
701
615
  const read = fitted === null ? [] : fileAt(fitted, "features", parseFeatures);
702
- const n = joined(() => mergedSince({ table, features, verdicts, fitted: read }));
616
+ const n = joined(() => mergedSince({ table, features, members, verdicts, fitted: read }));
703
617
  process.stdout.write(`DUE=${n >= FIT_EVERY_MERGED ? "yes" : "no"} MERGED=${n}/${FIT_EVERY_MERGED}\n`);
704
618
  return;
705
619
  }
706
620
  const next = joined(() => fitTable({ prior: table, features, members, verdicts, guard, commit: sha }));
707
621
  writeFileSync(`${tablePath}.tmp`, `${JSON.stringify(next, null, 2)}\n`);
708
622
  renameSync(`${tablePath}.tmp`, tablePath);
709
- process.stdout.write(`${NAME}: fitted ${next.n_rows} tickets at ${short(sha)}; rows ${JSON.stringify(next.rows)}; burn_in ${next.burn_in}; stage ${next.stage}\n`);
623
+ process.stdout.write(`${NAME}: fitted ${next.n_rows} Pulls at ${short(sha)}; rows ${JSON.stringify(next.rows)}; burn_in ${next.burn_in}; stage ${next.stage}; cells ${cellCounts(next.estimates)}\n`);
710
624
  }
711
625
 
712
626
  if (isCLI(import.meta.url)) main();