@feigi/fleet-ctl 3.92.2 → 3.93.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/cell-readout.mjs +3 -7
- package/scripts/member-outcomes.mjs +1 -1
- package/scripts/metrics-refresh.mjs +1 -1
- package/scripts/pr-cost.mjs +88 -230
- package/scripts/pull-attribution.mjs +260 -0
- package/scripts/ticket-router.mjs +62 -148
package/package.json
CHANGED
package/scripts/cell-readout.mjs
CHANGED
|
@@ -76,9 +76,9 @@
|
|
|
76
76
|
import { readFileSync } from "node:fs";
|
|
77
77
|
import { makeDie, defineFlags } from "./arg.mjs";
|
|
78
78
|
import { isCLI } from "./is-cli.mjs";
|
|
79
|
-
import {
|
|
79
|
+
import { POLICY_CELL } from "./ledger-grammar.mjs";
|
|
80
80
|
import { formatTsv as formatMemberTsv, parseTsv as parseMemberTsv } from "./member-outcomes.mjs";
|
|
81
|
-
import { parseFeatures } from "./
|
|
81
|
+
import { admissibleRow, parseFeatures } from "./pull-attribution.mjs";
|
|
82
82
|
import { DATE, rulingFor, rulingsByTicket } from "./tier-outcomes.mjs";
|
|
83
83
|
|
|
84
84
|
const NAME = "cell-readout";
|
|
@@ -86,7 +86,6 @@ export const GATE = Object.freeze({ comparisons: 10, runDates: 5 });
|
|
|
86
86
|
export const STOP = Object.freeze({ verdicts: 10, failRate: 0.8 });
|
|
87
87
|
|
|
88
88
|
const key = (session, agent) => `${session}\0${agent}`;
|
|
89
|
-
const levelOf = (cell) => CELL.exec(cell)?.[2] ?? null;
|
|
90
89
|
|
|
91
90
|
/**
|
|
92
91
|
* `features` keyed by session+agent. A repeat naming the same `chosen_cell`
|
|
@@ -106,10 +105,7 @@ function pullsOf(features) {
|
|
|
106
105
|
/** The join's member row for a Pull when the Pull is admissible, else null. */
|
|
107
106
|
function admissibleMember(pull, members) {
|
|
108
107
|
const m = members.get(key(pull.session, pull.agent));
|
|
109
|
-
|
|
110
|
-
const level = levelOf(pull.chosen_cell);
|
|
111
|
-
if (level === null || m.subagentType !== `fleet-implementer-${pull.chosen_cell}` || m.effort !== level || !m.model) return null;
|
|
112
|
-
return m;
|
|
108
|
+
return m && admissibleRow(m, pull.chosen_cell) ? m : null;
|
|
113
109
|
}
|
|
114
110
|
|
|
115
111
|
// One member row per session+agent. A key repeated with different fields is
|
|
@@ -141,7 +141,7 @@ export function withReports(rows, scratch) {
|
|
|
141
141
|
// crash the session never exited and wrote no yield sidecar;
|
|
142
142
|
// hollow none of the above: the member stopped on its own.
|
|
143
143
|
const IMPLEMENTER_LEAF = /^impl-\d+(?:-[a-z0-9]+)?$/;
|
|
144
|
-
const isImplementer = (member) => IMPLEMENTER_LEAF.test(member.split("/").pop());
|
|
144
|
+
export const isImplementer = (member) => IMPLEMENTER_LEAF.test(member.split("/").pop());
|
|
145
145
|
|
|
146
146
|
export function endingOf({ member, pr, signals = null, earlierPrs = new Set(), verifyMain = verifyMainWithGit }) {
|
|
147
147
|
if (!isImplementer(member)) return "";
|
|
@@ -42,7 +42,7 @@ import { makeDie, defineFlags } from "./arg.mjs";
|
|
|
42
42
|
import { gitEnv } from "./git-env.mjs";
|
|
43
43
|
import { FleetDirUnresolvable, fleetFile } from "./fleet-dir.mjs";
|
|
44
44
|
import { isCLI } from "./is-cli.mjs";
|
|
45
|
-
import { FEATURE_COLUMNS, parseFeatures } from "./
|
|
45
|
+
import { FEATURE_COLUMNS, parseFeatures } from "./pull-attribution.mjs";
|
|
46
46
|
|
|
47
47
|
const NAME = "metrics-refresh";
|
|
48
48
|
const PR_COST = fileURLToPath(new URL("./pr-cost.mjs", import.meta.url));
|
package/scripts/pr-cost.mjs
CHANGED
|
@@ -25,55 +25,42 @@
|
|
|
25
25
|
// fleet-tick's `router` row prints it today; a missing or unparseable file
|
|
26
26
|
// means the default cell only, so deleting it cannot evade the guard.
|
|
27
27
|
//
|
|
28
|
-
// BOOKING
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
//
|
|
32
|
-
//
|
|
33
|
-
//
|
|
34
|
-
//
|
|
35
|
-
//
|
|
36
|
-
//
|
|
37
|
-
// is
|
|
38
|
-
//
|
|
28
|
+
// BOOKING is pull-attribution.mjs's, the one the router's fit reads too: each
|
|
29
|
+
// Pull in the window is a unit booked as routed to its `chosen_cell`, with
|
|
30
|
+
// every implementer under it, its PR's review, fix and finisher members and
|
|
31
|
+
// their nested members, and its merge through its Ruling — merged when that
|
|
32
|
+
// PR's state is MERGED. A Pull is counted, or excluded (no Admissible row),
|
|
33
|
+
// pending (its PR still open, or not in the list), left out (never merged, no
|
|
34
|
+
// `fixed` bail, and every ending the ticket's or neither's) or unpriced (a
|
|
35
|
+
// booked row with a blank `cost`, or a Ruling whose PR none of its
|
|
36
|
+
// implementers carries); only counted Pulls enter a cell's figures. A `cost`
|
|
37
|
+
// that is neither blank nor a number is an input error (exit 2), as is a
|
|
38
|
+
// ticket-features `run_date` that is neither blank nor YYYY-MM-DD, or a ruling
|
|
39
|
+
// of a windowed Pull's ticket whose run_date is not YYYY-MM-DD or whose verdict
|
|
40
|
+
// is not yes or no.
|
|
39
41
|
//
|
|
40
|
-
//
|
|
41
|
-
//
|
|
42
|
-
//
|
|
43
|
-
//
|
|
44
|
-
//
|
|
45
|
-
//
|
|
46
|
-
//
|
|
47
|
-
//
|
|
48
|
-
//
|
|
49
|
-
//
|
|
50
|
-
// `fleet-implementer-<cell>`, or an `effort` other than the cell's level,
|
|
51
|
-
// lists the PR under `mismatch` and books it to no cell at all. A PR still
|
|
52
|
-
// OPEN is pending and booked nowhere yet. A Pull with no ruling books to its
|
|
53
|
-
// own `chosen_cell` as unmerged spend — unless its member opened a PR that is
|
|
54
|
-
// still open, which is pending too.
|
|
55
|
-
//
|
|
56
|
-
// PER CELL: `n_pulls`; `n_merged`; `n_pass` (merged PRs passing the quality
|
|
57
|
-
// floor — failure is `minted_false_claim=yes` or `closed_own_ticket=no`);
|
|
58
|
-
// `fail_rate` = failed / merged; `mean_usd` = every $ booked to the cell —
|
|
59
|
-
// merged, unmerged and superseded — / merged, so a cell that abandons work
|
|
60
|
-
// carries that spend; `median_usd` over merged PRs alone, descriptive;
|
|
61
|
-
// `router_usd`, the Pulls' own routing cost, summed. `usd_diff_ci95` is a
|
|
62
|
-
// bootstrap 95% interval on `mean_usd` minus the baseline's — printed to be
|
|
63
|
-
// read, never judged on.
|
|
42
|
+
// PER CELL, over its counted Pulls: `n_pulls`; `n_merged`; `n_claims` (merged
|
|
43
|
+
// PRs plus every `bail:fixed` or `bail:fixed-refuted` ending); `n_failed`
|
|
44
|
+
// (merged PRs ruled `minted_false_claim=yes` or `closed_own_ticket=no`, plus
|
|
45
|
+
// every `bail:fixed-refuted`); `fail_rate` = failed / claims; `mean_usd` = the
|
|
46
|
+
// Pulls' spend — merged or not — / merged, so a cell that abandons work carries
|
|
47
|
+
// that spend; `median_usd` over the merged Pulls alone, descriptive;
|
|
48
|
+
// `router_usd`, the Pulls' own routing cost, summed and never in their spend.
|
|
49
|
+
// `usd_diff_ci95` is a bootstrap 95% interval on `mean_usd` minus the
|
|
50
|
+
// baseline's — printed to be read, never judged on. Then the cell's
|
|
51
|
+
// `excluded`, `left_out` and `unpriced` Pull counts.
|
|
64
52
|
|
|
65
53
|
import { execFileSync } from "node:child_process";
|
|
66
54
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
67
55
|
import { dirname, join } from "node:path";
|
|
68
56
|
import { fileURLToPath } from "node:url";
|
|
69
|
-
import { makeDie, defineFlags
|
|
57
|
+
import { makeDie, defineFlags } from "./arg.mjs";
|
|
70
58
|
import { gitEnv } from "./git-env.mjs";
|
|
71
59
|
import { fleetFile, FleetDirUnresolvable } from "./fleet-dir.mjs";
|
|
72
60
|
import { isCLI } from "./is-cli.mjs";
|
|
73
|
-
import { CELL } from "./ledger-grammar.mjs";
|
|
74
|
-
import { parseMemberName } from "./member-record.mjs";
|
|
75
61
|
import { parseTsv as parseMemberTsv } from "./member-outcomes.mjs";
|
|
76
|
-
import {
|
|
62
|
+
import { AttributionError, attribute, parseFeatures, tally } from "./pull-attribution.mjs";
|
|
63
|
+
import { RulingError, parseTierOutcomes } from "./tier-outcomes.mjs";
|
|
77
64
|
|
|
78
65
|
const NAME = "pr-cost";
|
|
79
66
|
const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
|
|
@@ -91,38 +78,7 @@ export const TRIP_POINTS = 15;
|
|
|
91
78
|
export const EXIT = { ok: 0, tripped: 3, none: 4 };
|
|
92
79
|
export const GH_PR_LIMIT = 1000;
|
|
93
80
|
|
|
94
|
-
|
|
95
|
-
"run_date", "session", "agent", "ticket", "policy_cell", "chosen_cell", "exploration_draw",
|
|
96
|
-
"sizing_src", "sizing_pre", "router_usd", "brief_chars", "criteria", "comments", "age_days",
|
|
97
|
-
"paths", "test_paths", "xrefs", "kind",
|
|
98
|
-
];
|
|
99
|
-
|
|
100
|
-
// ticket-features.tsv: one bare header line naming the columns, then rows.
|
|
101
|
-
// Read by header name; a row whose width differs from the header's is refused.
|
|
102
|
-
export function parseFeatures(text) {
|
|
103
|
-
const lines = String(text ?? "").split("\n").filter((l) => l.trim() && !l.startsWith("#"));
|
|
104
|
-
if (lines.length === 0) throw new Error("ticket-features.tsv has no header line");
|
|
105
|
-
const header = lines[0].split("\t");
|
|
106
|
-
const missing = FEATURE_COLUMNS.filter((c) => !header.includes(c));
|
|
107
|
-
if (missing.length) throw new Error(`ticket-features.tsv header lacks ${missing.join(", ")}`);
|
|
108
|
-
return lines.slice(1).map((l, i) => {
|
|
109
|
-
const cells = l.split("\t");
|
|
110
|
-
if (cells.length !== header.length) {
|
|
111
|
-
throw new Error(`ticket-features.tsv row ${i + 1}: ${cells.length} fields, expected ${header.length} — ${l.slice(0, 60)}`);
|
|
112
|
-
}
|
|
113
|
-
const row = Object.fromEntries(header.map((c, j) => [c, cells[j]]));
|
|
114
|
-
if (!isDigits(row.ticket)) throw new Error(`ticket-features.tsv row ${i + 1}: ticket ${JSON.stringify(row.ticket)} is not a number`);
|
|
115
|
-
if (!CELL.test(row.chosen_cell)) throw new Error(`ticket-features.tsv row ${i + 1}: chosen_cell ${JSON.stringify(row.chosen_cell)} is not a cell`);
|
|
116
|
-
return row;
|
|
117
|
-
});
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
export const EXCLUDED_MEMBER = /^(?:merge-?bot|memory$|__advisor$)/i;
|
|
121
|
-
// omp nests a member's id as `<parent>/<parent>.<child>`: the child is the last segment after either separator.
|
|
122
|
-
export const lastSegment = (agent) => String(agent).split(/[/.]/).at(-1);
|
|
123
|
-
const usd = (r) => (r.cost === "" || r.cost === undefined ? 0 : Number(r.cost));
|
|
124
|
-
const isMerged = (pr) => pr?.state === "MERGED";
|
|
125
|
-
const failed = (tier) => tier.minted_false_claim === "yes" || tier.closed_own_ticket === "no";
|
|
81
|
+
const routerUsd = (pull) => (pull.router_usd === "" ? 0 : Number(pull.router_usd));
|
|
126
82
|
const round = (x, d = 4) => (x === null || !Number.isFinite(x) ? null : Number(x.toFixed(d)));
|
|
127
83
|
|
|
128
84
|
function median(xs) {
|
|
@@ -167,7 +123,8 @@ export function bootstrapDiffCI(cellUnits, baseUnits, { iterations = 2000, seed
|
|
|
167
123
|
}
|
|
168
124
|
|
|
169
125
|
// The trip rule, in integer counts so a 3-of-20 difference is exactly 15
|
|
170
|
-
// points rather than 14.999… in floating point.
|
|
126
|
+
// points rather than 14.999… in floating point. `n` is a cell's claims and
|
|
127
|
+
// `n_pass` the claims among them that held.
|
|
171
128
|
export function trips(cell, base) {
|
|
172
129
|
const failC = cell.n - cell.n_pass, failB = base.n - base.n_pass;
|
|
173
130
|
const failRise = (failC * base.n - failB * cell.n) * 100 >= TRIP_POINTS * cell.n * base.n;
|
|
@@ -175,142 +132,46 @@ export function trips(cell, base) {
|
|
|
175
132
|
}
|
|
176
133
|
|
|
177
134
|
export function computeReport({ members, tiers, features, prs, routerTable = null, pricing = null, windowStart = WINDOW_START }) {
|
|
178
|
-
const
|
|
179
|
-
const
|
|
180
|
-
const
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
const
|
|
184
|
-
const
|
|
185
|
-
const
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
for (let i = segs.length; i > 0; i--) {
|
|
200
|
-
const k = memberKey(r.session, segs.slice(0, i).join("/"));
|
|
201
|
-
const pull = pullByKey.get(k);
|
|
202
|
-
const pr = prMemberKeys.get(k);
|
|
203
|
-
if (!pull && !pr) continue;
|
|
204
|
-
booked.push([r, pull ?? null, pull ? null : pr]);
|
|
205
|
-
if (pull) {
|
|
206
|
-
pullUsd.set(pull, (pullUsd.get(pull) ?? 0) + usd(r));
|
|
207
|
-
if (i === segs.length) memberOfPull.set(pull, r);
|
|
208
|
-
} else prMemberUsd.set(pr, (prMemberUsd.get(pr) ?? 0) + usd(r));
|
|
209
|
-
break;
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
// Rulings: per ticket, `rulingFor` of the ticket's first Pull — the last
|
|
214
|
-
// ruling row in file order dated on or after it, both-blank rows skipped.
|
|
215
|
-
const rulings = rulingsByTicket(tiers);
|
|
216
|
-
const fileOrder = new Map(tiers.map((t, i) => [t, i]));
|
|
217
|
-
const pullsByTicket = new Map();
|
|
218
|
-
for (const p of pulls) {
|
|
219
|
-
if (!pullsByTicket.has(p.ticket)) pullsByTicket.set(p.ticket, []);
|
|
220
|
-
pullsByTicket.get(p.ticket).push(p);
|
|
221
|
-
}
|
|
222
|
-
const groups = new Map(); // pr -> { pr, tier, pulls[] }
|
|
223
|
-
const unruled = [];
|
|
224
|
-
for (const [ticket, ps] of pullsByTicket) {
|
|
225
|
-
const ruling = rulingFor(rulings, ticket, ps[0].run_date);
|
|
226
|
-
if (!ruling) { unruled.push(...ps); continue; }
|
|
227
|
-
const g = groups.get(ruling.pr) ?? { pr: ruling.pr, tier: ruling, pulls: [] };
|
|
228
|
-
if (fileOrder.get(ruling) > fileOrder.get(g.tier)) g.tier = ruling;
|
|
229
|
-
for (const p of ps) (p.run_date <= ruling.run_date ? g.pulls : unruled).push(p);
|
|
230
|
-
groups.set(ruling.pr, g);
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
const cells = new Map();
|
|
234
|
-
const cellOf = (c) => {
|
|
235
|
-
if (!cells.has(c)) {
|
|
236
|
-
cells.set(c, { cell: c, n_pulls: 0, n_merged: 0, n_pass: 0, usd: 0, router_usd: 0, merged_usd: [], units: [] });
|
|
237
|
-
}
|
|
238
|
-
return cells.get(c);
|
|
239
|
-
};
|
|
240
|
-
const routerUsd = (p) => (p.router_usd === "" ? 0 : Number(p.router_usd));
|
|
241
|
-
const mismatch = [], pending = [];
|
|
242
|
-
|
|
243
|
-
for (const g of groups.values()) {
|
|
244
|
-
if (g.pulls.length === 0) continue;
|
|
245
|
-
const carrier = g.pulls.at(-1);
|
|
246
|
-
const cell = carrier.chosen_cell;
|
|
247
|
-
const m = memberOfPull.get(carrier);
|
|
248
|
-
const level = cell.split("-").slice(1).join("-");
|
|
249
|
-
if (m && (m.subagentType !== `fleet-implementer-${cell}` || m.effort !== level)) {
|
|
250
|
-
mismatch.push({ pr: g.pr, cell, subagent_type: m.subagentType, effort: m.effort });
|
|
251
|
-
continue;
|
|
252
|
-
}
|
|
253
|
-
const state = prState.get(g.pr);
|
|
254
|
-
if (!state || state.state === "OPEN") { pending.push(g.pr); continue; }
|
|
255
|
-
const total = g.pulls.reduce((s, p) => s + (pullUsd.get(p) ?? 0), 0) + (prMemberUsd.get(g.pr) ?? 0);
|
|
256
|
-
const c = cellOf(cell);
|
|
257
|
-
for (const p of g.pulls) { cellOf(p.chosen_cell).n_pulls++; c.router_usd += routerUsd(p); }
|
|
258
|
-
c.usd += total;
|
|
259
|
-
const merged = isMerged(state) ? 1 : 0;
|
|
260
|
-
c.units.push({ usd: total, merged });
|
|
261
|
-
if (merged) {
|
|
262
|
-
c.n_merged++;
|
|
263
|
-
c.merged_usd.push(total);
|
|
264
|
-
if (!failed(g.tier)) c.n_pass++;
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
for (const p of unruled) {
|
|
268
|
-
const opened = memberOfPull.get(p)?.pr;
|
|
269
|
-
if (opened && prState.get(opened)?.state === "OPEN") { pending.push(opened); continue; }
|
|
270
|
-
const c = cellOf(p.chosen_cell);
|
|
271
|
-
c.n_pulls++;
|
|
272
|
-
c.router_usd += routerUsd(p);
|
|
273
|
-
const spent = pullUsd.get(p) ?? 0;
|
|
274
|
-
c.usd += spent;
|
|
275
|
-
c.units.push({ usd: spent, merged: 0 });
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
const base = cells.get(BASELINE_CELL) ?? null;
|
|
279
|
-
const meanUsd = (c) => (c.n_merged ? c.usd / c.n_merged : null);
|
|
280
|
-
const figures = [...cells.values()].sort((a, b) => a.cell.localeCompare(b.cell)).map((c) => ({
|
|
281
|
-
cell: c.cell,
|
|
282
|
-
n_pulls: c.n_pulls,
|
|
283
|
-
n_merged: c.n_merged,
|
|
284
|
-
n_pass: c.n_pass,
|
|
285
|
-
fail_rate: c.n_merged ? round((c.n_merged - c.n_pass) / c.n_merged) : null,
|
|
286
|
-
mean_usd: round(meanUsd(c), 2),
|
|
287
|
-
median_usd: round(median(c.merged_usd), 2),
|
|
288
|
-
router_usd: round(c.router_usd, 4),
|
|
289
|
-
usd_diff_ci95: c.cell === BASELINE_CELL || !base ? null : bootstrapDiffCI(c.units, base.units),
|
|
135
|
+
const units = attribute({ features, members, tiers, prs, windowStart });
|
|
136
|
+
const counted = units.filter((u) => u.status === "counted");
|
|
137
|
+
const tallies = new Map(tally(units).map((t) => [t.cell, t]));
|
|
138
|
+
const countedIn = (cell) => counted.filter((u) => u.cell === cell);
|
|
139
|
+
const bootUnits = (cell) => countedIn(cell).map((u) => ({ usd: u.spend, merged: u.merged ? 1 : 0 }));
|
|
140
|
+
const meanUsd = (t) => (t.merged ? t.spend / t.merged : null);
|
|
141
|
+
const base = tallies.get(BASELINE_CELL) ?? null;
|
|
142
|
+
const figures = [...tallies.values()].map((t) => ({
|
|
143
|
+
cell: t.cell,
|
|
144
|
+
n_pulls: t.n,
|
|
145
|
+
n_merged: t.merged,
|
|
146
|
+
n_claims: t.claims,
|
|
147
|
+
n_failed: t.failed,
|
|
148
|
+
fail_rate: t.claims ? round(t.failed / t.claims) : null,
|
|
149
|
+
mean_usd: round(meanUsd(t), 2),
|
|
150
|
+
median_usd: round(median(countedIn(t.cell).filter((u) => u.merged).map((u) => u.spend)), 2),
|
|
151
|
+
router_usd: round(countedIn(t.cell).reduce((s, u) => s + routerUsd(u.pull), 0), 4),
|
|
152
|
+
usd_diff_ci95: t.cell === BASELINE_CELL || !base ? null : bootstrapDiffCI(bootUnits(t.cell), bootUnits(BASELINE_CELL)),
|
|
153
|
+
excluded: t.excluded,
|
|
154
|
+
left_out: t.left_out,
|
|
155
|
+
unpriced: t.unpriced,
|
|
290
156
|
}));
|
|
291
157
|
|
|
292
158
|
// The guard.
|
|
293
|
-
const
|
|
294
|
-
const b = byCell.get(BASELINE_CELL) ?? { n_merged: 0, n_pass: 0, mean_usd: null, fail_rate: null };
|
|
159
|
+
const b = figures.find((f) => f.cell === BASELINE_CELL) ?? { n_merged: 0, mean_usd: null, fail_rate: null };
|
|
295
160
|
const baseline = { cell: BASELINE_CELL, n: b.n_merged, mean_usd: b.mean_usd, fail_rate: b.fail_rate };
|
|
161
|
+
// The $ leg takes the unrounded means: the cent-rounded figures are for
|
|
162
|
+
// display, and a trip is sticky, so a cell cheaper by under half a cent
|
|
163
|
+
// must not trip on rounding alone. The fail leg reads the claims.
|
|
164
|
+
const quality = (t) => ({ n: t.claims, n_pass: t.claims - t.failed, mean_usd: meanUsd(t) });
|
|
296
165
|
const tripped = [];
|
|
297
166
|
if (baseline.n >= MIN_N) {
|
|
298
|
-
for (const
|
|
299
|
-
if (
|
|
300
|
-
// The $ leg takes the unrounded means: the cent-rounded figures are for
|
|
301
|
-
// display, and a trip is sticky, so a cell cheaper by under half a cent
|
|
302
|
-
// must not trip on rounding alone.
|
|
303
|
-
const c = cells.get(f.cell);
|
|
304
|
-
if (trips({ n: f.n_merged, n_pass: f.n_pass, mean_usd: meanUsd(c) }, { n: b.n_merged, n_pass: b.n_pass, mean_usd: meanUsd(base) })) tripped.push(f.cell);
|
|
167
|
+
for (const t of tallies.values()) {
|
|
168
|
+
if (t.cell !== BASELINE_CELL && t.merged >= MIN_N && trips(quality(t), quality(base))) tripped.push(t.cell);
|
|
305
169
|
}
|
|
306
170
|
}
|
|
307
171
|
const verdict = baseline.n < MIN_N ? "none" : tripped.length ? "tripped" : "ok";
|
|
308
172
|
// Retire: every non-default stage-1 cell tripped before any stratum adopted one.
|
|
309
173
|
const adopted = Object.values(routerTable?.rows ?? {}).some((c) => c !== BASELINE_CELL);
|
|
310
174
|
const retire = !adopted && STAGE_1.filter((c) => c !== BASELINE_CELL).every((c) => tripped.includes(c));
|
|
311
|
-
// The member rows this report books: every windowed Pull's, and those of
|
|
312
|
-
// PR-named members whose PR a windowed ticket was ruled on.
|
|
313
|
-
const inReport = booked.filter(([, pull, pr]) => pull || groups.has(pr)).map(([r]) => r);
|
|
314
175
|
|
|
315
176
|
return {
|
|
316
177
|
window_start: windowStart,
|
|
@@ -319,44 +180,40 @@ export function computeReport({ members, tiers, features, prs, routerTable = nul
|
|
|
319
180
|
tripped,
|
|
320
181
|
verdict,
|
|
321
182
|
retire,
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
// A Pull whose member row never reached member-outcomes.tsv books $0 into
|
|
326
|
-
// its cell's mean, which reads as a cheap run unless it is listed here.
|
|
327
|
-
unbooked_pulls: pulls.filter((p) => !pullUsd.has(p)).map((p) => p.agent),
|
|
328
|
-
ab: abReport(groups, pulls, prState, pullUsd, prMemberUsd),
|
|
329
|
-
cross_check: crossCheck(inReport, pricing),
|
|
183
|
+
pending: [...new Set(units.filter((u) => u.status === "pending").map((u) => u.pendingPr))].sort((x, y) => Number(x) - Number(y)),
|
|
184
|
+
ab: abReport(units),
|
|
185
|
+
cross_check: crossCheck(units.flatMap((u) => u.rows), pricing),
|
|
330
186
|
};
|
|
331
187
|
}
|
|
332
188
|
|
|
333
189
|
// The reasoning test: arm A on an even ticket, B on an odd one. B has run only
|
|
334
|
-
// if some odd-ticket Pull carries a `sizing_pre`.
|
|
335
|
-
|
|
190
|
+
// if some odd-ticket Pull carries a `sizing_pre`. An arm's $ is its counted
|
|
191
|
+
// Pulls' spend plus their own `router_usd`, since the arms compare policies,
|
|
192
|
+
// not cells.
|
|
193
|
+
export function abReport(units) {
|
|
336
194
|
const arm = (ticket) => (Number(ticket) % 2 === 0 ? "A" : "B");
|
|
337
|
-
const bPulls =
|
|
195
|
+
const bPulls = units.map((u) => u.pull).filter((p) => arm(p.ticket) === "B");
|
|
338
196
|
if (!bPulls.some((p) => p.sizing_pre !== "")) {
|
|
339
197
|
return { status: "not-run", reason: "B not run — the table has no row a better classifier could change" };
|
|
340
198
|
}
|
|
341
|
-
const arms = { A: { merged: 0, failed: 0, usd: 0 }, B: { merged: 0, failed: 0, usd: 0 } };
|
|
342
|
-
for (const
|
|
343
|
-
if (
|
|
344
|
-
const a = arms[arm(
|
|
345
|
-
a.merged++;
|
|
346
|
-
|
|
347
|
-
a.
|
|
348
|
-
|
|
199
|
+
const arms = { A: { merged: 0, claims: 0, failed: 0, usd: 0 }, B: { merged: 0, claims: 0, failed: 0, usd: 0 } };
|
|
200
|
+
for (const u of units) {
|
|
201
|
+
if (u.status !== "counted") continue;
|
|
202
|
+
const a = arms[arm(u.pull.ticket)];
|
|
203
|
+
if (u.merged) a.merged++;
|
|
204
|
+
a.claims += u.claims;
|
|
205
|
+
a.failed += u.falseClaims;
|
|
206
|
+
a.usd += u.spend + routerUsd(u.pull);
|
|
349
207
|
}
|
|
350
208
|
const disagree = bPulls.filter((p) => p.policy_cell !== p.chosen_cell).length;
|
|
351
|
-
const
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
};
|
|
209
|
+
const side = (a) => ({
|
|
210
|
+
n_merged: a.merged, fail_rate: a.claims ? round(a.failed / a.claims) : null, mean_usd: a.merged ? round(a.usd / a.merged, 2) : null,
|
|
211
|
+
});
|
|
212
|
+
const out = { A: side(arms.A), B: side(arms.B), disagreement: bPulls.length ? round(disagree / bPulls.length) : null };
|
|
356
213
|
// B can only show an effect through the Pulls where it chose differently.
|
|
357
214
|
if (arms.A.merged < MIN_N || arms.B.merged < MIN_N) return { status: "insufficient", ...out };
|
|
358
215
|
if (disagree < MIN_N) return { status: "underpowered", ...out };
|
|
359
|
-
const bWins = out.B.mean_usd < out.A.mean_usd && arms.B.failed * arms.A.
|
|
216
|
+
const bWins = out.B.mean_usd < out.A.mean_usd && arms.B.failed * arms.A.claims <= arms.A.failed * arms.B.claims;
|
|
360
217
|
return { status: bWins ? "B-wins" : "B-off", ...out };
|
|
361
218
|
}
|
|
362
219
|
|
|
@@ -381,7 +238,10 @@ export function crossCheck(rows, pricing) {
|
|
|
381
238
|
return { rows: n, skipped, ratio: expected > 0 ? round(recorded / expected) : null };
|
|
382
239
|
}
|
|
383
240
|
|
|
384
|
-
const TSV_COLUMNS = [
|
|
241
|
+
const TSV_COLUMNS = [
|
|
242
|
+
"cell", "n_pulls", "n_merged", "n_claims", "n_failed", "fail_rate", "mean_usd", "median_usd", "router_usd", "usd_diff_ci95",
|
|
243
|
+
"excluded", "left_out", "unpriced", "guard",
|
|
244
|
+
];
|
|
385
245
|
|
|
386
246
|
export function formatReport(report) {
|
|
387
247
|
const guardOf = (f) => {
|
|
@@ -394,13 +254,7 @@ export function formatReport(report) {
|
|
|
394
254
|
for (const f of report.cells) lines.push(TSV_COLUMNS.map((c) => (c === "guard" ? guardOf(f) : cell(f[c]))).join("\t"));
|
|
395
255
|
const b = report.baseline;
|
|
396
256
|
lines.push(`# verdict: ${report.verdict} (baseline ${b.cell} n=${b.n}/${MIN_N}${report.tripped.length ? `; tripped ${report.tripped.join(" ")}` : ""})${report.retire ? " — retire: every non-default stage-1 cell tripped" : ""}`);
|
|
397
|
-
lines.push(`# window_start ${report.window_start}; pending PRs: ${report.pending.length ? report.pending.join(" ") : "none"}
|
|
398
|
-
if (report.mismatch.length) {
|
|
399
|
-
lines.push(`# mismatch (excluded from every cell): ${report.mismatch.map((m) => `PR#${m.pr} ${m.cell} vs ${m.subagent_type || "(none)"}/${m.effort || "(none)"}`).join("; ")}`);
|
|
400
|
-
}
|
|
401
|
-
if (report.unbooked_pulls.length) {
|
|
402
|
-
lines.push(`# unbooked Pulls (no member row, so $0 in the mean): ${report.unbooked_pulls.join(" ")}`);
|
|
403
|
-
}
|
|
257
|
+
lines.push(`# window_start ${report.window_start}; pending PRs: ${report.pending.length ? report.pending.join(" ") : "none"}`);
|
|
404
258
|
const ab = report.ab;
|
|
405
259
|
lines.push(ab.status === "not-run"
|
|
406
260
|
? `# A/B: ${ab.reason}`
|
|
@@ -493,7 +347,11 @@ function main() {
|
|
|
493
347
|
|
|
494
348
|
let report;
|
|
495
349
|
try { report = computeReport({ members, tiers, features, prs, routerTable, pricing }); }
|
|
496
|
-
catch (e) {
|
|
350
|
+
catch (e) {
|
|
351
|
+
if (e instanceof RulingError) die(`${tiersPath}: ${e.message}`);
|
|
352
|
+
if (e instanceof AttributionError) die(e.message);
|
|
353
|
+
throw e;
|
|
354
|
+
}
|
|
497
355
|
process.stdout.write(has("json") ? JSON.stringify(report, null, 2) + "\n" : formatReport(report));
|
|
498
356
|
if (!has("guard")) return;
|
|
499
357
|
const file = arg("out") ?? defaultGuardPath();
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
// The one booking of member rows, merges and claims to Pulls, read by both the
|
|
2
|
+
// cost guard (pr-cost.mjs) and the router's fit (ticket-router.mjs), so the two
|
|
3
|
+
// print the same figures for a cell. Neither CLI imports the other; both import
|
|
4
|
+
// this.
|
|
5
|
+
//
|
|
6
|
+
// THE UNIT is the Pull, a ticket-features.tsv row, booked AS ROUTED to its
|
|
7
|
+
// `chosen_cell`: its merge, its Ruling and all its spend. A ticket Pulled twice
|
|
8
|
+
// is two units. Nothing splits across cells.
|
|
9
|
+
//
|
|
10
|
+
// JOINS. Only omp member rows are read, and a member whose name's last segment
|
|
11
|
+
// starts with merge-bot (or mergebot), or is `memory` or `__advisor`, is booked
|
|
12
|
+
// nowhere: its cost does not vary with the cell.
|
|
13
|
+
// - An implementer row (member-outcomes' implementer leaf) joins the Pull of
|
|
14
|
+
// its `ticket` in its own session. A session with no Pull of that ticket
|
|
15
|
+
// (a claim resumed in a later run) joins the ticket's latest earlier Pull.
|
|
16
|
+
// - A review, fix or finisher row — any row whose name carries a PR — joins
|
|
17
|
+
// the Pull whose implementer rows carry that PR in `pr`.
|
|
18
|
+
// - A nested member books with its nearest booked ancestor.
|
|
19
|
+
//
|
|
20
|
+
// RULING. A Pull's candidate Ruling is `rulingFor` of its ticket and date. One
|
|
21
|
+
// PR's merge counts once: among the Pulls whose candidate names it, the Pull
|
|
22
|
+
// whose implementer rows carry the PR takes it, with the verdict of the latest
|
|
23
|
+
// such ruling row in file order. When none carries it, the last of them in
|
|
24
|
+
// file order takes it and is unpriced, since its PR's review and fix spend
|
|
25
|
+
// cannot be booked. The other candidates stay unruled.
|
|
26
|
+
//
|
|
27
|
+
// MERGED: the Pull holds a Ruling and, when PR states are given, its PR is
|
|
28
|
+
// MERGED. With PR states, a ruled PR that is OPEN or not in the list, or an
|
|
29
|
+
// unruled Pull whose implementer opened a PR still OPEN, is pending.
|
|
30
|
+
//
|
|
31
|
+
// STATUS, first that applies:
|
|
32
|
+
// excluded no Admissible row among the Pull's implementer rows;
|
|
33
|
+
// pending see above (only with PR states);
|
|
34
|
+
// left-out holds no claim (never merged, no `fixed` bail), and every
|
|
35
|
+
// implementer ending is the ticket's or neither's;
|
|
36
|
+
// unpriced a booked row's `cost` is blank, or the Pull took its Ruling
|
|
37
|
+
// without carrying its PR;
|
|
38
|
+
// counted every other Pull: the only ones in a cell's figures.
|
|
39
|
+
//
|
|
40
|
+
// ENDINGS come from member-outcomes' `ending` column. The cell's: `hollow`,
|
|
41
|
+
// `budget`, `bail:fixed-refuted`, `unknown`. The ticket's: every other
|
|
42
|
+
// `bail:<cause>`. Neither's: `provider`, `crash`, `abort`.
|
|
43
|
+
//
|
|
44
|
+
// CLAIMS. A merged Pull is one claim, false when its Ruling fails the floor
|
|
45
|
+
// (`minted_false_claim=yes` or `closed_own_ticket=no`). Each `bail:fixed` or
|
|
46
|
+
// `bail:fixed-refuted` ending is one claim, false when refuted. A cell's fail
|
|
47
|
+
// rate is its counted Pulls' false claims over their claims.
|
|
48
|
+
//
|
|
49
|
+
// SPEND is the recorded `cost` of every row booked to the Pull. A `cost` that
|
|
50
|
+
// is neither blank nor a number is refused, naming the row.
|
|
51
|
+
|
|
52
|
+
import { isDigits } from "./arg.mjs";
|
|
53
|
+
import { CELL } from "./ledger-grammar.mjs";
|
|
54
|
+
import { parseMemberName } from "./member-record.mjs";
|
|
55
|
+
import { isImplementer } from "./member-outcomes.mjs";
|
|
56
|
+
import { DATE, rulingFor, rulingsByTicket } from "./tier-outcomes.mjs";
|
|
57
|
+
|
|
58
|
+
// The ticket-features row, one per Pull, in this order.
|
|
59
|
+
export const FEATURE_COLUMNS = Object.freeze([
|
|
60
|
+
"run_date", "session", "agent", "ticket", "policy_cell", "chosen_cell", "exploration_draw",
|
|
61
|
+
"sizing_src", "sizing_pre", "router_usd", "brief_chars", "criteria", "comments", "age_days",
|
|
62
|
+
"paths", "test_paths", "xrefs", "kind",
|
|
63
|
+
]);
|
|
64
|
+
|
|
65
|
+
// ticket-features.tsv: one bare header line naming the columns, then rows.
|
|
66
|
+
// Read by header name; a row whose width differs from the header's is refused.
|
|
67
|
+
export function parseFeatures(text) {
|
|
68
|
+
const lines = String(text ?? "").split("\n").filter((l) => l.trim() && !l.startsWith("#"));
|
|
69
|
+
if (lines.length === 0) throw new Error("ticket-features.tsv has no header line");
|
|
70
|
+
const header = lines[0].split("\t");
|
|
71
|
+
const missing = FEATURE_COLUMNS.filter((c) => !header.includes(c));
|
|
72
|
+
if (missing.length) throw new Error(`ticket-features.tsv header lacks ${missing.join(", ")}`);
|
|
73
|
+
return lines.slice(1).map((l, i) => {
|
|
74
|
+
const cells = l.split("\t");
|
|
75
|
+
if (cells.length !== header.length) {
|
|
76
|
+
throw new Error(`ticket-features.tsv row ${i + 1}: ${cells.length} fields, expected ${header.length} — ${l.slice(0, 60)}`);
|
|
77
|
+
}
|
|
78
|
+
const row = Object.fromEntries(header.map((c, j) => [c, cells[j]]));
|
|
79
|
+
if (!isDigits(row.ticket)) throw new Error(`ticket-features.tsv row ${i + 1}: ticket ${JSON.stringify(row.ticket)} is not a number`);
|
|
80
|
+
if (!CELL.test(row.chosen_cell)) throw new Error(`ticket-features.tsv row ${i + 1}: chosen_cell ${JSON.stringify(row.chosen_cell)} is not a cell`);
|
|
81
|
+
return row;
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export const EXCLUDED_MEMBER = /^(?:merge-?bot|memory$|__advisor$)/i;
|
|
86
|
+
// omp nests a member's id as `<parent>/<parent>.<child>`: the child is the last segment after either separator.
|
|
87
|
+
export const lastSegment = (agent) => String(agent).split(/[/.]/).at(-1);
|
|
88
|
+
|
|
89
|
+
// An Admissible row for `cell`: dispatched as the cell's own definition, run at
|
|
90
|
+
// the cell's level, with a resolved model on record.
|
|
91
|
+
export function admissibleRow(row, cell) {
|
|
92
|
+
const level = CELL.exec(cell)?.[2] ?? null;
|
|
93
|
+
return level !== null && row.subagentType === `fleet-implementer-${cell}` && row.effort === level && !!row.model;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// A bad input row, as opposed to a bug: a caller that reports bad input catches this class alone.
|
|
97
|
+
export class AttributionError extends Error {}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The features rows with a YYYY-MM-DD `run_date`, the rows that can be placed
|
|
101
|
+
* against the window and a ruling. A blank `run_date` (route writes one for a
|
|
102
|
+
* session id with no date) is no Pull and is left out. Any other date is a
|
|
103
|
+
* hand-edit that cannot be placed and is refused: dropping it would shrink
|
|
104
|
+
* every figure unseen.
|
|
105
|
+
*/
|
|
106
|
+
function datedRows(features) {
|
|
107
|
+
return features.filter((r) => {
|
|
108
|
+
if (DATE.test(r.run_date)) return true;
|
|
109
|
+
if (r.run_date !== "") throw new AttributionError(`ticket #${r.ticket}: a ticket-features row's run_date is '${r.run_date}', expected YYYY-MM-DD or blank`);
|
|
110
|
+
return false;
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const CELL_ENDINGS = new Set(["hollow", "budget", "bail:fixed-refuted", "unknown"]);
|
|
115
|
+
const NEITHER_ENDINGS = new Set(["provider", "crash", "abort"]);
|
|
116
|
+
const ticketsEnding = (e) => e.startsWith("bail:") && !CELL_ENDINGS.has(e);
|
|
117
|
+
const failsFloor = (ruling) => ruling.minted_false_claim === "yes" || ruling.closed_own_ticket === "no";
|
|
118
|
+
// An omp session id opens with its ISO start time, so two of them sort in time order.
|
|
119
|
+
const STARTED = /^\d{4}-\d{2}-\d{2}T/;
|
|
120
|
+
const startedBefore = (pull, row) => (STARTED.test(pull.session) && STARTED.test(row.session) ? pull.session < row.session : pull.run_date <= row.run_date);
|
|
121
|
+
|
|
122
|
+
function costOf(row) {
|
|
123
|
+
const c = String(row.cost ?? "");
|
|
124
|
+
if (c === "") return null;
|
|
125
|
+
const n = Number(c);
|
|
126
|
+
if (c.trim() === "" || !Number.isFinite(n)) throw new AttributionError(`member-outcomes row session ${row.session} agent ${row.agent}: cost '${c}' is not a number`);
|
|
127
|
+
return n;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* One unit per ticket-features row dated on or after `windowStart` (every dated
|
|
132
|
+
* row when null), in file order: `{ pull, cell, stratum, status, merged, pr,
|
|
133
|
+
* ruling, pendingPr, claims, falseClaims, rows, spend }`. `prs` is `gh pr list`'s
|
|
134
|
+
* `{ number, state }` rows, or null when PR state is unknown (the fit). `stratum`
|
|
135
|
+
* maps a features row to its stratum. Throws `AttributionError` on a features
|
|
136
|
+
* row it cannot date or a booked `cost` it cannot read, and `RulingError` as
|
|
137
|
+
* `rulingFor` does.
|
|
138
|
+
*/
|
|
139
|
+
export function attribute({ features, members, tiers, prs = null, windowStart = null, stratum = () => null }) {
|
|
140
|
+
const key = (session, agent) => `${session}\0${agent}`;
|
|
141
|
+
const pulls = datedRows(features);
|
|
142
|
+
const bySessionTicket = new Map(), byTicket = new Map();
|
|
143
|
+
for (const p of pulls) {
|
|
144
|
+
const k = key(p.session, p.ticket);
|
|
145
|
+
(bySessionTicket.get(k) ?? bySessionTicket.set(k, []).get(k)).push(p);
|
|
146
|
+
(byTicket.get(p.ticket) ?? byTicket.set(p.ticket, []).get(p.ticket)).push(p);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const omp = members.filter((r) => r.harness === "omp" && !EXCLUDED_MEMBER.test(lastSegment(r.member)));
|
|
150
|
+
// Implementer rows to their Pull, and a PR-named member's key to its PR.
|
|
151
|
+
const implPull = new Map(), prOfKey = new Map();
|
|
152
|
+
const impls = new Map(pulls.map((p) => [p, []]));
|
|
153
|
+
for (const r of omp) {
|
|
154
|
+
if (isImplementer(r.member)) {
|
|
155
|
+
if (!r.ticket) continue;
|
|
156
|
+
const own = bySessionTicket.get(key(r.session, r.ticket));
|
|
157
|
+
const pull = own ? (own.find((p) => p.agent === r.agent) ?? own.at(-1)) : byTicket.get(r.ticket)?.findLast((p) => startedBefore(p, r));
|
|
158
|
+
if (!pull) continue;
|
|
159
|
+
implPull.set(key(r.session, r.agent), pull);
|
|
160
|
+
impls.get(pull).push(r);
|
|
161
|
+
} else {
|
|
162
|
+
const { pr } = parseMemberName(r.member);
|
|
163
|
+
if (pr) prOfKey.set(key(r.session, r.agent), pr);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
// Each PR to the last Pull, in file order, whose implementer rows carry it.
|
|
167
|
+
const carrier = new Map();
|
|
168
|
+
for (const p of pulls) for (const r of impls.get(p)) if (r.pr) carrier.set(r.pr, p);
|
|
169
|
+
|
|
170
|
+
// Every row books to its nearest booked ancestor, itself included.
|
|
171
|
+
const rows = new Map(pulls.map((p) => [p, []]));
|
|
172
|
+
for (const r of omp) {
|
|
173
|
+
const segs = String(r.agent).split("/");
|
|
174
|
+
for (let i = segs.length; i > 0; i--) {
|
|
175
|
+
const k = key(r.session, segs.slice(0, i).join("/"));
|
|
176
|
+
const pull = implPull.get(k) ?? carrier.get(prOfKey.get(k));
|
|
177
|
+
if (pull) rows.get(pull).push(r);
|
|
178
|
+
if (pull || prOfKey.has(k)) break;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// Rulings: each PR's merge to one Pull.
|
|
183
|
+
const inWindow = pulls.filter((p) => windowStart === null || p.run_date >= windowStart);
|
|
184
|
+
const rulings = rulingsByTicket(tiers);
|
|
185
|
+
const fileOrder = new Map(tiers.map((t, i) => [t, i]));
|
|
186
|
+
const candidates = new Map(); // pr -> [{ pull, ruling }]
|
|
187
|
+
for (const p of inWindow) {
|
|
188
|
+
const ruling = rulingFor(rulings, p.ticket, p.run_date);
|
|
189
|
+
if (ruling) (candidates.get(ruling.pr) ?? candidates.set(ruling.pr, []).get(ruling.pr)).push({ pull: p, ruling });
|
|
190
|
+
}
|
|
191
|
+
const ruled = new Map(); // pull -> { pr, ruling, carried }
|
|
192
|
+
for (const [pr, cands] of candidates) {
|
|
193
|
+
const ruling = cands.reduce((a, c) => (fileOrder.get(c.ruling) > fileOrder.get(a) ? c.ruling : a), cands[0].ruling);
|
|
194
|
+
const held = cands.find((c) => c.pull === carrier.get(pr));
|
|
195
|
+
ruled.set(held?.pull ?? cands.at(-1).pull, { pr, ruling, carried: !!held });
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
const state = prs && new Map(prs.map((p) => [String(p.number), p.state]));
|
|
199
|
+
return inWindow.map((pull) => {
|
|
200
|
+
const own = impls.get(pull);
|
|
201
|
+
const booked = rows.get(pull);
|
|
202
|
+
const r = ruled.get(pull) ?? null;
|
|
203
|
+
let merged = r !== null;
|
|
204
|
+
let pendingPr = null;
|
|
205
|
+
if (state) {
|
|
206
|
+
if (r) {
|
|
207
|
+
const s = state.get(r.pr);
|
|
208
|
+
if (s === undefined || s === "OPEN") pendingPr = r.pr;
|
|
209
|
+
merged = s === "MERGED";
|
|
210
|
+
} else pendingPr = own.find((m) => m.pr && state.get(m.pr) === "OPEN")?.pr ?? null;
|
|
211
|
+
}
|
|
212
|
+
let claims = 0, falseClaims = 0;
|
|
213
|
+
if (merged) {
|
|
214
|
+
claims++;
|
|
215
|
+
if (failsFloor(r.ruling)) falseClaims++;
|
|
216
|
+
}
|
|
217
|
+
for (const m of own) {
|
|
218
|
+
if (m.ending === "bail:fixed" || m.ending === "bail:fixed-refuted") claims++;
|
|
219
|
+
if (m.ending === "bail:fixed-refuted") falseClaims++;
|
|
220
|
+
}
|
|
221
|
+
let spend = 0, blank = false;
|
|
222
|
+
for (const m of booked) {
|
|
223
|
+
const c = costOf(m);
|
|
224
|
+
if (c === null) blank = true;
|
|
225
|
+
else spend += c;
|
|
226
|
+
}
|
|
227
|
+
let status = "counted";
|
|
228
|
+
if (!own.some((m) => admissibleRow(m, pull.chosen_cell))) status = "excluded";
|
|
229
|
+
else if (pendingPr !== null) status = "pending";
|
|
230
|
+
else if (claims === 0 && own.every((m) => ticketsEnding(m.ending) || NEITHER_ENDINGS.has(m.ending))) status = "left-out";
|
|
231
|
+
else if (blank || (r && !r.carried)) status = "unpriced";
|
|
232
|
+
return {
|
|
233
|
+
pull, cell: pull.chosen_cell, stratum: stratum(pull), status, merged, pr: r?.pr ?? null, ruling: r?.ruling ?? null,
|
|
234
|
+
pendingPr, claims, falseClaims, rows: booked, spend: blank ? null : spend,
|
|
235
|
+
};
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Per cell, in cell-name order: `{ cell, n, merged, claims, failed, spend,
|
|
241
|
+
* excluded, left_out, unpriced, pending }`. `n`, `merged`, `claims`, `failed`
|
|
242
|
+
* and `spend` are over counted Pulls alone; the last four count the Pulls of
|
|
243
|
+
* each other status.
|
|
244
|
+
*/
|
|
245
|
+
export function tally(units) {
|
|
246
|
+
const cells = new Map();
|
|
247
|
+
for (const u of units) {
|
|
248
|
+
const c = cells.get(u.cell) ?? cells.set(u.cell, {
|
|
249
|
+
cell: u.cell, n: 0, merged: 0, claims: 0, failed: 0, spend: 0, excluded: 0, left_out: 0, unpriced: 0, pending: 0,
|
|
250
|
+
}).get(u.cell);
|
|
251
|
+
if (u.status === "counted") {
|
|
252
|
+
c.n++;
|
|
253
|
+
if (u.merged) c.merged++;
|
|
254
|
+
c.claims += u.claims;
|
|
255
|
+
c.failed += u.falseClaims;
|
|
256
|
+
c.spend += u.spend;
|
|
257
|
+
} else c[u.status.replace("-", "_")]++;
|
|
258
|
+
}
|
|
259
|
+
return [...cells.values()].sort((a, b) => (a.cell < b.cell ? -1 : a.cell > b.cell ? 1 : 0));
|
|
260
|
+
}
|
|
@@ -47,19 +47,13 @@ import { makeDie, defineFlags } from "./arg.mjs";
|
|
|
47
47
|
import { CELL, POLICY_CELL, drawCell, parseMember } from "./ledger-grammar.mjs";
|
|
48
48
|
import { sessionFromReceipt } from "./phase0-receipt.mjs";
|
|
49
49
|
import { parseTsv as parseMemberTsv } from "./member-outcomes.mjs";
|
|
50
|
-
import {
|
|
50
|
+
import { AttributionError, FEATURE_COLUMNS, attribute, parseFeatures, tally } from "./pull-attribution.mjs";
|
|
51
|
+
import { RulingError, parseTierOutcomes } from "./tier-outcomes.mjs";
|
|
51
52
|
|
|
52
53
|
const NAME = "ticket-router";
|
|
53
54
|
const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
|
|
54
55
|
export const DEFAULT_TABLE = join(SCRIPT_DIR, "router-table.json");
|
|
55
56
|
|
|
56
|
-
// The ticket-features row, one per Pull, in this order.
|
|
57
|
-
export const COLUMNS = Object.freeze([
|
|
58
|
-
"run_date", "session", "agent", "ticket", "policy_cell", "chosen_cell",
|
|
59
|
-
"exploration_draw", "sizing_src", "sizing_pre", "router_usd", "brief_chars",
|
|
60
|
-
"criteria", "comments", "age_days", "paths", "test_paths", "xrefs", "kind",
|
|
61
|
-
]);
|
|
62
|
-
|
|
63
57
|
export const STRATA = Object.freeze(["light", "heavy", "unknown"]);
|
|
64
58
|
// The free classifier's thresholds: heavy past the median brief length of
|
|
65
59
|
// the tickets whose sizing verdict is known, or past five criteria.
|
|
@@ -323,140 +317,52 @@ export function route({ session, ticket, arm, implRow, chainHead = false, issue,
|
|
|
323
317
|
}
|
|
324
318
|
|
|
325
319
|
export function formatFeatureRow(row) {
|
|
326
|
-
return
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
/** Data rows of a ticket-features TSV as objects; throws on a missing header or a short row. */
|
|
330
|
-
export function parseFeatures(text) {
|
|
331
|
-
const lines = String(text ?? "").split("\n").filter((l) => l.trim() && !l.startsWith("#"));
|
|
332
|
-
if (lines.length === 0) throw new Error("no header line");
|
|
333
|
-
if (lines[0] !== COLUMNS.join("\t")) throw new Error(`header is not the ${COLUMNS.length} ticket-features columns`);
|
|
334
|
-
return lines.slice(1).map((l) => {
|
|
335
|
-
const cells = l.split("\t");
|
|
336
|
-
if (cells.length !== COLUMNS.length) throw new Error(`malformed row: ${cells.length} fields, expected ${COLUMNS.length} — ${l.slice(0, 60)}`);
|
|
337
|
-
return Object.fromEntries(COLUMNS.map((c, i) => [c, cells[i]]));
|
|
338
|
-
});
|
|
320
|
+
return FEATURE_COLUMNS.map((c) => String(row[c] ?? "").replace(/[\t\r\n]/g, " ")).join("\t");
|
|
339
321
|
}
|
|
340
322
|
|
|
341
323
|
// ---------------------------------------------------------------------------
|
|
342
324
|
// fit
|
|
343
325
|
// ---------------------------------------------------------------------------
|
|
344
326
|
|
|
345
|
-
// Members whose cost does not vary with the implementer cell.
|
|
346
|
-
const UNBOOKED = (name) => name.startsWith("merge-bot-") || name === "memory" || name === "__advisor";
|
|
347
327
|
const round4 = (x) => Math.round(x * 1e4) / 1e4;
|
|
348
328
|
|
|
349
329
|
/**
|
|
350
|
-
* The
|
|
351
|
-
*
|
|
352
|
-
*
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
/**
|
|
360
|
-
* The features rows with a YYYY-MM-DD `run_date`, the rows that can be placed
|
|
361
|
-
* against the window, the cut and a ruling. A blank `run_date` (route writes one
|
|
362
|
-
* for a session id with no date) is no input row and is left out. Any other
|
|
363
|
-
* date is a hand-edit that cannot be placed and is refused, as `rulingFor`
|
|
364
|
-
* refuses a ruling's: dropping it would shrink the fit unseen.
|
|
330
|
+
* The fit's Pulls: pull-attribution.mjs's units over the window — booked as
|
|
331
|
+
* routed, each to its own row's cell and stratum — restricted to rows the free
|
|
332
|
+
* classifier routed plus exploration rows: a row a live B classifier routed is
|
|
333
|
+
* the A/B's test set, not the fit's. The fit sees no PR state, so a Pull with
|
|
334
|
+
* a Ruling is merged. A fit reads everything in the commit it is given: no row
|
|
335
|
+
* is cut by date, because `--check` re-reads that same commit rather than
|
|
336
|
+
* re-cutting the working tree's files.
|
|
365
337
|
*/
|
|
366
|
-
function
|
|
367
|
-
return
|
|
368
|
-
|
|
369
|
-
if (r.run_date !== "") throw new Error(`ticket #${r.ticket}: a ticket-features row's run_date is '${r.run_date}', expected YYYY-MM-DD or blank`);
|
|
370
|
-
return false;
|
|
371
|
-
});
|
|
338
|
+
function fitPulls({ features, members, verdicts, window }) {
|
|
339
|
+
return attribute({ features, members, tiers: verdicts, windowStart: window || null, stratum: ruleStratum })
|
|
340
|
+
.filter((u) => u.pull.exploration_draw !== "" || u.pull.sizing_src === "rule");
|
|
372
341
|
}
|
|
373
342
|
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
* routed is the A/B's test set, not the fit's. Its verdict rules its last
|
|
379
|
-
* input row. A row with a blank `run_date` (route writes one for a session id
|
|
380
|
-
* with no date) is no input row: it cannot be placed against the window or
|
|
381
|
-
* its ruling, and a window bound drops it anyway. A `run_date` that is
|
|
382
|
-
* neither blank nor YYYY-MM-DD is refused (see `datedRows`).
|
|
383
|
-
* A fit reads everything in the commit it is given: no row is cut by date,
|
|
384
|
-
* because `--check` re-reads that same commit rather than re-cutting the
|
|
385
|
-
* working tree's files.
|
|
386
|
-
*/
|
|
387
|
-
function fitTickets({ features, members, verdicts, window }) {
|
|
388
|
-
const inRange = datedRows(features).filter((r) => !window || r.run_date >= window);
|
|
389
|
-
const byTicket = new Map();
|
|
390
|
-
for (const r of inRange) {
|
|
391
|
-
if (!byTicket.has(r.ticket)) byTicket.set(r.ticket, []);
|
|
392
|
-
byTicket.get(r.ticket).push(r);
|
|
393
|
-
}
|
|
394
|
-
// `rulingFor` refuses a ruling whose date is not YYYY-MM-DD when its ticket
|
|
395
|
-
// is an input ticket. An input ticket is one the fit reads — its last
|
|
396
|
-
// in-window row is an exploration row or has `sizing_src` `rule`, which the
|
|
397
|
-
// skip below keeps — not every ticket with an in-window features row.
|
|
398
|
-
// `fit --due` also refuses for a ticket the skip drops: `mergedSince` calls
|
|
399
|
-
// `verdictOf` on every ticket with an in-window row the fitted commit does
|
|
400
|
-
// not hold, with no such skip.
|
|
401
|
-
const rulings = rulingsByTicket(verdicts);
|
|
402
|
-
const lastPulls = lastPullByTicket(inRange);
|
|
403
|
-
const out = [];
|
|
404
|
-
for (const [ticket, rows] of byTicket) {
|
|
405
|
-
const last = lastPulls.get(ticket);
|
|
406
|
-
if (last.exploration_draw === "" && last.sizing_src !== "rule") continue;
|
|
407
|
-
const verdict = verdictOf(rulings, ticket, last.run_date);
|
|
408
|
-
const keys = new Set(rows.map((r) => `${r.session}\0${r.agent}`));
|
|
409
|
-
let cost = 0;
|
|
410
|
-
let costKnown = true;
|
|
411
|
-
let booked = 0;
|
|
412
|
-
let fixRounds = 0;
|
|
413
|
-
for (const mr of members) {
|
|
414
|
-
const name = String(mr.member ?? mr.agent ?? "");
|
|
415
|
-
if (UNBOOKED(name)) continue;
|
|
416
|
-
if (!(keys.has(`${mr.session}\0${mr.agent}`) || mr.ticket === ticket || (verdict && verdict.pr && mr.pr === verdict.pr))) continue;
|
|
417
|
-
booked++;
|
|
418
|
-
// `cost` is a priced member-outcomes column; a row without it makes
|
|
419
|
-
// the ticket's $ unknown, so the fit adopts nothing off a TSV that lacks the column.
|
|
420
|
-
if (mr.cost === undefined || mr.cost === "" || !Number.isFinite(Number(mr.cost))) costKnown = false;
|
|
421
|
-
else cost += Number(mr.cost);
|
|
422
|
-
if (parseMember(name)?.family === "fix-pr") fixRounds++;
|
|
423
|
-
}
|
|
424
|
-
// A ticket with no member row on record has an unknown cost, not a free one.
|
|
425
|
-
if (booked === 0) costKnown = false;
|
|
426
|
-
for (const r of rows) if (r.router_usd !== "") cost += Number(r.router_usd) || 0;
|
|
427
|
-
out.push({ ticket, cell: last.chosen_cell, stratum: ruleStratum(last), verdict, cost, costKnown, fixRounds });
|
|
428
|
-
}
|
|
429
|
-
return out;
|
|
430
|
-
}
|
|
431
|
-
|
|
432
|
-
function estimate(tickets) {
|
|
433
|
-
const est = {};
|
|
434
|
-
for (const t of tickets) {
|
|
435
|
-
for (const s of [t.stratum, "*"]) {
|
|
436
|
-
est[s] ??= {};
|
|
437
|
-
const e = (est[s][t.cell] ??= { n: 0, merged: 0, fails: 0, cost: 0, costKnown: true, fix: 0 });
|
|
438
|
-
e.n++;
|
|
439
|
-
e.cost += t.cost;
|
|
440
|
-
e.costKnown &&= t.costKnown;
|
|
441
|
-
if (t.verdict) {
|
|
442
|
-
e.merged++;
|
|
443
|
-
if (t.verdict.fail) e.fails++;
|
|
444
|
-
e.fix += t.fixRounds;
|
|
445
|
-
}
|
|
446
|
-
}
|
|
447
|
-
}
|
|
343
|
+
// Per stratum and `*`, per cell: `n` counted Pulls, `merged`, $ per merged PR,
|
|
344
|
+
// false claims over claims, fix rounds per merged PR, and the cell's excluded,
|
|
345
|
+
// left-out and unpriced Pull counts.
|
|
346
|
+
function estimate(units) {
|
|
448
347
|
const out = {};
|
|
449
|
-
for (const s of
|
|
348
|
+
for (const s of units.length ? [...new Set([...units.map((u) => u.stratum), "*"])].sort() : []) {
|
|
349
|
+
const mine = s === "*" ? units : units.filter((u) => u.stratum === s);
|
|
450
350
|
out[s] = {};
|
|
451
|
-
for (const
|
|
452
|
-
const
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
351
|
+
for (const t of tally(mine)) {
|
|
352
|
+
const fix = mine.filter((u) => u.cell === t.cell && u.status === "counted" && u.merged)
|
|
353
|
+
.reduce((n, u) => n + u.rows.filter((r) => parseMember(r.member)?.family === "fix-pr").length, 0);
|
|
354
|
+
out[s][t.cell] = {
|
|
355
|
+
n: t.n,
|
|
356
|
+
merged: t.merged,
|
|
357
|
+
claims: t.claims,
|
|
358
|
+
failed: t.failed,
|
|
359
|
+
usd_per_merged: t.merged ? round4(t.spend / t.merged) : null,
|
|
360
|
+
fail_rate: t.claims ? round4(t.failed / t.claims) : null,
|
|
361
|
+
fix_rounds: t.merged ? round4(fix / t.merged) : null,
|
|
459
362
|
review_findings: null,
|
|
363
|
+
excluded: t.excluded,
|
|
364
|
+
left_out: t.left_out,
|
|
365
|
+
unpriced: t.unpriced,
|
|
460
366
|
};
|
|
461
367
|
}
|
|
462
368
|
}
|
|
@@ -487,8 +393,8 @@ function adopt(est, cells, tripped) {
|
|
|
487
393
|
*/
|
|
488
394
|
export function fitTable({ prior, features, members, verdicts, guard, commit = null }) {
|
|
489
395
|
const window = guard.window_start ?? prior.window_start ?? null;
|
|
490
|
-
const
|
|
491
|
-
const estimates = estimate(
|
|
396
|
+
const units = fitPulls({ features, members, verdicts, window });
|
|
397
|
+
const estimates = estimate(units);
|
|
492
398
|
const tripped = new Set(guard.tripped);
|
|
493
399
|
|
|
494
400
|
let { stage, cells } = prior;
|
|
@@ -508,7 +414,7 @@ export function fitTable({ prior, features, members, verdicts, guard, commit = n
|
|
|
508
414
|
return {
|
|
509
415
|
window_start: window,
|
|
510
416
|
fitted_commit: commit,
|
|
511
|
-
n_rows:
|
|
417
|
+
n_rows: units.length,
|
|
512
418
|
stage,
|
|
513
419
|
cells,
|
|
514
420
|
burn_in: !cells.every(atN),
|
|
@@ -520,16 +426,22 @@ export function fitTable({ prior, features, members, verdicts, guard, commit = n
|
|
|
520
426
|
}
|
|
521
427
|
|
|
522
428
|
/**
|
|
523
|
-
* Merged PRs
|
|
524
|
-
*
|
|
525
|
-
*
|
|
526
|
-
*
|
|
429
|
+
* Merged PRs since the fit: pull-attribution.mjs's counted, merged Pulls on or
|
|
430
|
+
* after `window_start` whose features row the fitted commit's features file
|
|
431
|
+
* (`fitted`, parsed rows; empty before any fit) does not hold, matched by
|
|
432
|
+
* `session` and `agent`.
|
|
527
433
|
*/
|
|
528
|
-
export function mergedSince({ table, features, verdicts, fitted }) {
|
|
529
|
-
const rulings = rulingsByTicket(verdicts);
|
|
434
|
+
export function mergedSince({ table, features, members, verdicts, fitted }) {
|
|
530
435
|
const read = new Set(fitted.map((r) => `${r.session}\0${r.agent}`));
|
|
531
|
-
|
|
532
|
-
|
|
436
|
+
return attribute({ features, members, tiers: verdicts, windowStart: table.window_start || null })
|
|
437
|
+
.filter((u) => u.status === "counted" && u.merged && !read.has(`${u.pull.session}\0${u.pull.agent}`)).length;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
// Each cell's Pull counts over every stratum, as the fit prints them.
|
|
441
|
+
function cellCounts(estimates) {
|
|
442
|
+
return Object.entries(estimates["*"] ?? {})
|
|
443
|
+
.map(([c, e]) => `${c} n=${e.n} merged=${e.merged} failed=${e.failed} excluded=${e.excluded} left_out=${e.left_out} unpriced=${e.unpriced}`)
|
|
444
|
+
.join(", ") || "none";
|
|
533
445
|
}
|
|
534
446
|
|
|
535
447
|
// ---------------------------------------------------------------------------
|
|
@@ -630,7 +542,7 @@ function main() {
|
|
|
630
542
|
// the rule, and neither this append nor the drain checks it.
|
|
631
543
|
try {
|
|
632
544
|
mkdirSync(dirname(pending), { recursive: true });
|
|
633
|
-
const header = statSync(pending, { throwIfNoEntry: false })?.size > 0 ? "" : `${
|
|
545
|
+
const header = statSync(pending, { throwIfNoEntry: false })?.size > 0 ? "" : `${FEATURE_COLUMNS.join("\t")}\n`;
|
|
634
546
|
appendFileSync(pending, `${header}${formatFeatureRow(row)}\n`);
|
|
635
547
|
} catch (e) {
|
|
636
548
|
process.stderr.write(`${NAME}: WARNING could not append to ${pending}: ${e.message} — this Pull's ticket-features row is not recorded\n`);
|
|
@@ -664,13 +576,15 @@ function main() {
|
|
|
664
576
|
}
|
|
665
577
|
return sha;
|
|
666
578
|
};
|
|
667
|
-
// The join throws on a features row it cannot
|
|
668
|
-
//
|
|
669
|
-
//
|
|
670
|
-
//
|
|
671
|
-
// row the fitted commit does not hold, with no such skip.
|
|
579
|
+
// The join throws on a features row it cannot date, a booked member row whose `cost` is not a number, and a
|
|
580
|
+
// ruling of a windowed Pull's ticket it cannot place — for `fit` and `--check` too on a Pull the fit then skips
|
|
581
|
+
// as a live B classifier's, since every windowed Pull is booked before that skip. Anything else it throws is a
|
|
582
|
+
// bug, never bad input, so it surfaces with its stack.
|
|
672
583
|
const joined = (fit) => {
|
|
673
|
-
try { return fit(); } catch (e) {
|
|
584
|
+
try { return fit(); } catch (e) {
|
|
585
|
+
if (!(e instanceof RulingError || e instanceof AttributionError)) throw e;
|
|
586
|
+
die(`cannot join --features ${need("features")}, --members ${need("members")} and --verdicts ${need("verdicts")}: ${e.message}`);
|
|
587
|
+
}
|
|
674
588
|
};
|
|
675
589
|
|
|
676
590
|
if (mode === "check") {
|
|
@@ -685,7 +599,7 @@ function main() {
|
|
|
685
599
|
process.exitCode = 1;
|
|
686
600
|
return;
|
|
687
601
|
}
|
|
688
|
-
process.stdout.write(`${NAME}: ${tablePath} matches a re-fit of ${refit.n_rows}
|
|
602
|
+
process.stdout.write(`${NAME}: ${tablePath} matches a re-fit of ${refit.n_rows} Pulls at ${short(fitted)}\n`);
|
|
689
603
|
return;
|
|
690
604
|
}
|
|
691
605
|
|
|
@@ -699,14 +613,14 @@ function main() {
|
|
|
699
613
|
if (F.has("due")) {
|
|
700
614
|
const fitted = fittedCommit();
|
|
701
615
|
const read = fitted === null ? [] : fileAt(fitted, "features", parseFeatures);
|
|
702
|
-
const n = joined(() => mergedSince({ table, features, verdicts, fitted: read }));
|
|
616
|
+
const n = joined(() => mergedSince({ table, features, members, verdicts, fitted: read }));
|
|
703
617
|
process.stdout.write(`DUE=${n >= FIT_EVERY_MERGED ? "yes" : "no"} MERGED=${n}/${FIT_EVERY_MERGED}\n`);
|
|
704
618
|
return;
|
|
705
619
|
}
|
|
706
620
|
const next = joined(() => fitTable({ prior: table, features, members, verdicts, guard, commit: sha }));
|
|
707
621
|
writeFileSync(`${tablePath}.tmp`, `${JSON.stringify(next, null, 2)}\n`);
|
|
708
622
|
renameSync(`${tablePath}.tmp`, tablePath);
|
|
709
|
-
process.stdout.write(`${NAME}: fitted ${next.n_rows}
|
|
623
|
+
process.stdout.write(`${NAME}: fitted ${next.n_rows} Pulls at ${short(sha)}; rows ${JSON.stringify(next.rows)}; burn_in ${next.burn_in}; stage ${next.stage}; cells ${cellCounts(next.estimates)}\n`);
|
|
710
624
|
}
|
|
711
625
|
|
|
712
626
|
if (isCLI(import.meta.url)) main();
|