@feigi/fleet-ctl 3.21.12 → 3.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/cell-readout.mjs +148 -0
- package/scripts/cell-readout.test.mjs +341 -0
- package/scripts/member-outcomes-header.test.mjs +45 -85
- package/scripts/member-outcomes.mjs +2 -2
- package/scripts/member-outcomes.test.mjs +3 -3
- package/scripts/tier-outcomes-header.test.mjs +37 -62
- package/skills/run-team/SKILL.md +31 -30
package/package.json
CHANGED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// The per-cell readout: how many within-session comparisons each implementer
|
|
3
|
+
// cell has against the policy cell, and whether that is enough to read it.
|
|
4
|
+
//
|
|
5
|
+
// cell-readout.mjs [--ticket-features <tsv>] [--member-outcomes <tsv>]
|
|
6
|
+
//
|
|
7
|
+
// Inputs default to docs/metrics/ under the working directory. A Pull is a
|
|
8
|
+
// ticket-features.tsv row; what it ran is the member-outcomes.tsv row with the
|
|
9
|
+
// same `session` + `agent`.
|
|
10
|
+
//
|
|
11
|
+
// ADMISSIBLE ROW. A Pull counts only when its member row exists, was
|
|
12
|
+
// dispatched as the drawn cell's own definition (`subagent_type` =
|
|
13
|
+
// `fleet-implementer-<chosen_cell>`), ran at the cell's level (`effort` = the
|
|
14
|
+
// cell's `<level>`) and has a resolved `model` on record. So a generic `task`
|
|
15
|
+
// dispatch, a clamped effort and every pre-cutover `fleet-implementer` /
|
|
16
|
+
// `fleet-implementer-alt` row are never admissible. Whether the resolved model
|
|
17
|
+
// was the role's target at dispatch is recorded in neither file; that half of
|
|
18
|
+
// the tier check is the run ledger's, not this readout's.
|
|
19
|
+
//
|
|
20
|
+
// COMPARISON. For a cell X other than slow-high, a comparison is one session
|
|
21
|
+
// holding at least one admissible X row and at least one admissible slow-high
|
|
22
|
+
// row whose resolved (`model`, `effort`) differ from that X row's. A session
|
|
23
|
+
// counts once however many rows it holds, and its date is its member-outcomes
|
|
24
|
+
// `run_date`; a blank `run_date` means unknown, so that session is still a
|
|
25
|
+
// comparison but adds no date to the distinct-date count.
|
|
26
|
+
//
|
|
27
|
+
// GATE. A cell is read only once it has at least GATE.comparisons comparisons
|
|
28
|
+
// across at least GATE.runDates distinct run_dates. A gated cell prints one
|
|
29
|
+
// line on stdout:
|
|
30
|
+
//
|
|
31
|
+
// <cell> <comparisons> <run_dates> <resolved models>
|
|
32
|
+
//
|
|
33
|
+
// `<resolved models>` is the model of every admissible row at the cell — one
|
|
34
|
+
// model by name, or `mixed (<model-a> n=…, <model-b> n=…)` when they span more
|
|
35
|
+
// than one, because a role's target is operator config that can move under a
|
|
36
|
+
// cell's history. A cell below the gate prints nothing on stdout and one
|
|
37
|
+
// `below the gate` count on stderr. Nothing on either stream names a session,
|
|
38
|
+
// ticket or member: the gate forbids reading a comparison early.
|
|
39
|
+
//
|
|
40
|
+
// Exit 0 whatever the counts; 2 on a usage error or an unreadable or
|
|
41
|
+
// malformed input, with nothing on stdout.
|
|
42
|
+
|
|
43
|
+
import { readFileSync } from "node:fs";
|
|
44
|
+
import { makeDie, defineFlags } from "./arg.mjs";
|
|
45
|
+
import { isCLI } from "./is-cli.mjs";
|
|
46
|
+
import { CELL, POLICY_CELL } from "./ledger-grammar.mjs";
|
|
47
|
+
import { parseTsv as parseMemberTsv } from "./member-outcomes.mjs";
|
|
48
|
+
import { parseFeatures } from "./pr-cost.mjs";
|
|
49
|
+
|
|
50
|
+
const NAME = "cell-readout";
|
|
51
|
+
export const GATE = Object.freeze({ comparisons: 10, runDates: 5 });
|
|
52
|
+
|
|
53
|
+
const key = (session, agent) => `${session}\0${agent}`;
|
|
54
|
+
const levelOf = (cell) => CELL.exec(cell)?.[2] ?? null;
|
|
55
|
+
|
|
56
|
+
/** The join's member row for a Pull when the Pull is admissible, else null. */
|
|
57
|
+
function admissibleMember(pull, members) {
|
|
58
|
+
const m = members.get(key(pull.session, pull.agent));
|
|
59
|
+
if (!m) return null;
|
|
60
|
+
const level = levelOf(pull.chosen_cell);
|
|
61
|
+
if (level === null || m.subagentType !== `fleet-implementer-${pull.chosen_cell}` || m.effort !== level || !m.model) return null;
|
|
62
|
+
return m;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* `cells`: one entry per cell other than the policy cell that has a
|
|
67
|
+
* ticket-features row, `{ cell, comparisons, runDates, dates, models, gated }`,
|
|
68
|
+
* sorted by cell. `dates` is the comparisons' distinct run_dates, sorted;
|
|
69
|
+
* `models` maps each resolved model of the cell's admissible rows to its row
|
|
70
|
+
* count, largest first. `unjoined` counts Pulls with no member row.
|
|
71
|
+
*/
|
|
72
|
+
export function readout({ features, members }) {
|
|
73
|
+
const byKey = new Map(members.map((m) => [key(m.session, m.agent), m]));
|
|
74
|
+
// One Pull per session+agent: a re-dispatch lands on the same cell and the
|
|
75
|
+
// same member row.
|
|
76
|
+
const pulls = new Map(features.map((p) => [key(p.session, p.agent), p]));
|
|
77
|
+
const sessions = new Map();
|
|
78
|
+
const cells = new Map();
|
|
79
|
+
let unjoined = 0;
|
|
80
|
+
for (const p of pulls.values()) {
|
|
81
|
+
if (p.chosen_cell !== POLICY_CELL && !cells.has(p.chosen_cell)) cells.set(p.chosen_cell, new Map());
|
|
82
|
+
if (!byKey.has(key(p.session, p.agent))) unjoined++;
|
|
83
|
+
const m = admissibleMember(p, byKey);
|
|
84
|
+
if (!m) continue;
|
|
85
|
+
if (p.chosen_cell !== POLICY_CELL) cells.get(p.chosen_cell).set(m.model, (cells.get(p.chosen_cell).get(m.model) ?? 0) + 1);
|
|
86
|
+
if (!sessions.has(p.session)) sessions.set(p.session, { runDate: m.run_date, rows: [] });
|
|
87
|
+
sessions.get(p.session).rows.push({ cell: p.chosen_cell, model: m.model, effort: m.effort });
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const out = [...cells].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)).map(([cell, modelCounts]) => {
|
|
91
|
+
let comparisons = 0;
|
|
92
|
+
const dates = new Set();
|
|
93
|
+
for (const { runDate, rows } of sessions.values()) {
|
|
94
|
+
const base = rows.filter((r) => r.cell === POLICY_CELL);
|
|
95
|
+
const compared = rows.some((x) => x.cell === cell && base.some((s) => s.model !== x.model || s.effort !== x.effort));
|
|
96
|
+
if (!compared) continue;
|
|
97
|
+
comparisons++;
|
|
98
|
+
if (runDate) dates.add(runDate);
|
|
99
|
+
}
|
|
100
|
+
const models = new Map([...modelCounts].sort(([a, n], [b, k]) => k - n || (a < b ? -1 : a > b ? 1 : 0)));
|
|
101
|
+
const runDates = dates.size;
|
|
102
|
+
return { cell, comparisons, runDates, dates: [...dates].sort(), models, gated: comparisons >= GATE.comparisons && runDates >= GATE.runDates };
|
|
103
|
+
});
|
|
104
|
+
return { cells: out, unjoined };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// Only a gated cell is formatted, and a comparison needs an admissible row at
|
|
108
|
+
// the cell, so `models` is never empty here.
|
|
109
|
+
function formatModels(models) {
|
|
110
|
+
if (models.size === 1) return [...models.keys()][0];
|
|
111
|
+
return `mixed (${[...models].map(([m, n]) => `${m} n=${n}`).join(", ")})`;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function main() {
|
|
115
|
+
const die = makeDie(NAME);
|
|
116
|
+
const { arg, sweep, stray } = defineFlags(die, {
|
|
117
|
+
flags: { "ticket-features": "value", "member-outcomes": "value" },
|
|
118
|
+
});
|
|
119
|
+
sweep();
|
|
120
|
+
stray();
|
|
121
|
+
const featuresPath = arg("ticket-features") ?? "docs/metrics/ticket-features.tsv";
|
|
122
|
+
const membersPath = arg("member-outcomes") ?? "docs/metrics/member-outcomes.tsv";
|
|
123
|
+
const load = (path, parse) => {
|
|
124
|
+
let text;
|
|
125
|
+
try { text = readFileSync(path, "utf8"); }
|
|
126
|
+
catch (e) { die(`cannot read ${path}: ${e.code ?? e.message}`); }
|
|
127
|
+
try { return parse(text); }
|
|
128
|
+
catch (e) { die(`${path}: ${e.message}`); }
|
|
129
|
+
};
|
|
130
|
+
const features = load(featuresPath, parseFeatures);
|
|
131
|
+
const members = load(membersPath, parseMemberTsv);
|
|
132
|
+
|
|
133
|
+
const { cells, unjoined } = readout({ features, members });
|
|
134
|
+
const lines = [];
|
|
135
|
+
const notes = [];
|
|
136
|
+
if (unjoined > 0) {
|
|
137
|
+
notes.push(`${unjoined} ticket-features row${unjoined === 1 ? " has" : "s have"} no member-outcomes row — never admissible`);
|
|
138
|
+
}
|
|
139
|
+
if (cells.length === 0) notes.push(`no cell other than ${POLICY_CELL} has a ticket-features row`);
|
|
140
|
+
for (const c of cells) {
|
|
141
|
+
if (c.gated) lines.push(`${c.cell} ${c.comparisons} ${c.runDates} ${formatModels(c.models)}`);
|
|
142
|
+
else notes.push(`${c.cell} below the gate: ${c.comparisons} comparisons across ${c.runDates} run_dates (needs ${GATE.comparisons} across ${GATE.runDates})`);
|
|
143
|
+
}
|
|
144
|
+
for (const n of notes) process.stderr.write(`${NAME}: ${n}\n`);
|
|
145
|
+
if (lines.length) process.stdout.write(`${lines.join("\n")}\n`);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
if (isCLI(import.meta.url)) main();
|
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
// cell-readout.mjs: the per-cell gate over ticket-features.tsv joined to
|
|
2
|
+
// member-outcomes.tsv, run as the CLI against synthetic fixture files.
|
|
3
|
+
import { test } from "node:test";
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import { spawnSync } from "node:child_process";
|
|
6
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
7
|
+
import { dirname, join } from "node:path";
|
|
8
|
+
import { fileURLToPath } from "node:url";
|
|
9
|
+
import { tempDir } from "./temp-dir.mjs";
|
|
10
|
+
import { paragraph, phrase, unemphasized } from "./prose-pin.mjs";
|
|
11
|
+
import { formatTsv } from "./member-outcomes.mjs";
|
|
12
|
+
import { FEATURE_COLUMNS } from "./pr-cost.mjs";
|
|
13
|
+
import { readout, GATE } from "./cell-readout.mjs";
|
|
14
|
+
|
|
15
|
+
const SCRIPT = fileURLToPath(new URL("./cell-readout.mjs", import.meta.url));
|
|
16
|
+
|
|
17
|
+
const member = (o) => ({
|
|
18
|
+
run_date: "2026-10-01", role: "implementer", member: o.agent, model: "claude-opus-5", effort: "high",
|
|
19
|
+
ticket: "", pr: "", tokensCacheCreate: 0, tokensOut: 0, wallS: 0, turns: 1, harness: "omp",
|
|
20
|
+
tokensIn: 0, tokensCacheRead: 0, tokensCacheWrite1h: 0, cost: 0, ...o,
|
|
21
|
+
});
|
|
22
|
+
const pull = (o) => ({
|
|
23
|
+
run_date: "2026-10-01", policy_cell: "slow-high", exploration_draw: "", sizing_src: "rule", sizing_pre: "",
|
|
24
|
+
router_usd: "", brief_chars: "100", criteria: "1", comments: "0", age_days: "0", paths: "0",
|
|
25
|
+
test_paths: "0", xrefs: "0", kind: "enhancement", ...o,
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
function world() {
|
|
29
|
+
return { features: [], members: [], ticket: 100 };
|
|
30
|
+
}
|
|
31
|
+
// One implementer Pull at `cell` in `session`, and the member row it left.
|
|
32
|
+
// `model`/`effort`/`subagentType` override what resolved and what was dispatched.
|
|
33
|
+
function addRow(w, { session, date, cell, model, effort, subagentType, member: withMember = true }) {
|
|
34
|
+
const ticket = String(w.ticket++);
|
|
35
|
+
const agent = `impl-${ticket}`;
|
|
36
|
+
const level = cell.split("-").slice(1).join("-");
|
|
37
|
+
w.features.push(pull({ session, agent, ticket, chosen_cell: cell }));
|
|
38
|
+
if (withMember) {
|
|
39
|
+
w.members.push(member({
|
|
40
|
+
session, agent, run_date: date, ticket,
|
|
41
|
+
model: model ?? (cell.startsWith("slow-") ? "claude-opus-5" : "claude-sonnet-5"),
|
|
42
|
+
effort: effort ?? level, subagentType: subagentType ?? `fleet-implementer-${cell}`,
|
|
43
|
+
}));
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
// `n` sessions, each one comparison for `cell`: a `cell` row beside a slow-high
|
|
47
|
+
// row, spread round-robin over `dates` distinct run_dates.
|
|
48
|
+
let nextSession = 0;
|
|
49
|
+
function addComparisons(w, cell, n, dates, rowOpts = {}) {
|
|
50
|
+
for (let i = 0; i < n; i++) {
|
|
51
|
+
const session = `2026-10-01T00-00-00-000Z_s${nextSession++}`;
|
|
52
|
+
const date = `2026-10-${String(1 + (i % dates)).padStart(2, "0")}`;
|
|
53
|
+
addRow(w, { session, date, cell, ...rowOpts });
|
|
54
|
+
addRow(w, { session, date, cell: "slow-high" });
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function files(w) {
|
|
59
|
+
const dir = tempDir("cell-readout-");
|
|
60
|
+
const features = join(dir, "ticket-features.tsv");
|
|
61
|
+
const members = join(dir, "member-outcomes.tsv");
|
|
62
|
+
writeFileSync(features, [FEATURE_COLUMNS.join("\t"), ...w.features.map((r) => FEATURE_COLUMNS.map((c) => r[c] ?? "").join("\t"))].join("\n") + "\n");
|
|
63
|
+
writeFileSync(members, "# header\n" + formatTsv(w.members));
|
|
64
|
+
return { features, members, dir };
|
|
65
|
+
}
|
|
66
|
+
function cli(w, extra = []) {
|
|
67
|
+
const f = files(w);
|
|
68
|
+
return spawnSync(process.execPath, [SCRIPT, "--ticket-features", f.features, "--member-outcomes", f.members, ...extra], { encoding: "utf8", cwd: f.dir });
|
|
69
|
+
}
|
|
70
|
+
const lineFor = (stdout, cell) => stdout.split("\n").find((l) => l.split(" ")[0] === cell);
|
|
71
|
+
|
|
72
|
+
test("the gate is ten comparisons across five run_dates: nine across five does not gate, ten across five does", () => {
|
|
73
|
+
const nine = world();
|
|
74
|
+
addComparisons(nine, "task-high", 9, 5);
|
|
75
|
+
const below = cli(nine);
|
|
76
|
+
assert.equal(below.status, 0, below.stderr);
|
|
77
|
+
assert.equal(lineFor(below.stdout, "task-high"), undefined, "a cell below the gate prints no line");
|
|
78
|
+
assert.match(below.stderr, /task-high below the gate: 9 comparisons across 5 run_dates/);
|
|
79
|
+
|
|
80
|
+
const ten = world();
|
|
81
|
+
addComparisons(ten, "task-high", 10, 5);
|
|
82
|
+
const at = cli(ten);
|
|
83
|
+
assert.equal(at.status, 0, at.stderr);
|
|
84
|
+
assert.equal(lineFor(at.stdout, "task-high"), "task-high 10 5 claude-sonnet-5");
|
|
85
|
+
assert.doesNotMatch(at.stderr, /task-high below the gate/);
|
|
86
|
+
assert.deepEqual(GATE, { comparisons: 10, runDates: 5 });
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("ten comparisons across four run_dates does not gate: both halves of the gate bind", () => {
|
|
90
|
+
const w = world();
|
|
91
|
+
addComparisons(w, "task-high", 12, 4);
|
|
92
|
+
const r = cli(w);
|
|
93
|
+
assert.equal(r.status, 0, r.stderr);
|
|
94
|
+
assert.equal(lineFor(r.stdout, "task-high"), undefined);
|
|
95
|
+
assert.match(r.stderr, /task-high below the gate: 12 comparisons across 4 run_dates/);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
test("two resolved models in one cell's gate window print `mixed (...)` with each model's row count", () => {
|
|
99
|
+
const w = world();
|
|
100
|
+
addComparisons(w, "task-high", 7, 5);
|
|
101
|
+
addComparisons(w, "task-high", 3, 3, { model: "claude-opus-5" });
|
|
102
|
+
const r = cli(w);
|
|
103
|
+
assert.equal(r.status, 0, r.stderr);
|
|
104
|
+
// claude-opus-5 at task-high/high still differs from slow-high only if its
|
|
105
|
+
// effort does — it does not, so those three sessions are no comparison.
|
|
106
|
+
assert.match(r.stderr, /task-high below the gate: 7 comparisons across 5 run_dates/);
|
|
107
|
+
|
|
108
|
+
const gated = world();
|
|
109
|
+
addComparisons(gated, "task-high", 10, 5);
|
|
110
|
+
addComparisons(gated, "task-high", 2, 1, { model: "claude-haiku-4-5" });
|
|
111
|
+
const g = cli(gated);
|
|
112
|
+
assert.equal(g.status, 0, g.stderr);
|
|
113
|
+
assert.equal(lineFor(g.stdout, "task-high"), "task-high 12 5 mixed (claude-sonnet-5 n=10, claude-haiku-4-5 n=2)");
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
test("a comparison is one session, however many rows it holds, and a row on each side is required", () => {
|
|
117
|
+
const w = world();
|
|
118
|
+
// One session, three task-high rows and two slow-high rows: one comparison.
|
|
119
|
+
addRow(w, { session: "sA", date: "2026-10-01", cell: "task-high" });
|
|
120
|
+
addRow(w, { session: "sA", date: "2026-10-01", cell: "task-high" });
|
|
121
|
+
addRow(w, { session: "sA", date: "2026-10-01", cell: "task-high" });
|
|
122
|
+
addRow(w, { session: "sA", date: "2026-10-01", cell: "slow-high" });
|
|
123
|
+
addRow(w, { session: "sA", date: "2026-10-01", cell: "slow-high" });
|
|
124
|
+
// task-high with no slow-high beside it, and slow-high alone: no comparison.
|
|
125
|
+
addRow(w, { session: "sB", date: "2026-10-02", cell: "task-high" });
|
|
126
|
+
addRow(w, { session: "sC", date: "2026-10-03", cell: "slow-high" });
|
|
127
|
+
// A pair split across two sessions is no comparison either.
|
|
128
|
+
addRow(w, { session: "sD", date: "2026-10-04", cell: "smol-high" });
|
|
129
|
+
addRow(w, { session: "sE", date: "2026-10-04", cell: "slow-high" });
|
|
130
|
+
const [task, smol] = ["task-high", "smol-high"].map((c) => readout(parsed(w)).cells.find((x) => x.cell === c));
|
|
131
|
+
assert.deepEqual([task.comparisons, task.runDates], [1, 1]);
|
|
132
|
+
assert.deepEqual([smol.comparisons, smol.runDates], [0, 0]);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
test("only admissible rows count: dispatched as the drawn cell's definition, at the cell's level, with a model on record", () => {
|
|
136
|
+
const w = world();
|
|
137
|
+
const s = (n) => `s-adm-${n}`;
|
|
138
|
+
// effort off the cell's level (a clamp): inadmissible
|
|
139
|
+
addRow(w, { session: s(1), date: "2026-10-01", cell: "task-high", effort: "medium" });
|
|
140
|
+
addRow(w, { session: s(1), date: "2026-10-01", cell: "slow-high" });
|
|
141
|
+
// dispatched as a generic task, not the cell's definition: inadmissible
|
|
142
|
+
addRow(w, { session: s(2), date: "2026-10-01", cell: "task-high", subagentType: "task" });
|
|
143
|
+
addRow(w, { session: s(2), date: "2026-10-01", cell: "slow-high" });
|
|
144
|
+
// the slow-high side dispatched as the pre-cutover definition: inadmissible
|
|
145
|
+
addRow(w, { session: s(3), date: "2026-10-01", cell: "task-high" });
|
|
146
|
+
addRow(w, { session: s(3), date: "2026-10-01", cell: "slow-high", subagentType: "fleet-implementer" });
|
|
147
|
+
// the slow-high side has no member row at all: no join, inadmissible
|
|
148
|
+
addRow(w, { session: s(4), date: "2026-10-01", cell: "task-high" });
|
|
149
|
+
addRow(w, { session: s(4), date: "2026-10-01", cell: "slow-high", member: false });
|
|
150
|
+
// blank model: the resolved model is unknown, so nothing can differ from it
|
|
151
|
+
addRow(w, { session: s(5), date: "2026-10-01", cell: "task-high", model: "" });
|
|
152
|
+
addRow(w, { session: s(5), date: "2026-10-01", cell: "slow-high" });
|
|
153
|
+
// the control: admissible on both sides
|
|
154
|
+
addRow(w, { session: s(6), date: "2026-10-01", cell: "task-high" });
|
|
155
|
+
addRow(w, { session: s(6), date: "2026-10-01", cell: "slow-high" });
|
|
156
|
+
const task = readout(parsed(w)).cells.find((x) => x.cell === "task-high");
|
|
157
|
+
assert.equal(task.comparisons, 1, "only the admissible session counts");
|
|
158
|
+
assert.deepEqual([...task.models], [["claude-sonnet-5", 3]], "the models column reads admissible rows only — sessions 3, 4 and 6");
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
test("a comparison needs a resolved (model, effort) that differs from slow-high's — the same model at a different level counts", () => {
|
|
162
|
+
const w = world();
|
|
163
|
+
// task-high resolved to slow-high's own model at the same level: empty comparison
|
|
164
|
+
addRow(w, { session: "sSame", date: "2026-10-01", cell: "task-high", model: "claude-opus-5" });
|
|
165
|
+
addRow(w, { session: "sSame", date: "2026-10-01", cell: "slow-high" });
|
|
166
|
+
// slow-medium: same model, a different effort — a real comparison
|
|
167
|
+
addRow(w, { session: "sEff", date: "2026-10-02", cell: "slow-medium" });
|
|
168
|
+
addRow(w, { session: "sEff", date: "2026-10-02", cell: "slow-high" });
|
|
169
|
+
const rows = readout(parsed(w)).cells;
|
|
170
|
+
assert.equal(rows.find((x) => x.cell === "task-high").comparisons, 0);
|
|
171
|
+
assert.equal(rows.find((x) => x.cell === "slow-medium").comparisons, 1);
|
|
172
|
+
assert.equal(rows.find((x) => x.cell === "slow-high"), undefined, "slow-high is the baseline, never a readout line");
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
test("a run_date is the comparison session's member-outcomes run_date, not the ticket-features one", () => {
|
|
176
|
+
const w = world();
|
|
177
|
+
addComparisons(w, "task-high", 10, 5);
|
|
178
|
+
for (const f of w.features) f.run_date = "2026-09-01";
|
|
179
|
+
const r = cli(w);
|
|
180
|
+
assert.equal(lineFor(r.stdout, "task-high"), "task-high 10 5 claude-sonnet-5");
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
test("a re-dispatched Pull is one Pull: two ticket-features rows for one session+agent count the member row once", () => {
|
|
184
|
+
const w = world();
|
|
185
|
+
addRow(w, { session: "sRe", date: "2026-10-01", cell: "task-high" });
|
|
186
|
+
addRow(w, { session: "sRe", date: "2026-10-01", cell: "slow-high" });
|
|
187
|
+
// The re-dispatch: the same Pull's features row again, on the same member row.
|
|
188
|
+
w.features.push({ ...w.features[0] });
|
|
189
|
+
// An unjoined Pull, likewise written twice.
|
|
190
|
+
addRow(w, { session: "sOrphan", date: "2026-10-01", cell: "smol-high", member: false });
|
|
191
|
+
w.features.push({ ...w.features[w.features.length - 1] });
|
|
192
|
+
const { cells, unjoined } = readout(parsed(w));
|
|
193
|
+
const task = cells.find((x) => x.cell === "task-high");
|
|
194
|
+
assert.equal(task.comparisons, 1);
|
|
195
|
+
assert.deepEqual([...task.models], [["claude-sonnet-5", 1]], "the member row behind a re-dispatched Pull is counted once");
|
|
196
|
+
assert.equal(unjoined, 1, "a re-dispatched Pull with no member row is one unjoined Pull");
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
test("a blank member-outcomes run_date is no date: its session is a comparison but adds nothing to the distinct-date count", () => {
|
|
200
|
+
const w = world();
|
|
201
|
+
addComparisons(w, "task-high", 4, 4);
|
|
202
|
+
for (let i = 0; i < 6; i++) {
|
|
203
|
+
const session = `sBlank${i}`;
|
|
204
|
+
addRow(w, { session, date: "", cell: "task-high" });
|
|
205
|
+
addRow(w, { session, date: "", cell: "slow-high" });
|
|
206
|
+
}
|
|
207
|
+
const task = readout(parsed(w)).cells.find((x) => x.cell === "task-high");
|
|
208
|
+
assert.equal(task.comparisons, 10);
|
|
209
|
+
assert.deepEqual([task.runDates, task.gated], [4, false], "ten comparisons over four real dates is below the five-date floor");
|
|
210
|
+
assert.ok(!task.dates.includes(""), "a blank run_date is listed as a date");
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
test("gated cells print sorted by cell name, not in ticket-features order", () => {
|
|
214
|
+
const w = world();
|
|
215
|
+
// Rows are added in reverse alphabetical order: task-high first, smol-high second.
|
|
216
|
+
addComparisons(w, "task-high", 10, 5);
|
|
217
|
+
addComparisons(w, "smol-high", 10, 5);
|
|
218
|
+
const r = cli(w);
|
|
219
|
+
assert.equal(r.status, 0, r.stderr);
|
|
220
|
+
assert.deepEqual(
|
|
221
|
+
r.stdout.split("\n").filter(Boolean),
|
|
222
|
+
["smol-high 10 5 claude-sonnet-5", "task-high 10 5 claude-sonnet-5"],
|
|
223
|
+
);
|
|
224
|
+
});
|
|
225
|
+
|
|
226
|
+
test("equal per-model counts in a mixed cell list models alphabetically", () => {
|
|
227
|
+
const w = world();
|
|
228
|
+
// Reverse-alphabetical insertion order: sonnet rows first, then haiku, five each.
|
|
229
|
+
addComparisons(w, "task-high", 5, 5, { model: "claude-sonnet-5" });
|
|
230
|
+
addComparisons(w, "task-high", 5, 5, { model: "claude-haiku-4-5" });
|
|
231
|
+
const r = cli(w);
|
|
232
|
+
assert.equal(r.status, 0, r.stderr);
|
|
233
|
+
assert.equal(lineFor(r.stdout, "task-high"), "task-high 10 5 mixed (claude-haiku-4-5 n=5, claude-sonnet-5 n=5)");
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
// run-team/SKILL.md sends the controller here for a cell's comparison count.
|
|
237
|
+
// The prose is the only thing that does, and nothing else reads it: renaming
|
|
238
|
+
// the script or pointing the floor back at another query leaves every test
|
|
239
|
+
// above green. Each slice starts at an anchor that must occur exactly once and
|
|
240
|
+
// ends at the paragraph's blank line, so a restatement elsewhere in the file
|
|
241
|
+
// cannot satisfy a pin on the paragraph that carries the rule.
|
|
242
|
+
const RUN_TEAM = readFileSync(join(import.meta.dirname, "..", "skills", "run-team", "SKILL.md"), "utf8");
|
|
243
|
+
|
|
244
|
+
test("SKILL.md's floor names the script that prints a cell's count, and that script exists", () => {
|
|
245
|
+
const floor = unemphasized(paragraph(RUN_TEAM, "**Report the count the per-cell readout prints", "run-team's per-cell floor"));
|
|
246
|
+
const named = /`~\/\.fleet\/bin\/fleet-run (\S+\.mjs)`/.exec(floor)?.[1];
|
|
247
|
+
assert.ok(named, "the floor no longer tells the controller which script to run");
|
|
248
|
+
assert.equal(named, "cell-readout.mjs");
|
|
249
|
+
assert.ok(existsSync(join(dirname(SCRIPT), named)), `${named} is not a script beside cell-readout.mjs`);
|
|
250
|
+
assert.match(floor, phrase("prints `<cell> <comparisons> <run_dates> <resolved models>` for each cell past that floor"));
|
|
251
|
+
assert.match(floor, phrase("a cell below it only as a count on stderr"));
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
test("the line SKILL.md says the readout prints is the line it prints: four fields, a cell below the floor on stderr only", () => {
|
|
255
|
+
const gated = world();
|
|
256
|
+
addComparisons(gated, "task-high", 10, 5);
|
|
257
|
+
addComparisons(gated, "smol-high", 2, 2);
|
|
258
|
+
const r = cli(gated);
|
|
259
|
+
assert.equal(r.status, 0, r.stderr);
|
|
260
|
+
assert.equal(lineFor(r.stdout, "task-high"), "task-high 10 5 claude-sonnet-5");
|
|
261
|
+
assert.equal(lineFor(r.stdout, "smol-high"), undefined, "a cell below the floor must not print on stdout");
|
|
262
|
+
assert.match(r.stderr, /smol-high below the gate: 2 comparisons/);
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
test("SKILL.md says the readout counts a pair only when the two rows ran a different model or effort, as the readout does", () => {
|
|
266
|
+
const deliberate = unemphasized(paragraph(RUN_TEAM, "**The readout counts DELIBERATE comparisons.**", "run-team's DELIBERATE paragraph"));
|
|
267
|
+
assert.match(deliberate, phrase("the two rows must actually have RUN a different model or effort"));
|
|
268
|
+
// Run, not read: the same model at the same level is no comparison, the same
|
|
269
|
+
// model at a different level is one, a different model at the same level is one.
|
|
270
|
+
const w = world();
|
|
271
|
+
addRow(w, { session: "sNone", date: "2026-10-01", cell: "task-high", model: "claude-opus-5" });
|
|
272
|
+
addRow(w, { session: "sNone", date: "2026-10-01", cell: "slow-high" });
|
|
273
|
+
addRow(w, { session: "sModel", date: "2026-10-02", cell: "task-high" });
|
|
274
|
+
addRow(w, { session: "sModel", date: "2026-10-02", cell: "slow-high" });
|
|
275
|
+
addRow(w, { session: "sEffort", date: "2026-10-03", cell: "slow-medium" });
|
|
276
|
+
addRow(w, { session: "sEffort", date: "2026-10-03", cell: "slow-high" });
|
|
277
|
+
const { cells } = readout(parsed(w));
|
|
278
|
+
const countOf = (cell) => cells.find((c) => c.cell === cell).comparisons;
|
|
279
|
+
assert.deepEqual([countOf("task-high"), countOf("slow-medium")], [1, 1]);
|
|
280
|
+
});
|
|
281
|
+
|
|
282
|
+
test("the output identifies no comparison: no session, ticket or agent appears on either stream", () => {
|
|
283
|
+
const w = world();
|
|
284
|
+
addComparisons(w, "task-high", 10, 5);
|
|
285
|
+
addComparisons(w, "smol-high", 3, 2);
|
|
286
|
+
addRow(w, { session: "sOrphan", date: "2026-10-01", cell: "task-high", member: false });
|
|
287
|
+
const r = cli(w);
|
|
288
|
+
assert.equal(r.status, 0, r.stderr);
|
|
289
|
+
const out = r.stdout + r.stderr;
|
|
290
|
+
for (const f of w.features) {
|
|
291
|
+
assert.ok(!out.includes(f.session), `output names session ${f.session}`);
|
|
292
|
+
assert.ok(!out.includes(f.agent), `output names agent ${f.agent}`);
|
|
293
|
+
}
|
|
294
|
+
assert.match(r.stderr, /1 ticket-features row has no member-outcomes row/);
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
test("an empty ticket-features.tsv prints no line and says why, exit 0", () => {
|
|
298
|
+
const r = cli(world());
|
|
299
|
+
assert.equal(r.status, 0, r.stderr);
|
|
300
|
+
assert.equal(r.stdout, "");
|
|
301
|
+
assert.match(r.stderr, /no cell other than slow-high has a ticket-features row/);
|
|
302
|
+
});
|
|
303
|
+
|
|
304
|
+
test("usage and input errors exit 2 and print nothing on stdout", () => {
|
|
305
|
+
const w = world();
|
|
306
|
+
addComparisons(w, "task-high", 1, 1);
|
|
307
|
+
const f = files(w);
|
|
308
|
+
for (const args of [
|
|
309
|
+
["--ticket-features", join(f.dir, "absent.tsv"), "--member-outcomes", f.members],
|
|
310
|
+
["--ticket-features", f.features, "--member-outcomes", join(f.dir, "absent.tsv")],
|
|
311
|
+
["--ticket-features", f.features, "--member-outcomes", f.members, "--bogus"],
|
|
312
|
+
]) {
|
|
313
|
+
const r = spawnSync(process.execPath, [SCRIPT, ...args], { encoding: "utf8", cwd: f.dir });
|
|
314
|
+
assert.equal(r.status, 2, `${args.join(" ")}: ${r.stderr}`);
|
|
315
|
+
assert.equal(r.stdout, "");
|
|
316
|
+
}
|
|
317
|
+
writeFileSync(f.members, "# header\nshort\trow\n");
|
|
318
|
+
const bad = spawnSync(process.execPath, [SCRIPT, "--ticket-features", f.features, "--member-outcomes", f.members], { encoding: "utf8", cwd: f.dir });
|
|
319
|
+
assert.equal(bad.status, 2, bad.stderr);
|
|
320
|
+
assert.match(bad.stderr, /malformed row/);
|
|
321
|
+
});
|
|
322
|
+
|
|
323
|
+
test("with no flags it reads docs/metrics/ under the working directory", () => {
|
|
324
|
+
const w = world();
|
|
325
|
+
addComparisons(w, "task-high", 10, 5);
|
|
326
|
+
const f = files(w);
|
|
327
|
+
const metrics = join(f.dir, "docs", "metrics");
|
|
328
|
+
spawnSync("mkdir", ["-p", metrics]);
|
|
329
|
+
spawnSync("cp", [f.features, f.members, metrics]);
|
|
330
|
+
const r = spawnSync(process.execPath, [SCRIPT], { encoding: "utf8", cwd: f.dir });
|
|
331
|
+
assert.equal(r.status, 0, r.stderr);
|
|
332
|
+
assert.equal(lineFor(r.stdout, "task-high"), "task-high 10 5 claude-sonnet-5");
|
|
333
|
+
});
|
|
334
|
+
|
|
335
|
+
// The pure function's inputs, parsed the way the CLI parses them.
|
|
336
|
+
function parsed(w) {
|
|
337
|
+
return {
|
|
338
|
+
features: w.features.map((r) => ({ ...r })),
|
|
339
|
+
members: w.members.map((r) => ({ ...r })),
|
|
340
|
+
};
|
|
341
|
+
}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { test } from "node:test";
|
|
2
2
|
import assert from "node:assert/strict";
|
|
3
|
-
import {
|
|
4
|
-
import { tempDir } from "./temp-dir.mjs";
|
|
3
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
5
4
|
import { spawnSync } from "node:child_process";
|
|
6
5
|
import { join } from "node:path";
|
|
7
6
|
import { COLUMNS } from "./member-outcomes.mjs";
|
|
7
|
+
import { GATE } from "./cell-readout.mjs";
|
|
8
8
|
|
|
9
9
|
const REPO = join(import.meta.dirname, "..", "..");
|
|
10
10
|
const TSV = readFileSync(join(REPO, "docs", "metrics", "member-outcomes.tsv"), "utf8");
|
|
@@ -55,94 +55,54 @@ test("the header states the blank, hand-edit, and superseded-generation rules",
|
|
|
55
55
|
assert.match(HEADER, /workflows\/wf_/);
|
|
56
56
|
});
|
|
57
57
|
|
|
58
|
-
test("the header's awk read-
|
|
59
|
-
//
|
|
60
|
-
//
|
|
61
|
-
//
|
|
62
|
-
//
|
|
63
|
-
// and distinguish by effort with nothing to catch it.
|
|
64
|
-
//
|
|
65
|
-
// So read the `$n` references out of the header's own awk lines and resolve
|
|
66
|
-
// each against COLUMNS. Mutation this must survive: changing any `$n` in the
|
|
67
|
-
// header without changing COLUMNS.
|
|
68
|
-
const awkLines = headerLines.filter((l) => l.includes("awk -F"));
|
|
69
|
-
assert.ok(awkLines.length >= 2, "header lost its awk read-outs");
|
|
70
|
-
|
|
71
|
-
const pair = pairQuery();
|
|
72
|
-
// BOTH arms are identified by the dispatch record (#1066), never by the role
|
|
73
|
-
// classifier and never by "the models differ" — so both `$n ~ /…/` tests
|
|
74
|
-
// must land on subagent_type, and there must be two of them.
|
|
75
|
-
const arms = [...pair.matchAll(/\$(\d+) ~ \/\(\^\|:\)fleet-implementer(-alt)?\$\//g)];
|
|
76
|
-
assert.equal(arms.length, 2, "the pair query no longer tests BOTH implementer definitions");
|
|
77
|
-
assert.deepEqual(arms.map((m) => Boolean(m[2])), [true, false], "the -alt arm must be tested FIRST — /fleet-implementer$/ is checked in the else branch");
|
|
78
|
-
for (const [, n] of arms) assert.equal(COLUMNS[n - 1], "subagent_type", "deliberateness comes from the dispatch record");
|
|
79
|
-
|
|
80
|
-
const keys = [...pair.matchAll(/[at]\[\$(\d+) FS \$(\d+)\]/g)];
|
|
81
|
-
assert.equal(keys.length, 2, "the pair query no longer keys both arms");
|
|
82
|
-
for (const [, session, model] of keys) {
|
|
83
|
-
assert.equal(COLUMNS[session - 1], "session", "a pair is WITHIN one session");
|
|
84
|
-
assert.equal(COLUMNS[model - 1], "model", "the two arms must have run different MODELS");
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
const [, dateKey, dateValue] = /d\[\$(\d+)\]=\$(\d+)/.exec(pair);
|
|
88
|
-
assert.equal(COLUMNS[dateKey - 1], "session");
|
|
89
|
-
assert.equal(COLUMNS[dateValue - 1], "run_date", "the gate's second number is distinct run_dateS");
|
|
90
|
-
|
|
58
|
+
test("the header's awk read-out indexes the column it names", () => {
|
|
59
|
+
// Read the `$n` reference out of the header's own awk line and resolve it
|
|
60
|
+
// against COLUMNS — a fact about the CODE alone (COLUMNS[4]) stays green when
|
|
61
|
+
// the header is repointed. Mutation this must survive: changing the `$n` in
|
|
62
|
+
// the header without changing COLUMNS.
|
|
91
63
|
const byModel = headerLines.find((l) => /\{n\[\$\d+\]\+\+\}/.test(l));
|
|
92
64
|
assert.ok(byModel, "header lost its rows-by-model read-out");
|
|
93
65
|
assert.equal(COLUMNS[/\{n\[\$(\d+)\]\+\+\}/.exec(byModel)[1] - 1], "model");
|
|
94
66
|
});
|
|
95
67
|
|
|
96
|
-
// The
|
|
97
|
-
//
|
|
98
|
-
//
|
|
99
|
-
//
|
|
100
|
-
|
|
101
|
-
const
|
|
102
|
-
assert.ok(
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
assert.ok(
|
|
106
|
-
|
|
107
|
-
|
|
68
|
+
// The comparison count is cell-readout.mjs's, and the header only points at it:
|
|
69
|
+
// a second copy of the definition as a runnable query is two countings free to
|
|
70
|
+
// drift apart, which is how the pre-cell pair query came to count pairs no cell
|
|
71
|
+
// owns once the cell grid replaced the two definitions it keyed on.
|
|
72
|
+
test("the header points at cell-readout.mjs for a cell's comparisons and carries no query of its own", () => {
|
|
73
|
+
const invocation = headerLines.find((l) => /^#\s+node \S*cell-readout\.mjs\s*$/.test(l));
|
|
74
|
+
assert.ok(invocation, "header lost its `node plugin/scripts/cell-readout.mjs` pointer");
|
|
75
|
+
const script = invocation.replace(/^#\s+node\s+/, "").trim();
|
|
76
|
+
assert.equal(script, "plugin/scripts/cell-readout.mjs");
|
|
77
|
+
assert.ok(existsSync(join(REPO, script)), `the header points at ${script}, which does not exist`);
|
|
78
|
+
// The pointer is runnable as written, from the repo root, against this
|
|
79
|
+
// repo's own metrics files.
|
|
80
|
+
const r = spawnSync(process.execPath, [script], { cwd: REPO, encoding: "utf8" });
|
|
81
|
+
assert.equal(r.status, 0, r.stderr);
|
|
108
82
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
// which leave the column indexes (and therefore the test above) intact:
|
|
116
|
-
// dropping the `x[2]!=y[2]` model test reads 4 2, dropping the `-alt` anchor
|
|
117
|
-
// so both arms match one definition reads 0 0, and going back to #1066's own
|
|
118
|
-
// query — arms identified by `$3=="implementer"` rather than by the dispatch
|
|
119
|
-
// — reads 4 2, counting the untyped session and missing the omp one.
|
|
120
|
-
const rows = [
|
|
121
|
-
// a real pair: alt at sonnet against the top definition at opus
|
|
122
|
-
["sA", "2026-09-01", "implementer", "impl-1", "claude-sonnet-5", "high", "1", "", "0", "0", "0", "1", "agent-a1", "claude", "fleet-implementer-alt"],
|
|
123
|
-
["sA", "2026-09-01", "implementer", "impl-2", "claude-opus-5", "high", "2", "", "0", "0", "0", "1", "agent-a2", "claude", "fleet-implementer"],
|
|
124
|
-
// deliberate dispatch, empty comparison: both arms resolved to one model
|
|
125
|
-
["sB", "2026-09-01", "other", "impl-3", "claude-sonnet-5", "high", "3", "", "0", "0", "0", "1", "agent-a3", "omp", "fleet-implementer-alt"],
|
|
126
|
-
["sB", "2026-09-01", "other", "impl-4", "claude-sonnet-5", "high", "4", "", "0", "0", "0", "1", "agent-a4", "omp", "fleet-implementer"],
|
|
127
|
-
// two models, no deliberate dispatch: the accidental pair #1066 counted
|
|
128
|
-
["sC", "2026-09-01", "implementer", "impl-5", "claude-sonnet-5", "high", "5", "", "0", "0", "0", "1", "agent-a5", "claude", ""],
|
|
129
|
-
["sC", "2026-09-01", "implementer", "impl-6", "claude-opus-5", "high", "6", "", "0", "0", "0", "1", "agent-a6", "claude", ""],
|
|
130
|
-
// a second pair on the SAME day — two pairs, one run_date
|
|
131
|
-
["sD", "2026-09-01", "implementer", "impl-7", "claude-sonnet-5", "high", "7", "", "0", "0", "0", "1", "agent-a7", "claude", "fleet-implementer-alt"],
|
|
132
|
-
["sD", "2026-09-01", "implementer", "impl-8", "claude-opus-5", "high", "8", "", "0", "0", "0", "1", "agent-a8", "claude", "fleet-implementer"],
|
|
133
|
-
// and one on another day, so the second number is not the first
|
|
134
|
-
["sE", "2026-09-02", "implementer", "impl-9", "claude-sonnet-5", "high", "9", "", "0", "0", "0", "1", "agent-a9", "claude", "fleet-implementer-alt"],
|
|
135
|
-
["sE", "2026-09-02", "implementer", "impl-10", "claude-opus-5", "high", "10", "", "0", "0", "0", "1", "agent-a10", "claude", "fleet-implementer"],
|
|
136
|
-
// a typed non-implementer at a third model inside a pair session: the
|
|
137
|
-
// review fan-out runs one on every session and must not inflate anything
|
|
138
|
-
["sE", "2026-09-02", "specialist", "rev", "claude-haiku-4-5", "", "", "", "0", "0", "0", "1", "agent-a11", "claude", "fleet-review-tests"],
|
|
139
|
-
];
|
|
140
|
-
const corpus = join(tempDir("mo-hdr-"), "member-outcomes.tsv");
|
|
141
|
-
writeFileSync(corpus, "# header line, skipped by the query\n" + rows.map((r) => r.join("\t")).join("\n") + "\n");
|
|
83
|
+
assert.doesNotMatch(HEADER, /fleet-implementer-alt\$\//, "the header still carries the pre-cell pair query");
|
|
84
|
+
const subagentType = new RegExp(`\\$${COLUMNS.indexOf("subagent_type") + 1}(?!\\d)`);
|
|
85
|
+
assert.ok(
|
|
86
|
+
!headerLines.some((l) => subagentType.test(l)),
|
|
87
|
+
"an awk read-out over `subagent_type` is a second counting of comparisons",
|
|
88
|
+
);
|
|
142
89
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
assert.
|
|
147
|
-
assert.
|
|
90
|
+
// The definition the header states is the one the script applies.
|
|
91
|
+
const block = HEADER.replace(/^#\s?/gm, "").replace(/\s+/g, " ");
|
|
92
|
+
assert.match(block, /a comparison is one session holding an ADMISSIBLE row at X and an admissible slow-high row whose resolved \(`model`, `effort`\) differ from the X row's/);
|
|
93
|
+
assert.match(block, /admissible only when its `subagent_type` is `fleet-implementer-<chosen_cell>` and its `effort` is the cell's level/);
|
|
94
|
+
assert.match(block, /joined to its row here on `session`\+`agent`/);
|
|
95
|
+
const gate = /at least (\w+) comparisons across (\w+) or more distinct `run_date`s/.exec(block);
|
|
96
|
+
assert.ok(gate, "the header no longer states the gate");
|
|
97
|
+
const words = { five: 5, ten: 10 };
|
|
98
|
+
assert.deepEqual({ comparisons: words[gate[1]], runDates: words[gate[2]] }, { ...GATE }, "the header's gate is not cell-readout.mjs's GATE");
|
|
99
|
+
assert.match(block, /`<cell> <comparisons> <run_dates> <resolved models>`/);
|
|
100
|
+
assert.match(block, /`mixed \(<model-a> n=…, <model-b> n=…\)`/);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test("the header names a producer script that exists", () => {
|
|
104
|
+
const producer = /produced by (\S+\.mjs)/.exec(HEADER);
|
|
105
|
+
assert.ok(producer, "the header no longer names the script that produces its rows");
|
|
106
|
+
assert.ok(existsSync(join(REPO, producer[1])), `the header names ${producer[1]} as its producer, which does not exist`);
|
|
107
|
+
assert.equal(producer[1], "plugin/scripts/member-outcomes.mjs");
|
|
148
108
|
});
|
|
@@ -35,8 +35,8 @@ export { normalizeModel, parseMemberName };
|
|
|
35
35
|
//
|
|
36
36
|
// `subagentType` is the record's `subagent_type` unchanged — `""` stays
|
|
37
37
|
// `""` here, because unlike `effort` it does NOT mean unknown: it means the
|
|
38
|
-
// dispatch named no agent definition, which is the reading
|
|
39
|
-
//
|
|
38
|
+
// dispatch named no agent definition, which is the reading cell-readout.mjs's
|
|
39
|
+
// admissibility test depends on: a blank is never any cell's definition.
|
|
40
40
|
//
|
|
41
41
|
// A member is never torn in the sense this file historically tracked (no
|
|
42
42
|
// fold-back to tear mid-turn); `false` says so plainly rather than leaving
|
|
@@ -269,9 +269,9 @@ test("a nested member's fan-out is scraped too, keyed on its path-relative stem"
|
|
|
269
269
|
});
|
|
270
270
|
|
|
271
271
|
test("role comes from classifyRole and is not invented here", () => {
|
|
272
|
-
// `role` had no assertion anywhere
|
|
273
|
-
// pair query, so a classifyRole regression moved every
|
|
274
|
-
// while the suite stayed green.
|
|
272
|
+
// `role` had no assertion anywhere while it was the GROUPING column of the
|
|
273
|
+
// header's former pair query, so a classifyRole regression moved every
|
|
274
|
+
// bucket in that read-out while the suite stayed green.
|
|
275
275
|
const dir = fixture([
|
|
276
276
|
["impl-580", [sessionEvt("/x"), thinkingEvt("xhigh"), assistantEvt("claude-opus-5")]],
|
|
277
277
|
["merge-bot-3", [sessionEvt("/x"), thinkingEvt("xhigh"), sessionInitEvt("merge pass 3", "fleet-merge-bot"), assistantEvt("claude-opus-5")]],
|
|
@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
|
|
|
3
3
|
import { readFileSync } from "node:fs";
|
|
4
4
|
import { join } from "node:path";
|
|
5
5
|
import { between, phrase, sentences, stripHashGutter, unemphasized } from "./prose-pin.mjs";
|
|
6
|
-
import {
|
|
6
|
+
import { GATE, readout } from "./cell-readout.mjs";
|
|
7
7
|
import { COLUMNS } from "./tier-outcomes.mjs";
|
|
8
8
|
|
|
9
9
|
// #472: this file's header has never been pinned, while SKILL.md's guard tells
|
|
@@ -381,26 +381,20 @@ test("the ruling step refuses to record a sizing verdict it cannot date", () =>
|
|
|
381
381
|
});
|
|
382
382
|
|
|
383
383
|
// `run_date` is DERIVED in `member-outcomes.tsv` (a transcript mtime) and
|
|
384
|
-
// STAMPED AT RULING here, and phase 2's
|
|
385
|
-
// `run_date`s — ten
|
|
386
|
-
// one. Read as a count over THIS file, it answers a different question,
|
|
387
|
-
// nothing in this header used to say so.
|
|
384
|
+
// STAMPED AT RULING here, and phase 2's per-cell gate counts distinct
|
|
385
|
+
// `run_date`s — ten comparisons across five dates — over the other file, never
|
|
386
|
+
// this one. Read as a count over THIS file, it answers a different question,
|
|
387
|
+
// and nothing in this header used to say so.
|
|
388
388
|
//
|
|
389
389
|
// The paragraph that now says it makes claims about files this one does not
|
|
390
390
|
// contain, which is the shape that rots silently: it stays true only while
|
|
391
|
-
//
|
|
392
|
-
// keeps stamping dates off an mtime, and SKILL.md keeps
|
|
393
|
-
// So the pins below DERIVE each claim from its real
|
|
394
|
-
// the `profile` pin above derives the enum instead of
|
|
395
|
-
|
|
391
|
+
// cell-readout.mjs keeps dating comparisons off member-outcomes.tsv,
|
|
392
|
+
// `member-outcomes.mjs` keeps stamping dates off an mtime, and SKILL.md keeps
|
|
393
|
+
// the floor's numbers. So the pins below DERIVE each claim from its real
|
|
394
|
+
// source — the same reason the `profile` pin above derives the enum instead of
|
|
395
|
+
// restating it.
|
|
396
396
|
const MEMBER_SRC = readFileSync(join(REPO, "plugin", "scripts", "member-outcomes.mjs"), "utf8");
|
|
397
397
|
|
|
398
|
-
const memberHeaderLines = [];
|
|
399
|
-
for (const l of MEMBER_TSV.split("\n")) {
|
|
400
|
-
if (!l.startsWith("#") && l.trim()) break;
|
|
401
|
-
memberHeaderLines.push(l);
|
|
402
|
-
}
|
|
403
|
-
|
|
404
398
|
// Bounded at both ends, for `between`'s own stated reason: this header is 140
|
|
405
399
|
// lines about exactly these words, so an unbounded match is satisfiable from
|
|
406
400
|
// the provenance block above the paragraph or the column legend below it. The
|
|
@@ -418,9 +412,9 @@ test("the header says which file each distinct-`run_date` count is read from, an
|
|
|
418
412
|
assert.match(
|
|
419
413
|
block,
|
|
420
414
|
phrase(
|
|
421
|
-
"at least ten
|
|
415
|
+
"at least ten comparisons across five or more distinct `run_date`s, the floor that forbids reading a cell early — is NOT read here at all. It comes from `plugin/scripts/cell-readout.mjs`, which dates each comparison by its session's `run_date` in `docs/metrics/member-outcomes.tsv`",
|
|
422
416
|
),
|
|
423
|
-
"the header no longer says the
|
|
417
|
+
"the header no longer says the per-cell gate's distinct-date count is read from member-outcomes.tsv through cell-readout.mjs, not here",
|
|
424
418
|
);
|
|
425
419
|
|
|
426
420
|
// The definitional halves, each joined to what it makes the column MEAN. A
|
|
@@ -457,47 +451,27 @@ test("the header says which file each distinct-`run_date` count is read from, an
|
|
|
457
451
|
);
|
|
458
452
|
});
|
|
459
453
|
|
|
460
|
-
test("the header's `run_date` source claims still hold against the
|
|
454
|
+
test("the header's `run_date` source claims still hold against the readout, the scraper and the guard", () => {
|
|
461
455
|
const block = runDateSources();
|
|
462
456
|
|
|
463
|
-
// The
|
|
464
|
-
//
|
|
465
|
-
//
|
|
466
|
-
//
|
|
467
|
-
//
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
const
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
assert.
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
assert.ok(pairKey, "member-outcomes.tsv's header no longer carries the pairing query this header points at");
|
|
482
|
-
assert.equal(
|
|
483
|
-
MEMBER_COLUMNS[pairKey[2] - 1],
|
|
484
|
-
"run_date",
|
|
485
|
-
"the pairing query's distinct-date count is not member-outcomes.tsv's `run_date`",
|
|
486
|
-
);
|
|
487
|
-
assert.ok(
|
|
488
|
-
block.includes(pairKey[0]),
|
|
489
|
-
`the header cites an idiom for the pairing query's date key that member-outcomes.tsv no longer uses (\`${pairKey[0]}\`)`,
|
|
490
|
-
);
|
|
491
|
-
|
|
492
|
-
// The header also cites the query's distinct-date ACCUMULATOR
|
|
493
|
-
// (`length(r)`) — resolve the variable name against the query's own
|
|
494
|
-
// `length(...)` call rather than trusting the header's word for it.
|
|
495
|
-
const pairAccumulator = /print n\+0, length\((\w+)\)/.exec(pairQueryBlock);
|
|
496
|
-
assert.ok(pairAccumulator, "member-outcomes.tsv's pair query no longer prints its distinct-date count via `length(...)`");
|
|
497
|
-
assert.ok(
|
|
498
|
-
block.includes(`length(${pairAccumulator[1]})`),
|
|
499
|
-
`the header cites the wrong accumulator for the pairing query's distinct-date count (expected \`length(${pairAccumulator[1]})\`)`,
|
|
500
|
-
);
|
|
457
|
+
// The gate half: the count lives in cell-readout.mjs and dates each
|
|
458
|
+
// comparison off the OTHER file's `run_date`. Run, not read: one comparison
|
|
459
|
+
// whose ticket-features date and member-outcomes date differ, and the date
|
|
460
|
+
// the readout counts must be the member-outcomes one. If the readout ever
|
|
461
|
+
// keys its dates elsewhere, "it comes from there" stops being true.
|
|
462
|
+
const session = "s-tier-hdr";
|
|
463
|
+
const pull = (agent, chosen_cell) => ({ run_date: "2026-01-01", session, agent, chosen_cell });
|
|
464
|
+
const member = (agent, cell, model) => ({
|
|
465
|
+
session, agent, run_date: "2026-02-02", model, effort: "high", subagentType: `fleet-implementer-${cell}`,
|
|
466
|
+
});
|
|
467
|
+
const { cells: [task] } = readout({
|
|
468
|
+
features: [pull("impl-1", "task-high"), pull("impl-2", "slow-high")],
|
|
469
|
+
members: [member("impl-1", "task-high", "m-task"), member("impl-2", "slow-high", "m-slow")],
|
|
470
|
+
});
|
|
471
|
+
assert.deepEqual([task.cell, task.comparisons, task.runDates], ["task-high", 1, 1]);
|
|
472
|
+
assert.deepEqual([...task.dates], ["2026-02-02"], "cell-readout.mjs no longer dates a comparison by member-outcomes.tsv's `run_date`");
|
|
473
|
+
assert.match(block, /`plugin\/scripts\/cell-readout\.mjs`/);
|
|
474
|
+
assert.match(block, /`docs\/metrics\/member-outcomes\.tsv`/);
|
|
501
475
|
|
|
502
476
|
// The DERIVED half as code, not prose: the date comes off a transcript
|
|
503
477
|
// mtime. A scraper switched to a clock read would make this header's
|
|
@@ -528,16 +502,17 @@ test("the header's `run_date` source claims still hold against the query, the sc
|
|
|
528
502
|
// writes to the day it ran.
|
|
529
503
|
|
|
530
504
|
// The floor's numbers, as the paragraph quotes them. It names figures it
|
|
531
|
-
// does not own; a threshold changed in SKILL.md
|
|
532
|
-
// quietly citing the old one.
|
|
505
|
+
// does not own; a threshold changed in SKILL.md or in the readout alone
|
|
506
|
+
// leaves this header quietly citing the old one.
|
|
533
507
|
assert.match(
|
|
534
508
|
unemphasized(RUN_TEAM),
|
|
535
509
|
phrase("there are at least ten of them across five or more distinct `run_date`s"),
|
|
536
|
-
"SKILL.md's
|
|
510
|
+
"SKILL.md's per-cell floor is no longer ten comparisons across five dates — the header quotes those numbers",
|
|
537
511
|
);
|
|
512
|
+
assert.deepEqual({ ...GATE }, { comparisons: 10, runDates: 5 }, "cell-readout.mjs's gate is no longer ten across five — the header quotes those numbers");
|
|
538
513
|
assert.match(
|
|
539
514
|
block,
|
|
540
|
-
phrase("at least ten
|
|
541
|
-
"the header no longer states the
|
|
515
|
+
phrase("at least ten comparisons across five or more"),
|
|
516
|
+
"the header no longer states the per-cell floor it attributes to the other file",
|
|
542
517
|
);
|
|
543
518
|
});
|
package/skills/run-team/SKILL.md
CHANGED
|
@@ -1125,31 +1125,29 @@ Pull is one member, so a rate counted per Pull is a rate counted per member —
|
|
|
1125
1125
|
one implementer in five, however the Pulls fall across the run — and the
|
|
1126
1126
|
ledger, not your memory, is what still holds the count after a compaction.
|
|
1127
1127
|
|
|
1128
|
-
**Do not label it anywhere — the dispatch record already carries it.**
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
definition
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
to the same model (a `modelRoles` override, measured on omp 2026-09-12)
|
|
1152
|
-
controls nothing.
|
|
1128
|
+
**Do not label it anywhere — the dispatch record already carries it.** A
|
|
1129
|
+
cell's comparisons are a query, and `cell-readout.mjs` is that query: it joins
|
|
1130
|
+
`docs/metrics/ticket-features.tsv`, whose `chosen_cell` names the cell the
|
|
1131
|
+
router chose, to `docs/metrics/member-outcomes.tsv` on `session`+`agent`, and
|
|
1132
|
+
admits a row only when its `subagent_type` is that cell's
|
|
1133
|
+
`fleet-implementer-<cell>` definition and its `effort` is the cell's level.
|
|
1134
|
+
That column is the record of what each member was dispatched AS, scraped like
|
|
1135
|
+
every other column, so it survives the file's regeneration and mislabels no
|
|
1136
|
+
historical row — the pre-cutover `fleet-implementer` and
|
|
1137
|
+
`fleet-implementer-alt` rows are no cell's, and the readout never admits them.
|
|
1138
|
+
A hand-set column would fail both of those tests, and one derived against
|
|
1139
|
+
today's declared tiers would mislabel every historical row.
|
|
1140
|
+
|
|
1141
|
+
**The readout counts DELIBERATE comparisons.** An earlier rule asked
|
|
1142
|
+
only for a `session`+`role` carrying more than one distinct `model`, which any
|
|
1143
|
+
session that happened to stage a routine ticket beside a correction one
|
|
1144
|
+
satisfies. Measured 2026-09-12 on one corpus: 198 keys as that query was
|
|
1145
|
+
literally written, 35 sessions across 21 `run_date`s once restricted to
|
|
1146
|
+
implementers, against 17 real pairs across 8 — so the gate below read itself
|
|
1147
|
+
past its floor on sessions where nothing had been controlled. Dispatch alone
|
|
1148
|
+
is not enough either: the two rows must actually have RUN a different model or
|
|
1149
|
+
effort, because a deliberate dispatch whose two sides resolve to the same model
|
|
1150
|
+
(a `modelRoles` override, measured on omp 2026-09-12) controls nothing.
|
|
1153
1151
|
|
|
1154
1152
|
**Why one in five within the run, not a week of one tier then a week of the other:**
|
|
1155
1153
|
tier would then be confounded with calendar date and therefore with prompt
|
|
@@ -1397,11 +1395,14 @@ pooled across cells, and stop until
|
|
|
1397
1395
|
there are at least ten of them across five or more distinct `run_date`s; below
|
|
1398
1396
|
that, a comparison count is a number, not evidence, and the last guard fired
|
|
1399
1397
|
with n=1 on the control side. **Report the count the per-cell readout prints,
|
|
1400
|
-
never one from any other query:**
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1398
|
+
never one from any other query:** `~/.fleet/bin/fleet-run cell-readout.mjs`
|
|
1399
|
+
prints `<cell> <comparisons> <run_dates> <resolved models>` for each cell past
|
|
1400
|
+
that floor and a cell below it only as a count on stderr; a `mixed (…)` models
|
|
1401
|
+
column means the cell's history spans more than one model behind its role.
|
|
1402
|
+
The pre-cell pair query counted pairs that are no cell's comparisons, and the
|
|
1403
|
+
query before that one counted every session whose implementers merely
|
|
1404
|
+
differed, so it cleared this floor by an order of magnitude while the
|
|
1405
|
+
controlled comparison did not exist yet.
|
|
1405
1406
|
|
|
1406
1407
|
Why the agent body carries what it does — read this before editing any
|
|
1407
1408
|
`fleet-implementer-<cell>` file, and keep every cell's body byte-identical. Each rule in the body's
|