@feigi/fleet-ctl 3.23.34 → 3.23.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
package/scripts/cell-readout.mjs
CHANGED
|
@@ -6,7 +6,10 @@
|
|
|
6
6
|
//
|
|
7
7
|
// Inputs default to docs/metrics/ under the working directory. A Pull is a
|
|
8
8
|
// ticket-features.tsv row; what it ran is the member-outcomes.tsv row with the
|
|
9
|
-
// same `session` + `agent`.
|
|
9
|
+
// same `session` + `agent`. Rows of one `session` + `agent` that name one
|
|
10
|
+
// `chosen_cell` are one Pull; the readout refuses rows naming different cells,
|
|
11
|
+
// naming the file, the session, the agent and both cells, since which cell ran
|
|
12
|
+
// is then unknown.
|
|
10
13
|
//
|
|
11
14
|
// ADMISSIBLE ROW. A Pull counts only when its member row exists, was
|
|
12
15
|
// dispatched as the drawn cell's own definition (`subagent_type` =
|
|
@@ -34,8 +37,10 @@
|
|
|
34
37
|
// model by name, or `mixed (<model-a> n=…, <model-b> n=…)` when they span more
|
|
35
38
|
// than one, because a role's target is operator config that can move under a
|
|
36
39
|
// cell's history. A cell below the gate prints nothing on stdout and one
|
|
37
|
-
// `below the gate` count on stderr.
|
|
38
|
-
// ticket or member: the gate forbids reading a comparison early.
|
|
40
|
+
// `below the gate` count on stderr. Neither the gate line nor that count names
|
|
41
|
+
// a session, ticket or member: the gate forbids reading a comparison early.
|
|
42
|
+
// Only the refusal of conflicting ticket-features rows names a session and an
|
|
43
|
+
// agent, and it prints nothing on stdout.
|
|
39
44
|
//
|
|
40
45
|
// STOPPING RULE (exported as `stoppingRule`, not printed by this CLI). For a
|
|
41
46
|
// cell X other than slow-high whose definition is live, a VERDICT is a ticket
|
|
@@ -74,6 +79,21 @@ export const STOP = Object.freeze({ verdicts: 10, failRate: 0.8 });
|
|
|
74
79
|
const key = (session, agent) => `${session}\0${agent}`;
|
|
75
80
|
const levelOf = (cell) => CELL.exec(cell)?.[2] ?? null;
|
|
76
81
|
|
|
82
|
+
/**
|
|
83
|
+
* `features` keyed by session+agent. A repeat naming the same `chosen_cell`
|
|
84
|
+
* collapses into the first row; one naming another cell throws.
|
|
85
|
+
*/
|
|
86
|
+
function pullsOf(features) {
|
|
87
|
+
const pulls = new Map();
|
|
88
|
+
for (const p of features) {
|
|
89
|
+
const k = key(p.session, p.agent);
|
|
90
|
+
const prev = pulls.get(k);
|
|
91
|
+
if (!prev) pulls.set(k, p);
|
|
92
|
+
else if (prev.chosen_cell !== p.chosen_cell) throw new Error(`session ${p.session} agent ${p.agent} has two rows, chosen_cell '${prev.chosen_cell}' and '${p.chosen_cell}'`);
|
|
93
|
+
}
|
|
94
|
+
return pulls;
|
|
95
|
+
}
|
|
96
|
+
|
|
77
97
|
/** The join's member row for a Pull when the Pull is admissible, else null. */
|
|
78
98
|
function admissibleMember(pull, members) {
|
|
79
99
|
const m = members.get(key(pull.session, pull.agent));
|
|
@@ -88,13 +108,12 @@ function admissibleMember(pull, members) {
|
|
|
88
108
|
* ticket-features row, `{ cell, comparisons, runDates, dates, models, gated }`,
|
|
89
109
|
* sorted by cell. `dates` is the comparisons' distinct run_dates, sorted;
|
|
90
110
|
* `models` maps each resolved model of the cell's admissible rows to its row
|
|
91
|
-
* count, largest first. `unjoined` counts Pulls with no member row.
|
|
111
|
+
* count, largest first. `unjoined` counts Pulls with no member row. Throws on
|
|
112
|
+
* two ticket-features rows of one session+agent naming different cells.
|
|
92
113
|
*/
|
|
93
114
|
export function readout({ features, members }) {
|
|
94
115
|
const byKey = new Map(members.map((m) => [key(m.session, m.agent), m]));
|
|
95
|
-
|
|
96
|
-
// same member row.
|
|
97
|
-
const pulls = new Map(features.map((p) => [key(p.session, p.agent), p]));
|
|
116
|
+
const pulls = pullsOf(features);
|
|
98
117
|
const sessions = new Map();
|
|
99
118
|
const cells = new Map();
|
|
100
119
|
let unjoined = 0;
|
|
@@ -201,7 +220,9 @@ function main() {
|
|
|
201
220
|
const features = load(featuresPath, parseFeatures);
|
|
202
221
|
const members = load(membersPath, parseMemberTsv);
|
|
203
222
|
|
|
204
|
-
|
|
223
|
+
let cells, unjoined;
|
|
224
|
+
try { ({ cells, unjoined } = readout({ features, members })); }
|
|
225
|
+
catch (e) { die(`${featuresPath}: ${e.message}`); }
|
|
205
226
|
const lines = [];
|
|
206
227
|
const notes = [];
|
|
207
228
|
if (unjoined > 0) {
|
|
@@ -196,6 +196,41 @@ test("a re-dispatched Pull is one Pull: two ticket-features rows for one session
|
|
|
196
196
|
assert.equal(unjoined, 1, "a re-dispatched Pull with no member row is one unjoined Pull");
|
|
197
197
|
});
|
|
198
198
|
|
|
199
|
+
test("two ticket-features rows for one session+agent naming different cells are refused, naming the session, agent and both cells", () => {
|
|
200
|
+
const w = world();
|
|
201
|
+
addRow(w, { session: "sTorn", date: "2026-10-01", cell: "task-high" });
|
|
202
|
+
addRow(w, { session: "sTorn", date: "2026-10-01", cell: "slow-high" });
|
|
203
|
+
// The same Pull's row again, drawn at another cell: which cell ran is unknown.
|
|
204
|
+
w.features.push({ ...w.features[0], chosen_cell: "smol-high" });
|
|
205
|
+
const f = files(w);
|
|
206
|
+
const r = spawnSync(process.execPath, [SCRIPT, "--ticket-features", f.features, "--member-outcomes", f.members], { encoding: "utf8", cwd: f.dir });
|
|
207
|
+
assert.equal(r.status, 2, r.stderr);
|
|
208
|
+
assert.equal(r.stdout, "");
|
|
209
|
+
assert.equal(r.stderr, `\ncell-readout: ${f.features}: session sTorn agent impl-100 has two rows, chosen_cell 'task-high' and 'smol-high'\n`);
|
|
210
|
+
assert.throws(() => readout(parsed(w)), /session sTorn agent impl-100 has two rows, chosen_cell 'task-high' and 'smol-high'/);
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
// What the refusal must ACCEPT: a repeat that agrees on the cell, whatever else
|
|
214
|
+
// it differs on, and one agent name reused in another session.
|
|
215
|
+
test("a repeated session+agent on the same cell is one Pull even when its other columns differ, and the key is the session+agent pair", () => {
|
|
216
|
+
const w = world();
|
|
217
|
+
addRow(w, { session: "sSame", date: "2026-10-01", cell: "task-high" });
|
|
218
|
+
addRow(w, { session: "sSame", date: "2026-10-01", cell: "slow-high" });
|
|
219
|
+
w.features.push({ ...w.features[0], run_date: "2026-10-02", brief_chars: "200" });
|
|
220
|
+
// impl-100 again, in another session and at another cell: a different Pull.
|
|
221
|
+
addRow(w, { session: "sOther", date: "2026-10-02", cell: "smol-high" });
|
|
222
|
+
w.features.at(-1).agent = "impl-100";
|
|
223
|
+
w.members.at(-1).agent = "impl-100";
|
|
224
|
+
w.members.at(-1).member = "impl-100";
|
|
225
|
+
addRow(w, { session: "sOther", date: "2026-10-02", cell: "slow-high" });
|
|
226
|
+
const r = cli(w);
|
|
227
|
+
assert.equal(r.status, 0, r.stderr);
|
|
228
|
+
const { cells } = readout(parsed(w));
|
|
229
|
+
const of = (cell) => cells.find((c) => c.cell === cell);
|
|
230
|
+
assert.deepEqual([of("task-high").comparisons, [...of("task-high").models]], [1, [["claude-sonnet-5", 1]]]);
|
|
231
|
+
assert.equal(of("smol-high").comparisons, 1);
|
|
232
|
+
});
|
|
233
|
+
|
|
199
234
|
test("a blank member-outcomes run_date is no date: its session is a comparison but adds nothing to the distinct-date count", () => {
|
|
200
235
|
const w = world();
|
|
201
236
|
addComparisons(w, "task-high", 4, 4);
|