@feigi/fleet-ctl 3.23.27 → 3.23.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
package/scripts/cell-readout.mjs
CHANGED
|
@@ -42,10 +42,15 @@
|
|
|
42
42
|
// with an admissible Pull at X dated on or after the day X's
|
|
43
43
|
// `fleet-implementer-<cell>` definition was most recently added, ruled by its
|
|
44
44
|
// last tier-outcomes.tsv row (a `+`-joined `ticket` field rules each ticket
|
|
45
|
-
// it names)
|
|
45
|
+
// it names) dated on or after the ticket's last such Pull: a ruling never
|
|
46
|
+
// predates the Pull it rules, and a row with both verdict columns blank was
|
|
47
|
+
// never ruled, so it is skipped. A ticket with no such row is no verdict.
|
|
48
|
+
// A ticket counts once however many Pulls it took. A verdict FAILS
|
|
46
49
|
// the quality floor on `minted_false_claim=yes` or `closed_own_ticket=no`; a
|
|
47
|
-
// counted ruling holding anything but `yes` or `no` in either column,
|
|
48
|
-
// included, is refused rather than read as a pass.
|
|
50
|
+
// counted ruling holding anything but `yes` or `no` in either column, one
|
|
51
|
+
// blank included, is refused rather than read as a pass. So is a non-blank
|
|
52
|
+
// ruling row of such a ticket whose `run_date` is not `YYYY-MM-DD`: it cannot
|
|
53
|
+
// be placed against the Pull, and dropping it would uncount the ticket.
|
|
49
54
|
// X is to be withdrawn once it has at least STOP.verdicts verdicts and
|
|
50
55
|
// floor failures ÷ verdicts is at least STOP.failRate.
|
|
51
56
|
//
|
|
@@ -60,6 +65,8 @@ import { parseTsv as parseMemberTsv } from "./member-outcomes.mjs";
|
|
|
60
65
|
import { parseFeatures } from "./pr-cost.mjs";
|
|
61
66
|
import { VERDICT, VERDICT_COLUMNS } from "./tier-outcomes.mjs";
|
|
62
67
|
|
|
68
|
+
const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
|
|
69
|
+
|
|
63
70
|
const NAME = "cell-readout";
|
|
64
71
|
export const GATE = Object.freeze({ comparisons: 10, runDates: 5 });
|
|
65
72
|
export const STOP = Object.freeze({ verdicts: 10, failRate: 0.8 });
|
|
@@ -126,28 +133,42 @@ export function readout({ features, members }) {
|
|
|
126
133
|
* sorted by ticket, `run_date` the ruling's; `stop` is whether the stopping
|
|
127
134
|
* rule withdraws the cell. `verdicts` are parsed tier-outcomes.tsv rows.
|
|
128
135
|
* Throws on a counted ruling whose `closed_own_ticket` or `minted_false_claim`
|
|
129
|
-
* is not `yes` or `no
|
|
136
|
+
* is not `yes` or `no`, and on a non-blank ruling row of a Pulled ticket whose
|
|
137
|
+
* `run_date` is not `YYYY-MM-DD`.
|
|
130
138
|
*/
|
|
131
139
|
export function stoppingRule({ features, members, verdicts, added }) {
|
|
132
140
|
const byKey = new Map(members.map((m) => [key(m.session, m.agent), m]));
|
|
133
|
-
const
|
|
134
|
-
for (const v of verdicts)
|
|
135
|
-
|
|
141
|
+
const rulings = new Map();
|
|
142
|
+
for (const v of verdicts) {
|
|
143
|
+
if (VERDICT_COLUMNS.every((c) => v[c] === "")) continue;
|
|
144
|
+
for (const t of String(v.ticket).split("+")) (rulings.get(t) ?? rulings.set(t, []).get(t)).push(v);
|
|
145
|
+
}
|
|
146
|
+
// Per cell, each ticket's last admissible Pull: the one its ruling must follow.
|
|
147
|
+
const charged = new Map(Object.keys(added).filter((c) => c !== POLICY_CELL).map((c) => [c, new Map()]));
|
|
136
148
|
for (const p of features) {
|
|
137
|
-
const
|
|
138
|
-
if (!
|
|
139
|
-
|
|
140
|
-
for (const c of VERDICT_COLUMNS) {
|
|
141
|
-
if (!VERDICT.includes(v[c])) throw new Error(`ticket #${p.ticket} (PR #${v.pr}): ${c} is '${v[c]}', expected yes or no`);
|
|
142
|
-
}
|
|
143
|
-
const failed = v.minted_false_claim === "yes" || v.closed_own_ticket === "no";
|
|
144
|
-
tickets.set(p.ticket, {
|
|
145
|
-
ticket: p.ticket, pr: v.pr, run_date: v.run_date,
|
|
146
|
-
closed_own_ticket: v.closed_own_ticket, minted_false_claim: v.minted_false_claim, failed,
|
|
147
|
-
});
|
|
149
|
+
const pulls = charged.get(p.chosen_cell);
|
|
150
|
+
if (!pulls || p.run_date < added[p.chosen_cell] || !admissibleMember(p, byKey)) continue;
|
|
151
|
+
pulls.set(p.ticket, p);
|
|
148
152
|
}
|
|
149
|
-
return [...
|
|
150
|
-
const list = [
|
|
153
|
+
return [...charged].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)).map(([cell, pulls]) => {
|
|
154
|
+
const list = [];
|
|
155
|
+
for (const p of pulls.values()) {
|
|
156
|
+
const own = rulings.get(p.ticket) ?? [];
|
|
157
|
+
for (const r of own) {
|
|
158
|
+
if (!ISO_DATE.test(r.run_date)) throw new Error(`ticket #${p.ticket} (PR #${r.pr}): run_date is '${r.run_date}', expected YYYY-MM-DD`);
|
|
159
|
+
}
|
|
160
|
+
const v = own.findLast((r) => r.run_date >= p.run_date);
|
|
161
|
+
if (!v) continue;
|
|
162
|
+
for (const c of VERDICT_COLUMNS) {
|
|
163
|
+
if (!VERDICT.includes(v[c])) throw new Error(`ticket #${p.ticket} (PR #${v.pr}): ${c} is '${v[c]}', expected yes or no`);
|
|
164
|
+
}
|
|
165
|
+
list.push({
|
|
166
|
+
ticket: p.ticket, pr: v.pr, run_date: v.run_date,
|
|
167
|
+
closed_own_ticket: v.closed_own_ticket, minted_false_claim: v.minted_false_claim,
|
|
168
|
+
failed: v.minted_false_claim === "yes" || v.closed_own_ticket === "no",
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
list.sort((a, b) => Number(a.ticket) - Number(b.ticket));
|
|
151
172
|
const failures = list.filter((v) => v.failed).length;
|
|
152
173
|
const stop = list.length >= STOP.verdicts && failures / list.length >= STOP.failRate;
|
|
153
174
|
return { cell, since: added[cell], verdicts: list, failures, stop };
|
|
@@ -469,3 +469,112 @@ test("stopping rule: a malformed ruling on a ticket with no counted Pull is not
|
|
|
469
469
|
assert.equal(at.verdicts.length, 4);
|
|
470
470
|
assert.deepEqual(at.verdicts.map((v) => v.failed), [false, true, true, true]);
|
|
471
471
|
});
|
|
472
|
+
|
|
473
|
+
// ---------------------------------------------------------------------------
|
|
474
|
+
// Which ruling is a Pull's: the last row dated on or after the Pull, a
|
|
475
|
+
// both-blank (never ruled) row skipped.
|
|
476
|
+
|
|
477
|
+
// A second admissible Pull of an existing ticket at smol-high, in its own session.
|
|
478
|
+
function rePull(w, ticket, date) {
|
|
479
|
+
const session = `2026-10-02T00-00-00-000Z_v${nextSession++}`;
|
|
480
|
+
const agent = `impl-${ticket}-re`;
|
|
481
|
+
w.features.push(pull({ session, agent, ticket, chosen_cell: "smol-high", run_date: date }));
|
|
482
|
+
w.members.push(member({
|
|
483
|
+
session, agent, run_date: date, ticket, model: "claude-sonnet-5",
|
|
484
|
+
effort: "high", subagentType: "fleet-implementer-smol-high",
|
|
485
|
+
}));
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
test("stopping rule: a ruling dated before its Pull does not rule it, so only pre-Pull rulings leave the cell with no verdicts; one dated the Pull's own day does", () => {
|
|
489
|
+
const added = { "smol-high": "2026-10-02" };
|
|
490
|
+
const stale = world();
|
|
491
|
+
addVerdicts(stale, "smol-high", 10, 0, { date: "2026-10-05" });
|
|
492
|
+
for (const v of stale.verdicts) Object.assign(v, { run_date: "2026-09-17", closed_own_ticket: "no" });
|
|
493
|
+
const at = judged(stale, added, "smol-high");
|
|
494
|
+
assert.deepEqual(at.verdicts, []);
|
|
495
|
+
assert.equal(at.failures, 0);
|
|
496
|
+
assert.equal(at.stop, false);
|
|
497
|
+
|
|
498
|
+
const sameDay = world();
|
|
499
|
+
addVerdicts(sameDay, "smol-high", 10, 0, { date: "2026-10-05" });
|
|
500
|
+
for (const v of sameDay.verdicts) Object.assign(v, { run_date: "2026-10-05", closed_own_ticket: "no" });
|
|
501
|
+
const day = judged(sameDay, added, "smol-high");
|
|
502
|
+
assert.equal(day.verdicts.length, 10, "a ruling landing the same day as its Pull rules it");
|
|
503
|
+
assert.equal(day.failures, 10);
|
|
504
|
+
assert.equal(day.stop, true);
|
|
505
|
+
});
|
|
506
|
+
|
|
507
|
+
test("stopping rule: a re-Pulled ticket is charged with the ruling after its last Pull at the cell, in either corpus order, and with none while that Pull is unruled", () => {
|
|
508
|
+
const added = { "smol-high": "2026-09-01" };
|
|
509
|
+
// Pulled 2026-10-02 and ruled 2026-10-03 failing the floor, then re-Pulled 2026-10-05.
|
|
510
|
+
const build = () => {
|
|
511
|
+
const w = world();
|
|
512
|
+
addVerdicts(w, "smol-high", 1, 1);
|
|
513
|
+
rePull(w, w.features[0].ticket, "2026-10-05");
|
|
514
|
+
return w;
|
|
515
|
+
};
|
|
516
|
+
const unruled = build();
|
|
517
|
+
const { ticket } = unruled.features[0];
|
|
518
|
+
assert.deepEqual(judged(unruled, added, "smol-high").verdicts, [], "the earlier ruling predates the re-Pull");
|
|
519
|
+
|
|
520
|
+
const later = build();
|
|
521
|
+
later.verdicts.push(verdict({ ticket, pr: "9002", run_date: "2026-10-06" }));
|
|
522
|
+
const charged = judged(later, added, "smol-high").verdicts;
|
|
523
|
+
assert.deepEqual(charged.map((v) => [v.ticket, v.pr, v.run_date, v.failed]), [[ticket, "9002", "2026-10-06", false]]);
|
|
524
|
+
|
|
525
|
+
// The stale row appended after the later one, as a backfill would land it.
|
|
526
|
+
const backfilled = build();
|
|
527
|
+
backfilled.verdicts.unshift(verdict({ ticket, pr: "9002", run_date: "2026-10-06" }));
|
|
528
|
+
assert.deepEqual(judged(backfilled, added, "smol-high").verdicts.map((v) => v.pr), ["9002"]);
|
|
529
|
+
});
|
|
530
|
+
|
|
531
|
+
test("stopping rule: a both-blank row was never ruled: after a real ruling it does not hide it, and a ticket whose only row it is stays uncounted", () => {
|
|
532
|
+
const w = world();
|
|
533
|
+
addVerdicts(w, "smol-high", 2, 2);
|
|
534
|
+
const [real, blank] = w.features.map((p) => p.ticket);
|
|
535
|
+
w.verdicts.push(verdict({ ticket: real, pr: "1392", run_date: "2026-10-04", closed_own_ticket: "", minted_false_claim: "" }));
|
|
536
|
+
Object.assign(w.verdicts.find((v) => v.ticket === blank), { closed_own_ticket: "", minted_false_claim: "" });
|
|
537
|
+
const at = judged(w, { "smol-high": "2026-09-01" }, "smol-high");
|
|
538
|
+
assert.deepEqual(at.verdicts.map((v) => [v.ticket, v.pr, v.failed]), [[real, String(Number(real) + 5000), true]]);
|
|
539
|
+
});
|
|
540
|
+
|
|
541
|
+
// What the both-blank skip must NOT accept: a row with one verdict blank is a
|
|
542
|
+
// ruling written half, so selected it is still refused.
|
|
543
|
+
test("stopping rule: a half-blank row after a real ruling is the ticket's ruling and is refused", () => {
|
|
544
|
+
for (const [col, other] of [["closed_own_ticket", "minted_false_claim"], ["minted_false_claim", "closed_own_ticket"]]) {
|
|
545
|
+
const w = world();
|
|
546
|
+
addVerdicts(w, "smol-high", 1, 0);
|
|
547
|
+
const { ticket } = w.features[0];
|
|
548
|
+
w.verdicts.push(verdict({ ticket, pr: "1393", run_date: "2026-10-04", [col]: "", [other]: "no" }));
|
|
549
|
+
assert.throws(() => judged(w, { "smol-high": "2026-09-01" }, "smol-high"),
|
|
550
|
+
new RegExp(`ticket #${ticket} \\(PR #1393\\): ${col} is '', expected yes or no`), col);
|
|
551
|
+
}
|
|
552
|
+
});
|
|
553
|
+
|
|
554
|
+
// A filled ruling whose date cannot be placed against its Pull must not
|
|
555
|
+
// vanish: dropped, its ticket is uncounted and a floor failure goes unreported.
|
|
556
|
+
test("stopping rule: a filled ruling with a blank or malformed run_date is refused, naming the ticket", () => {
|
|
557
|
+
for (const bad of ["", "10/03/2026", "2026-10-3"]) {
|
|
558
|
+
const w = world();
|
|
559
|
+
addVerdicts(w, "smol-high", 1, 0);
|
|
560
|
+
const [{ ticket }] = w.features;
|
|
561
|
+
w.verdicts[0].run_date = bad;
|
|
562
|
+
assert.throws(
|
|
563
|
+
() => judged(w, { "smol-high": "2026-09-01" }, "smol-high"),
|
|
564
|
+
new RegExp(`ticket #${ticket} \\(PR #${w.verdicts[0].pr}\\): run_date is '${bad}', expected YYYY-MM-DD`),
|
|
565
|
+
JSON.stringify(bad),
|
|
566
|
+
);
|
|
567
|
+
}
|
|
568
|
+
});
|
|
569
|
+
|
|
570
|
+
// What that refusal must NOT touch: a never-ruled row has no date to place,
|
|
571
|
+
// and a ticket no live cell's Pull names is not a verdict candidate.
|
|
572
|
+
test("stopping rule: a both-blank row or an unPulled ticket's row needs no run_date", () => {
|
|
573
|
+
const w = world();
|
|
574
|
+
addVerdicts(w, "smol-high", 2, 0);
|
|
575
|
+
const [real, blank] = w.features.map((p) => p.ticket);
|
|
576
|
+
Object.assign(w.verdicts.find((v) => v.ticket === blank), { closed_own_ticket: "", minted_false_claim: "", run_date: "" });
|
|
577
|
+
w.verdicts.push(verdict({ ticket: "999", pr: "1", run_date: "n/a" }));
|
|
578
|
+
const at = judged(w, { "smol-high": "2026-09-01" }, "smol-high");
|
|
579
|
+
assert.deepEqual(at.verdicts.map((v) => v.ticket), [real]);
|
|
580
|
+
});
|
package/scripts/recipe-prove.mjs
CHANGED
|
@@ -57,10 +57,11 @@
|
|
|
57
57
|
// repository, origin/main missing, the worktree could not be made), or
|
|
58
58
|
// a git or filesystem fault stopped it before the cache was settled
|
|
59
59
|
// (a git command could not be started, the reader's own git
|
|
60
|
-
// included; the reader's `sh`,
|
|
61
|
-
//
|
|
62
|
-
//
|
|
63
|
-
// could not be written). The log
|
|
60
|
+
// included, or that git started and could not run; the reader's `sh`,
|
|
61
|
+
// `node`, `mktemp` or `cat` could not be started, or its `node` died
|
|
62
|
+
// mid-read — a `cat` that ran and failed stays the reader's refusal,
|
|
63
|
+
// exit 1; the cache or its temp file could not be written). The log
|
|
64
|
+
// directory under $TMPDIR is removed.
|
|
64
65
|
|
|
65
66
|
import { spawnSync } from "node:child_process";
|
|
66
67
|
import { closeSync, existsSync, mkdirSync, mkdtempSync, openSync, readFileSync, renameSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
|
|
@@ -287,9 +288,12 @@ function prove(o, wt, logs) {
|
|
|
287
288
|
// cache is undone just the same when the read-back is no verdict: an `sh` that
|
|
288
289
|
// could not be started read nothing; a reader that exits READER_COULD_NOT_RUN
|
|
289
290
|
// could not start its `node`, `mktemp` or `cat`, or lost its `node` mid-read;
|
|
290
|
-
// and the reader runs git itself and reads a git that will not
|
|
291
|
+
// and the reader runs git itself and reads a git that will not run as no
|
|
291
292
|
// repository at all, so its refusal is a verdict on what was proven only while
|
|
292
|
-
// git still
|
|
293
|
+
// git still runs. `git --version` is the probe: a working git always passes
|
|
294
|
+
// it, so a git that cannot be started fails it, and so does one that starts
|
|
295
|
+
// and cannot run — a wrapper whose target is gone exits 126 or 127, a killed
|
|
296
|
+
// git has no exit status at all.
|
|
293
297
|
function writeCache(cache, recipe, repo) {
|
|
294
298
|
mkdirSync(dirname(cache), { recursive: true });
|
|
295
299
|
const prior = existsSync(cache) ? readFileSync(cache) : null;
|
|
@@ -308,12 +312,15 @@ function writeCache(cache, recipe, repo) {
|
|
|
308
312
|
else writeFileSync(cache, prior);
|
|
309
313
|
if (r.error && !r.pid) throw cannot(`could not start sh to read the Recipe cache back: ${r.error.message}`);
|
|
310
314
|
if (r.status === READER_COULD_NOT_RUN) throw cannot(`${r.stderr.trim()} — the Recipe cache reader could not run a tool it needs, so its refusal is no verdict on what was proven`);
|
|
315
|
+
const noVerdict = (why) => cannot(`${why} — the Recipe cache reader runs git, so its refusal is no verdict on what was proven`);
|
|
316
|
+
let probe;
|
|
311
317
|
try {
|
|
312
|
-
git(["--version"]);
|
|
318
|
+
probe = git(["--version"]);
|
|
313
319
|
} catch (e) {
|
|
314
320
|
if (!(e instanceof Refusal)) throw e;
|
|
315
|
-
throw
|
|
321
|
+
throw noVerdict(e.message);
|
|
316
322
|
}
|
|
323
|
+
if (!probe.ok) throw noVerdict(`git started but does not run: \`git --version\` did not exit 0${probe.err ? `: ${probe.err}` : " and wrote nothing to stderr"}`);
|
|
317
324
|
throw notProven(`the Recipe cache reader refuses what was proven: ${(r.stderr || r.stdout).trim()}`);
|
|
318
325
|
}
|
|
319
326
|
}
|
|
@@ -665,6 +665,35 @@ test("a git that stops starting before the cache is read back is no verdict, nev
|
|
|
665
665
|
assert.equal(readFileSync(cachePath(dir), "utf8"), "prior bytes", "a cache the reader never settled is rolled back");
|
|
666
666
|
});
|
|
667
667
|
|
|
668
|
+
// A git that starts and cannot run fails the reader's own `git rev-parse` just
|
|
669
|
+
// as a git that will not start does, so its refusal is no verdict either. Each
|
|
670
|
+
// stub stands in for one such git: a wrapper whose target is gone exits 126 or
|
|
671
|
+
// 127, and a killed git has no exit status at all. The stubs differ in what
|
|
672
|
+
// they write to stderr, which the message quotes when there is anything to
|
|
673
|
+
// quote. The control — the same PATH with git left in place proves — is the
|
|
674
|
+
// unstartable-git test's. The Test entrypoint rewrites the stub's body, never
|
|
675
|
+
// the stub: that is a hard link to exec-stub.mjs's shared, read-only
|
|
676
|
+
// trampoline.
|
|
677
|
+
const GIT_THAT_CANNOT_RUN = [
|
|
678
|
+
{ name: "exits 126 and writes nothing", body: "exit 126", cause: " and wrote nothing to stderr" },
|
|
679
|
+
{ name: "exits 127 and writes to stderr", body: "echo boom >&2; exit 127", cause: ": boom" },
|
|
680
|
+
{ name: "is killed by a signal", body: "kill -9 $$", cause: ": git was killed by SIGKILL" },
|
|
681
|
+
];
|
|
682
|
+
for (const c of GIT_THAT_CANNOT_RUN) {
|
|
683
|
+
test(`a git that starts but ${c.name} before the cache is read back is no verdict, never NOT PROVEN — and the prior cache is restored`, () => {
|
|
684
|
+
const { dir } = repo(MAVEN_FILES);
|
|
685
|
+
mkdirSync(join(dir, ".fleet"));
|
|
686
|
+
writeFileSync(cachePath(dir), "prior bytes");
|
|
687
|
+
const bin = readerPath();
|
|
688
|
+
const r = prove(dir, ["--install", "true", "--test", `printf '#!/bin/sh\\n${c.body}\\n' > '${join(bin, ".stub-git")}'; echo 'tests 1'`, "--count-line", "tests 1", "--test-count", "1"], { env: { PATH: bin } });
|
|
689
|
+
assert.equal(r.status, 2, r.err);
|
|
690
|
+
assert.ok(r.err.includes(`recipe-prove: git started but does not run: \`git --version\` did not exit 0${c.cause} — the Recipe cache reader runs git, so its refusal is no verdict on what was proven`), r.err);
|
|
691
|
+
assert.doesNotMatch(r.err, /NOT PROVEN/);
|
|
692
|
+
assert.doesNotMatch(r.err, /not a git repository/);
|
|
693
|
+
assert.equal(readFileSync(cachePath(dir), "utf8"), "prior bytes", "a cache the reader never settled is rolled back");
|
|
694
|
+
});
|
|
695
|
+
}
|
|
696
|
+
|
|
668
697
|
// The reader's own `sh`, taken away the same way: a read-back that never ran
|
|
669
698
|
// read nothing, so it is no verdict either.
|
|
670
699
|
test("an sh that stops starting before the cache is read back is no verdict, never NOT PROVEN — and no cache is left", () => {
|
|
@@ -352,6 +352,16 @@ test("close-out: a gh that prints something other than JSON fails the duty namin
|
|
|
352
352
|
assert.match(r.stderr, /stopping rule failed: gh issue list printed non-JSON \(.*\): "<html>rate limited<\/html>"$/m);
|
|
353
353
|
});
|
|
354
354
|
|
|
355
|
+
test("close-out --dry-run: a both-blank tier-outcomes row after a ticket's real ruling leaves the count as it was, and the router fit still runs", () => {
|
|
356
|
+
const rows = floorFailing();
|
|
357
|
+
rows.tiers.push(tier({ ticket: "100", pr: "1392", closed_own_ticket: "", minted_false_claim: "" }));
|
|
358
|
+
const f = fixture({ rows });
|
|
359
|
+
const r = f.run("close-out", ["--dry-run"]);
|
|
360
|
+
assert.equal(r.status, 0, r.stderr);
|
|
361
|
+
assert.match(r.stdout, new RegExp(`stopping rule: smol-high 8/10 floor failures since 2026-09-01 — would file "${escape(TITLE)}"`));
|
|
362
|
+
assert.match(r.stdout, /^run-team-local: router fit: /m);
|
|
363
|
+
});
|
|
364
|
+
|
|
355
365
|
// ---------------------------------------------------------------------------
|
|
356
366
|
// close-out: the router re-fit cadence
|
|
357
367
|
|