@feigi/fleet-ctl 0.0.0 → 3.16.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/agents/{fleet-implementer.agent.md → fleet-implementer-slow-high.agent.md} +23 -2
  2. package/agents/fleet-implementer-slow-medium.agent.md +231 -0
  3. package/agents/fleet-implementer-smol-high.agent.md +231 -0
  4. package/agents/{fleet-implementer-alt.agent.md → fleet-implementer-task-high.agent.md} +23 -2
  5. package/agents/fleet-implementer-task-max.agent.md +231 -0
  6. package/agents/fleet-review-comments.agent.md +4 -2
  7. package/agents/fleet-review-correctness.agent.md +2 -1
  8. package/agents/fleet-review-runner.agent.md +14 -10
  9. package/agents/fleet-review-silent-failure.agent.md +4 -3
  10. package/agents/fleet-review-simplify.agent.md +2 -1
  11. package/agents/fleet-review-snapshot.agent.md +2 -1
  12. package/agents/fleet-review-test-run.agent.md +29 -0
  13. package/agents/fleet-review-tests.agent.md +10 -5
  14. package/agents/fleet-review-types.agent.md +4 -2
  15. package/agents/fleet-review-verifier.agent.md +4 -3
  16. package/commands/review-and-fix.md +7 -6
  17. package/commands/run-merge-bot.md +14 -4
  18. package/package.json +4 -2
  19. package/scripts/agent-test.test.mjs +36 -32
  20. package/scripts/ambient-git-vars-mjs-prose.test.mjs +23 -13
  21. package/scripts/apply-ruleset.test.mjs +4 -3
  22. package/scripts/arg-header-probes-prose.test.mjs +1 -0
  23. package/scripts/arg.test.mjs +18 -18
  24. package/scripts/board-cli.test.mjs +18 -21
  25. package/scripts/board-identity.test.mjs +9 -9
  26. package/scripts/board-liveness-banner.test.mjs +6 -6
  27. package/scripts/board-prev-shape.test.mjs +12 -13
  28. package/scripts/board-tryparse-null.test.mjs +7 -7
  29. package/scripts/board.mjs +4 -7
  30. package/scripts/board.test.mjs +57 -61
  31. package/scripts/candidates.test.mjs +228 -17
  32. package/scripts/ci-completes-premise-prose.test.mjs +1 -1
  33. package/scripts/ci-state-prose.test.mjs +2 -2
  34. package/scripts/ci-state.mjs +389 -187
  35. package/scripts/ci-state.test.mjs +458 -165
  36. package/scripts/claim-lifecycle.test.mjs +2 -1
  37. package/scripts/claim-ticket.test.mjs +35 -34
  38. package/scripts/closing-keyword-prose.test.mjs +1 -1
  39. package/scripts/cockpit-tick-nonblocking.test.mjs +8 -8
  40. package/scripts/compute-board.mjs +13 -3
  41. package/scripts/compute-board.test.mjs +64 -1
  42. package/scripts/compute-spend.mjs +46 -8
  43. package/scripts/compute-spend.test.mjs +64 -10
  44. package/scripts/cross-repo-citation-prose.test.mjs +18 -19
  45. package/scripts/delete-merged-branch.sh +218 -0
  46. package/scripts/delete-merged-branch.test.mjs +576 -0
  47. package/scripts/derive-testcmd.sh +18 -7
  48. package/scripts/derive-testcmd.test.mjs +9 -8
  49. package/scripts/diff-stats.mjs +2 -2
  50. package/scripts/diff-stats.test.mjs +7 -11
  51. package/scripts/dispatch-block-golden-prose.test.mjs +111 -6
  52. package/scripts/dispatch-block-pins-prose.test.mjs +5 -5
  53. package/scripts/dispositions-check.mjs +507 -0
  54. package/scripts/dispositions-check.test.mjs +843 -0
  55. package/scripts/dispositions-gate-prose.test.mjs +114 -0
  56. package/scripts/drop-merged-label.test.mjs +3 -4
  57. package/scripts/exec-stub.mjs +93 -0
  58. package/scripts/exec-stub.test.mjs +69 -0
  59. package/scripts/filing-bar-seat-prose.test.mjs +1 -1
  60. package/scripts/finisher-test-verdict-prose.test.mjs +102 -0
  61. package/scripts/fix-applier-correction-rules-prose.test.mjs +11 -10
  62. package/scripts/fleet-bootstrap +5 -20
  63. package/scripts/fleet-bootstrap.test.mjs +75 -0
  64. package/scripts/fleet-heartbeat.mjs +2 -2
  65. package/scripts/fleet-provenance +15 -24
  66. package/scripts/fleet-provenance.test.mjs +86 -12
  67. package/scripts/fleet-run.test.mjs +5 -5
  68. package/scripts/fleet-state.mjs +33 -11
  69. package/scripts/fleet-state.test.mjs +90 -3
  70. package/scripts/fleet-tick.mjs +346 -70
  71. package/scripts/fleet-tick.test.mjs +566 -16
  72. package/scripts/force-with-lease-reason-prose.test.mjs +1 -1
  73. package/scripts/frontmatter-check.mjs +3 -2
  74. package/scripts/frontmatter-check.test.mjs +6 -6
  75. package/scripts/immutable-body-claim-prose.test.mjs +53 -262
  76. package/scripts/implementer-model-tier.test.mjs +98 -506
  77. package/scripts/inflight.test.mjs +18 -127
  78. package/scripts/is-cli.mjs +26 -0
  79. package/scripts/is-cli.test.mjs +99 -0
  80. package/scripts/json.test.mjs +3 -3
  81. package/scripts/ledger-dispatch.test.mjs +368 -7
  82. package/scripts/ledger-grammar.mjs +85 -1
  83. package/scripts/ledger-grammar.test.mjs +86 -1
  84. package/scripts/ledger-read-require-file.test.mjs +7 -7
  85. package/scripts/ledger-rotate.test.mjs +142 -0
  86. package/scripts/ledger.mjs +181 -17
  87. package/scripts/ledger.test.mjs +12 -130
  88. package/scripts/main-checkout.mjs +312 -0
  89. package/scripts/main-checkout.test.mjs +332 -0
  90. package/scripts/member-outcomes-header.test.mjs +3 -3
  91. package/scripts/member-outcomes.mjs +4 -5
  92. package/scripts/member-outcomes.test.mjs +92 -18
  93. package/scripts/member-prompt-prose.test.mjs +17 -13
  94. package/scripts/member-record.mjs +8 -3
  95. package/scripts/member-record.test.mjs +5 -5
  96. package/scripts/member-write-guard.mjs +5 -5
  97. package/scripts/member-write-guard.test.mjs +6 -3
  98. package/scripts/no-undo-audit.sh +24 -21
  99. package/scripts/no-undo-audit.test.mjs +9 -13
  100. package/scripts/pointer-target-prose.test.mjs +1 -1
  101. package/scripts/pr-overlap.test.mjs +6 -9
  102. package/scripts/prompt-renderer.mjs +10 -9
  103. package/scripts/prose-pin.mjs +10 -11
  104. package/scripts/prove-merge.test.mjs +3 -3
  105. package/scripts/quiet-payload-prose.test.mjs +1 -1
  106. package/scripts/reap.test.mjs +3 -4
  107. package/scripts/release-label-split-prose.test.mjs +17 -2
  108. package/scripts/release-ticket.test.mjs +35 -44
  109. package/scripts/review-core-cwd-isolation.test.mjs +7 -6
  110. package/scripts/review-core-reads.test.mjs +1 -1
  111. package/scripts/review-core-scratch-refusal.test.mjs +64 -0
  112. package/scripts/review-core-shared-test-run.test.mjs +238 -0
  113. package/scripts/review-core-snapshot-path.test.mjs +59 -12
  114. package/scripts/review-core-specialist-scratch.test.mjs +12 -10
  115. package/scripts/review-core-testcmd.test.mjs +76 -45
  116. package/scripts/review-core-unrun.test.mjs +154 -168
  117. package/scripts/review-core.mjs +250 -56
  118. package/scripts/review-eval.mjs +11 -4
  119. package/scripts/review-host-fixture.mjs +12 -3
  120. package/scripts/review-in-run-retry.test.mjs +27 -3
  121. package/scripts/review-runner-spawns.test.mjs +5 -3
  122. package/scripts/review-runner.test.mjs +14 -3
  123. package/scripts/review-tier-audit.test.mjs +1 -1
  124. package/scripts/run-merge-bot-prose.test.mjs +29 -6
  125. package/scripts/scratch-partition-invariant-prose.test.mjs +1 -1
  126. package/scripts/section-between-guard-prose.test.mjs +4 -3
  127. package/scripts/select-dimensions.test.mjs +1 -1
  128. package/scripts/shell-traps-prose.test.mjs +6 -9
  129. package/scripts/shortlist.mjs +4 -1
  130. package/scripts/shortlist.test.mjs +30 -16
  131. package/scripts/snapshot-repo.test.mjs +177 -4
  132. package/scripts/stash-prohibition-prose.test.mjs +1 -1
  133. package/scripts/temp-dir.mjs +47 -0
  134. package/scripts/tier-check.mjs +95 -28
  135. package/scripts/tier-check.test.mjs +131 -29
  136. package/scripts/tier-outcomes-header.test.mjs +11 -54
  137. package/scripts/tier-outcomes.mjs +34 -12
  138. package/scripts/tier-outcomes.test.mjs +157 -4
  139. package/scripts/tier-roles.mjs +118 -29
  140. package/scripts/tier-roles.test.mjs +226 -61
  141. package/scripts/tracker-block-copy-prose.test.mjs +3 -3
  142. package/scripts/verify-sha.test.mjs +4 -5
  143. package/scripts/watcher-degraded-block.test.mjs +5 -8
  144. package/scripts/within-run-pair-prose.test.mjs +43 -75
  145. package/scripts/worktree-audit.test.mjs +2 -1
  146. package/skills/next-ticket/SKILL.md +2 -2
  147. package/skills/run-team/SKILL.md +433 -303
  148. package/skills/run-team/references/ci-and-staleness.md +7 -6
  149. package/skills/run-team/references/correction-tickets.md +3 -3
  150. package/skills/run-team/references/isolation.md +1 -1
  151. package/skills/sizing-a-ticket/SKILL.md +8 -8
  152. package/scripts/tier-guard-gate-prose.test.mjs +0 -216
@@ -5,7 +5,7 @@
5
5
  // `opus` for two implementers dispatched as a generic `task` that ran
6
6
  // sonnet/high, which is neither tier arm.
7
7
  //
8
- // append <pr> --class <c> --closed-own-ticket yes|no --minted-false-claim yes|no
8
+ // append <pr> --closed-own-ticket yes|no --minted-false-claim yes|no
9
9
  // --note <text> [--sizing light|heavy] [--profile <p>] [--loc <n>] [--files <n>]
10
10
  // Fills `run_date` (today, the ruling date), `pr`, `ticket` (off
11
11
  // `gh pr view --json closingIssuesReferences`) and `tier`. Idempotent by
@@ -13,7 +13,9 @@
13
13
  // check [--live]
14
14
  // Post-switch rows only: a filled `tier` must equal the short name of the
15
15
  // ticket's single implementer row in member-outcomes.tsv, and that row
16
- // must be a `fleet-implementer*` definition — exit 1 otherwise. `--live`
16
+ // must be a `fleet-implementer*` definition — exit 1 otherwise, and exit 1
17
+ // when none of the latest run_date's filled rows could be compared and some
18
+ // lack a member row (nothing from that run was checked). `--live`
17
19
  // (a run in progress; CI never passes it) adds a WARNING, never a
18
20
  // failure, per PR whose run-ledger row carries `reviewed=` but which has
19
21
  // no row here. `--ledger <path>` names another ledger than the run's own.
@@ -41,12 +43,14 @@
41
43
  // `opus`/`sonnet` spelling and are never checked; the header records the
42
44
  // switch and how the old values map.
43
45
  //
44
- // Exit codes: 0 ok, 1 `check` found a mismatch, 2 usage or unreadable input.
46
+ // Exit codes: 0 ok, 1 `check` found a mismatch or checked nothing from the latest
47
+ // run_date (see `check`), 2 usage or unreadable input.
45
48
 
46
49
  import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
47
50
  import { execFileSync } from "node:child_process";
48
51
  import { dirname, join } from "node:path";
49
- import { fileURLToPath, pathToFileURL } from "node:url";
52
+ import { fileURLToPath } from "node:url";
53
+ import { isCLI } from "./is-cli.mjs";
50
54
  import { makeDie, defineFlags, isDigits } from "./arg.mjs";
51
55
  import { parseTsv as parseMemberTsv } from "./member-outcomes.mjs";
52
56
  import { parseMember, parseToken, memberTokens } from "./ledger-grammar.mjs";
@@ -116,7 +120,10 @@ export function implementerRows(memberRows, ticket) {
116
120
  }
117
121
 
118
122
  // tier-check.mjs's verdict tokens on a ledger row, restricted to this
119
- // ticket's implementers.
123
+ // ticket's implementers. Only the two that say what ran: a
124
+ // `tier-unverifiable=` member settled with no transcript to check, so it
125
+ // matches neither kind here and resolves as having no ledger verdict — the
126
+ // fallback order below decides it, and it is never read as `tier-ok`.
120
127
  const TIER_VERDICT = /^tier-(ok|mismatch)=([^:\s]+):(\S+)$/;
121
128
 
122
129
  // A `{tier, source}` on success or `{tier: "", warning}` when the tier must
@@ -208,6 +215,8 @@ export function checkRows(rows, memberRows) {
208
215
  const failures = [];
209
216
  const skipped = { preSwitch: 0, blank: 0, noMemberRow: 0, severalMemberRows: 0 };
210
217
  let checked = 0;
218
+ // The filled post-switch rows of each run_date, for the vacuity guard below.
219
+ const byDate = new Map();
211
220
  for (const row of rows) {
212
221
  if (!DATE.test(row.run_date)) {
213
222
  failures.push(`PR #${row.pr}: run_date '${row.run_date}' is not YYYY-MM-DD`);
@@ -216,9 +225,11 @@ export function checkRows(rows, memberRows) {
216
225
  if (row.run_date < TIER_SWITCH_DATE) { skipped.preSwitch++; continue; }
217
226
  if (row.tier === "") { skipped.blank++; continue; }
218
227
  const members = implementerRows(memberRows, row.ticket);
219
- if (members.length === 0) { skipped.noMemberRow++; continue; }
228
+ const day = byDate.get(row.run_date) ?? byDate.set(row.run_date, { checked: 0, lost: [] }).get(row.run_date);
229
+ if (members.length === 0) { skipped.noMemberRow++; day.lost.push(`PR #${row.pr} (ticket #${row.ticket})`); continue; }
220
230
  if (members.length > 1) { skipped.severalMemberRows++; continue; }
221
231
  checked++;
232
+ day.checked++;
222
233
  const [m] = members;
223
234
  const actual = shortName(m.subagentType);
224
235
  if (actual === null) {
@@ -227,6 +238,18 @@ export function checkRows(rows, memberRows) {
227
238
  failures.push(`PR #${row.pr} (ticket #${row.ticket}): tier=${row.tier}, but ${m.member} ran as ${m.subagentType} (${actual})`);
228
239
  }
229
240
  }
241
+ // `0 checked` is only a pass when nothing was left to compare. The tsv is
242
+ // cumulative, so the whole file's `checked` stays positive once any earlier
243
+ // run was verified: the guard reads the latest run_date alone. When none of
244
+ // that day's filled post-switch rows got a member row compared and some lost
245
+ // their join (a blank ticket there, a stale or absent member file), that run
246
+ // verified nothing. A several-member-rows skip is by design and neither
247
+ // counts as checked nor exempts the lost rows.
248
+ const latest = [...byDate.keys()].sort().at(-1);
249
+ const latestDay = latest === undefined ? null : byDate.get(latest);
250
+ if (latestDay && latestDay.checked === 0 && latestDay.lost.length > 0) {
251
+ failures.push(`nothing from ${latest}, the latest run_date with a filled tier, was checked: no implementer row in the member file for ${latestDay.lost.join(", ")}`);
252
+ }
230
253
  return { failures, skipped, checked };
231
254
  }
232
255
 
@@ -250,13 +273,13 @@ export function unrecordedReviewedPrs(ledger, rows) {
250
273
  // CLI
251
274
  // ---------------------------------------------------------------------------
252
275
 
253
- const USAGE = "usage: tier-outcomes.mjs append <pr> --class <c> --closed-own-ticket yes|no --minted-false-claim yes|no --note <text> "
276
+ const USAGE = "usage: tier-outcomes.mjs append <pr> --closed-own-ticket yes|no --minted-false-claim yes|no --note <text> "
254
277
  + "[--sizing light|heavy] [--profile <p>] [--loc <n>] [--files <n>] [--file <tsv>] [--member-outcomes <tsv>] [--ledger <path>]\n"
255
278
  + " tier-outcomes.mjs check [--live] [--file <tsv>] [--member-outcomes <tsv>] [--ledger <path>]";
256
279
 
257
280
  const die = makeDie(NAME);
258
281
  const APPEND_FLAGS = {
259
- class: "value", "closed-own-ticket": "value", "minted-false-claim": "value", note: "value",
282
+ "closed-own-ticket": "value", "minted-false-claim": "value", note: "value",
260
283
  sizing: "value", profile: "value", loc: "value", files: "value",
261
284
  };
262
285
  const FLAGS = { ...APPEND_FLAGS, file: "value", "member-outcomes": "value", ledger: "value", live: "bool" };
@@ -356,10 +379,10 @@ function append(pr, paths) {
356
379
  pr = String(Number(pr));
357
380
  if (has("live")) die("--live is a check flag");
358
381
  const fields = {
359
- class: arg("class"), closed_own_ticket: arg("closed-own-ticket"), minted_false_claim: arg("minted-false-claim"),
382
+ closed_own_ticket: arg("closed-own-ticket"), minted_false_claim: arg("minted-false-claim"),
360
383
  note: arg("note"), sizing: arg("sizing") ?? "", profile: arg("profile") ?? "", loc: arg("loc") ?? "", files: arg("files") ?? "",
361
384
  };
362
- for (const col of ["class", "closed_own_ticket", "minted_false_claim", "note"]) {
385
+ for (const col of ["closed_own_ticket", "minted_false_claim", "note"]) {
363
386
  if (fields[col] === null) die(`append needs --${col.replaceAll("_", "-")}\n${USAGE}`);
364
387
  }
365
388
  for (const [col, value] of Object.entries(fields)) {
@@ -368,7 +391,6 @@ function append(pr, paths) {
368
391
  for (const col of ["closed_own_ticket", "minted_false_claim"]) {
369
392
  if (!["yes", "no"].includes(fields[col])) die(`--${col.replaceAll("_", "-")} must be yes or no, got '${fields[col]}'`);
370
393
  }
371
- if (!/^[a-z][a-z-]*$/.test(fields.class)) die(`--class must be one lowercase word, got '${fields.class}'`);
372
394
  if (fields.sizing && !["light", "heavy"].includes(fields.sizing)) die(`--sizing must be light or heavy, got '${fields.sizing}'`);
373
395
  for (const col of ["loc", "files"]) {
374
396
  if (fields[col] && !isDigits(fields[col])) die(`--${col} must be a count, got '${fields[col]}'`);
@@ -438,4 +460,4 @@ function main() {
438
460
  }
439
461
  }
440
462
 
441
- if (import.meta.url === pathToFileURL(process.argv[1] || "").href) main();
463
+ if (isCLI(import.meta.url)) main();
@@ -6,14 +6,17 @@
6
6
  import { test } from "node:test";
7
7
  import assert from "node:assert/strict";
8
8
  import { spawnSync } from "node:child_process";
9
- import { mkdtempSync, mkdirSync, rmSync, writeFileSync, readFileSync, chmodSync, existsSync } from "node:fs";
9
+ import { mkdtempSync, mkdirSync, rmSync, writeFileSync, readFileSync, existsSync } from "node:fs";
10
10
  import { tmpdir } from "node:os";
11
11
  import { join } from "node:path";
12
12
  import { fileURLToPath } from "node:url";
13
13
  import { COLUMNS, TIER_SWITCH_DATE } from "./tier-outcomes.mjs";
14
14
  import { COLUMNS as MEMBER_COLUMNS } from "./member-outcomes.mjs";
15
+ import { writeExecStub } from "./exec-stub.mjs";
15
16
 
16
17
  const SCRIPT = fileURLToPath(new URL("./tier-outcomes.mjs", import.meta.url));
18
+ const LATER = "2026-10-03";
19
+ assert.ok(LATER > TIER_SWITCH_DATE);
17
20
  const PRE_SWITCH = "2026-09-17";
18
21
  assert.ok(PRE_SWITCH < TIER_SWITCH_DATE);
19
22
 
@@ -39,8 +42,7 @@ function fixture(t, { members = [], tierRows = [], ledger = null, closes = [10]
39
42
  t.after(() => rmSync(dir, { recursive: true, force: true }));
40
43
  const bin = join(dir, "bin");
41
44
  mkdirSync(bin);
42
- writeFileSync(join(bin, "gh"), '#!/bin/sh\nprintf \'%s\\n\' "$*" >> "$GH_LOG"\nprintf \'%s\\n\' "$GH_JSON"\n');
43
- chmodSync(join(bin, "gh"), 0o755);
45
+ writeExecStub(join(bin, "gh"), '#!/bin/sh\nprintf \'%s\\n\' "$*" >> "$GH_LOG"\nprintf \'%s\\n\' "$GH_JSON"\n');
44
46
  const f = {
45
47
  tier: join(dir, "tier-outcomes.tsv"),
46
48
  members: join(dir, "member-outcomes.tsv"),
@@ -64,7 +66,7 @@ function fixture(t, { members = [], tierRows = [], ledger = null, closes = [10]
64
66
  return { code: r.status, stdout: r.stdout, stderr: r.stderr };
65
67
  };
66
68
  f.append = (pr = "20", { note = "a note with spaces" } = {}) =>
67
- f.run("append", pr, "--class", "routine", "--closed-own-ticket", "yes", "--minted-false-claim", "no", "--note", note);
69
+ f.run("append", pr, "--closed-own-ticket", "yes", "--minted-false-claim", "no", "--note", note);
68
70
  f.dataRows = () => readFileSync(f.tier, "utf8").split("\n").filter((l) => l.trim() && !l.startsWith("#")).map((l) => l.split("\t"));
69
71
  f.ghCalls = () => (existsSync(f.ghLog) ? readFileSync(f.ghLog, "utf8").trim().split("\n").filter(Boolean) : []);
70
72
  return f;
@@ -96,6 +98,7 @@ test("append: the ledger's tier-ok token wins over member-outcomes.tsv", (t) =>
96
98
  assert.equal(rows[0][col("pr")], "20");
97
99
  assert.equal(rows[0][col("ticket")], "10", "the ticket comes from gh's closingIssuesReferences");
98
100
  assert.equal(rows[0][col("tier")], "alt");
101
+ assert.equal(rows[0][col("class")], "", "append writes the retired class column empty");
99
102
  // Stamped at ruling: the day `append` ran, in local time — either side of a
100
103
  // midnight the run straddled.
101
104
  assert.ok([before, after].includes(rows[0][col("run_date")]), `run_date ${rows[0][col("run_date")]} is not the day append ran`);
@@ -188,6 +191,31 @@ test("append: a member settled tier-mismatch on the ledger blanks the tier even
188
191
  assert.match(r.stderr, /the ledger settled impl-10=tier-mismatch/);
189
192
  });
190
193
 
194
+ test("append: a tier-unverifiable token is no verdict — the tier falls through to member-outcomes.tsv, never read as tier-ok", (t) => {
195
+ // The token's tail is definition-shaped on purpose: a reader that took
196
+ // `tier-unverifiable=` for a tier verdict would read `alt` off it, where
197
+ // tier-check's own `:no-transcript` tail names no tier and would hide that.
198
+ const f = fixture(t, {
199
+ ledger: { rows: ["#10 impl-10=PR#20 · tier-unverifiable=impl-10:fleet-implementer-alt"], dispatched: ["impl-10=PR#20"] },
200
+ members: [{ member: "impl-10", ticket: 10, type: "fleet-implementer" }],
201
+ });
202
+ const r = f.append();
203
+ assert.equal(r.code, 0, r.stderr);
204
+ assert.equal(f.dataRows()[0][col("tier")], "default");
205
+ assert.doesNotMatch(r.stderr, /tier-ok|tier-unverifiable/);
206
+ });
207
+
208
+ test("append: a replaced attempt's tier-unverifiable leaves the PR owner's own tier-ok standing", (t) => {
209
+ const f = fixture(t, {
210
+ ledger: {
211
+ rows: ["#10 impl-10=killed · tier-unverifiable=impl-10:no-transcript · impl-10-b=PR#20 · tier-ok=impl-10-b:fleet-implementer-alt"],
212
+ dispatched: ["impl-10=killed", "impl-10-b=PR#20"],
213
+ },
214
+ });
215
+ assert.equal(f.append().code, 0);
216
+ assert.equal(f.dataRows()[0][col("tier")], "alt");
217
+ });
218
+
191
219
  test("append: running it twice is a no-op that names the existing row and never asks gh", (t) => {
192
220
  const f = fixture(t, { members: [{ member: "impl-10", ticket: 10, type: "fleet-implementer" }] });
193
221
  assert.equal(f.append().code, 0);
@@ -237,6 +265,16 @@ test("append: a PR closing no issue is refused, not written with a blank ticket"
237
265
  assert.equal(readFileSync(f.tier, "utf8"), before);
238
266
  });
239
267
 
268
+ test("append: --class is refused as an unknown flag before anything is written", (t) => {
269
+ const f = fixture(t, { members: [{ member: "impl-10", ticket: 10, type: "fleet-implementer" }] });
270
+ const before = readFileSync(f.tier, "utf8");
271
+ const r = f.run("append", "20", "--class", "routine", "--closed-own-ticket", "yes", "--minted-false-claim", "no", "--note", "n");
272
+ assert.equal(r.code, 2);
273
+ assert.match(r.stderr, /unknown flag --class/);
274
+ assert.equal(readFileSync(f.tier, "utf8"), before);
275
+ assert.deepEqual(f.ghCalls(), [], "refused before gh was asked");
276
+ });
277
+
240
278
  // ---------------------------------------------------------------------------
241
279
  // check
242
280
  // ---------------------------------------------------------------------------
@@ -303,6 +341,121 @@ test("check: rows dated before the switch are skipped, whatever they say", (t) =
303
341
  assert.match(r.stdout, /skipped 1 before/);
304
342
  });
305
343
 
344
+ // The vacuous pass #2433 closes: the latest run's filled post-switch rows all
345
+ // lost their join to the member file, so nothing from it was compared and
346
+ // `0 checked` used to exit 0.
347
+ test("check: a run whose filled post-switch rows all lack a member row fails, naming them", (t) => {
348
+ // impl-327-2 is an implementer name parseMemberName leaves with a blank ticket.
349
+ const f = fixture(t, {
350
+ members: [{ member: "impl-327-2", ticket: "", type: "fleet-implementer" }],
351
+ tierRows: [
352
+ tierRow({ date: PRE_SWITCH, pr: 19, ticket: 326, tier: "opus" }),
353
+ tierRow({ pr: 20, ticket: 327, tier: "default" }),
354
+ tierRow({ pr: 21, ticket: 328, tier: "default" }),
355
+ tierRow({ pr: 22, ticket: 329, tier: "" }),
356
+ ],
357
+ });
358
+ const r = f.run("check");
359
+ assert.equal(r.code, 1, r.stdout + r.stderr);
360
+ assert.match(r.stderr, new RegExp(`FAIL nothing from ${TIER_SWITCH_DATE}, .*was checked: .*PR #20 \\(ticket #327\\), PR #21 \\(ticket #328\\)$`, "m"));
361
+ assert.doesNotMatch(r.stderr, /PR #19|PR #22/, "a pre-switch or blank-tier row is not a lost join");
362
+ assert.match(r.stdout, /0 checked/);
363
+ });
364
+
365
+ test("check: a missing member file fails a filled post-switch row the same way", (t) => {
366
+ const f = fixture(t, { tierRows: [tierRow({ pr: 20, ticket: 10, tier: "default" })] });
367
+ rmSync(f.members);
368
+ const r = f.run("check");
369
+ assert.equal(r.code, 1, r.stdout + r.stderr);
370
+ assert.match(r.stderr, /FAIL nothing from .*was checked: .*PR #20/);
371
+ });
372
+
373
+ // The tsv is cumulative, so an earlier run that checked fine must not hide a
374
+ // later run that lost every join.
375
+ test("check: a later run that lost every join fails even though an earlier run was checked", (t) => {
376
+ const f = fixture(t, {
377
+ members: [
378
+ { member: "impl-10", ticket: 10, type: "fleet-implementer" },
379
+ { member: "impl-327-2", ticket: "", type: "fleet-implementer" },
380
+ ],
381
+ tierRows: [
382
+ tierRow({ pr: 20, ticket: 10, tier: "default" }),
383
+ tierRow({ date: LATER, pr: 21, ticket: 327, tier: "default" }),
384
+ tierRow({ date: LATER, pr: 22, ticket: 328, tier: "default" }),
385
+ ],
386
+ });
387
+ const r = f.run("check");
388
+ assert.equal(r.code, 1, r.stdout + r.stderr);
389
+ assert.match(r.stderr, new RegExp(`FAIL nothing from ${LATER}, .*PR #21 .*PR #22`));
390
+ assert.match(r.stdout, /1 checked, 1 failed/);
391
+ });
392
+
393
+ // A several-member-rows skip is by design, but it is no comparison and must
394
+ // not cancel the no-member-row alarm.
395
+ test("check: a several-member-rows row does not hide lost joins beside it", (t) => {
396
+ const f = fixture(t, {
397
+ members: [
398
+ { member: "impl-11", ticket: 11, type: "task" },
399
+ { member: "impl-11-b", ticket: 11, type: "task" },
400
+ ],
401
+ tierRows: [
402
+ tierRow({ pr: 20, ticket: 327, tier: "default" }),
403
+ tierRow({ pr: 21, ticket: 328, tier: "default" }),
404
+ tierRow({ pr: 22, ticket: 329, tier: "default" }),
405
+ tierRow({ pr: 23, ticket: 11, tier: "default" }),
406
+ ],
407
+ });
408
+ const r = f.run("check");
409
+ assert.equal(r.code, 1, r.stdout + r.stderr);
410
+ assert.match(r.stderr, /FAIL nothing from .*PR #20 .*PR #21 .*PR #22/);
411
+ assert.doesNotMatch(r.stderr, /PR #23/);
412
+ assert.match(r.stdout, /0 checked.*3 with no member row, 1 with several member rows/);
413
+ });
414
+
415
+ // What the vacuity guard must ACCEPT: 0 checked is fine whenever no filled
416
+ // post-switch row of the latest run is missing its member row.
417
+ test("check: 0 checked still passes when every row is pre-switch, blank, or has several member rows", (t) => {
418
+ const f = fixture(t, {
419
+ members: [
420
+ { member: "impl-11", ticket: 11, type: "task" },
421
+ { member: "impl-11-b", ticket: 11, type: "task" },
422
+ ],
423
+ tierRows: [
424
+ tierRow({ date: PRE_SWITCH, pr: 20, ticket: 10, tier: "opus" }),
425
+ tierRow({ pr: 21, ticket: 10, tier: "" }),
426
+ tierRow({ pr: 22, ticket: 11, tier: "default" }),
427
+ ],
428
+ });
429
+ const r = f.run("check");
430
+ assert.equal(r.code, 0, r.stderr);
431
+ assert.match(r.stdout, /0 checked, 0 failed/);
432
+ });
433
+
434
+ test("check: one checked row keeps a no-member-row sibling a skip, not a failure", (t) => {
435
+ const f = fixture(t, {
436
+ members: [{ member: "impl-10", ticket: 10, type: "fleet-implementer" }],
437
+ tierRows: [tierRow({ pr: 20, ticket: 10, tier: "default" }), tierRow({ pr: 21, ticket: 99, tier: "default" })],
438
+ });
439
+ const r = f.run("check");
440
+ assert.equal(r.code, 0, r.stderr);
441
+ assert.match(r.stdout, /1 checked, 0 failed.*1 with no member row/);
442
+ });
443
+
444
+ // Only the latest run is held to the guard: rows of an earlier run that lost
445
+ // their join stay a skip once a later run was checked.
446
+ test("check: an earlier run's lost joins do not fail a later run that was checked", (t) => {
447
+ const f = fixture(t, {
448
+ members: [{ member: "impl-10", ticket: 10, type: "fleet-implementer" }],
449
+ tierRows: [
450
+ tierRow({ pr: 20, ticket: 327, tier: "default" }),
451
+ tierRow({ date: LATER, pr: 21, ticket: 10, tier: "default" }),
452
+ ],
453
+ });
454
+ const r = f.run("check");
455
+ assert.equal(r.code, 0, r.stdout + r.stderr);
456
+ assert.match(r.stdout, /1 checked, 0 failed.*1 with no member row/);
457
+ });
458
+
306
459
  test("check: a missing tier-outcomes.tsv file fails loudly instead of passing 0 checked", (t) => {
307
460
  const f = fixture(t, {
308
461
  members: [{ member: "impl-10", ticket: 10, type: "fleet-implementer" }],
@@ -19,14 +19,25 @@
19
19
  // that config is model-precedence #1 for task/eval dispatch (ADR
20
20
  // 0011), so a leftover `fleet-*` key there would silently win over
21
21
  // the definition's own `model:`, which ADR 0014 retires as a lever;
22
- // (d) `modelRoles.slow`/`modelRoles.task` resolving to the same model is
23
- // flagged as a notice — legal, but the alt-tier comparison
24
- // (fleet-implementer-alt) then controls nothing.
22
+ // (d) every definition's `:<level>` is one its role's current target
23
+ // runs at — in that model's `thinking` efforts in omp's own model
24
+ // catalog (`omp models --json`). omp clamps an unsupported level
25
+ // silently (`claude-haiku-4-5` has no `max`, so `@smol:max` would run
26
+ // at something else), tier-check would then read a level the
27
+ // definition never declared, and every row the definition ran would
28
+ // be inadmissible (spec 2026-09-28 § 2). A target the catalog does
29
+ // not list cannot be checked, so it fails too;
30
+ // (e) `modelRoles.slow`/`modelRoles.task` resolving to the same model is
31
+ // flagged as a notice — legal, but every `slow-*` cell then measures
32
+ // the same model as its `task-*` twin;
33
+ // (f) the agents directory holds at least one `*.agent.md` definition —
34
+ // with none, (a), (b) and (d) have nothing to check.
25
35
 
26
36
  import { readFileSync, readdirSync } from "node:fs";
27
37
  import { execFileSync } from "node:child_process";
28
38
  import { join, dirname } from "node:path";
29
- import { fileURLToPath, pathToFileURL } from "node:url";
39
+ import { fileURLToPath } from "node:url";
40
+ import { isCLI } from "./is-cli.mjs";
30
41
  import { makeDie, defineFlags } from "./arg.mjs";
31
42
 
32
43
  const NAME = "tier-roles";
@@ -117,17 +128,52 @@ export function modelsEqual(a, b) {
117
128
  // `modelRoles.slow` dotted into a record is `Unknown setting` on this box,
118
129
  // so a record-valued key is always read whole. Never writes.
119
130
  export function readOmpConfigValue(key) {
131
+ return readOmpJson(["config", "get", key, "--json"]).value;
132
+ }
133
+
134
+ // omp's model catalog, `{ models: [{ selector, thinking, … }] }` — `selector`
135
+ // is `<provider>/<id>`, `thinking` the efforts the model runs at, `null` for
136
+ // a model with none (measured, omp 18: `anthropic/claude-haiku-4-5` →
137
+ // minimal…xhigh, `anthropic/claude-opus-5-5` → low…max). Never refreshes.
138
+ export function readOmpModelCatalog() {
139
+ return readOmpJson(["models", "--json"]);
140
+ }
141
+
142
+ // A plain JSON object — not null, an array or a scalar. Every read here
143
+ // dereferences the parsed value (`.value`, `.models`), so anything else is
144
+ // refused where it is read, naming its source.
145
+ function isJsonObject(v) {
146
+ return typeof v === "object" && v !== null && !Array.isArray(v);
147
+ }
148
+
149
+ function readOmpJson(argv) {
150
+ const cmd = `omp ${argv.join(" ")}`;
120
151
  let out;
121
152
  try {
122
- out = execFileSync("omp", ["config", "get", key, "--json"], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] });
153
+ out = execFileSync("omp", argv, { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], maxBuffer: 64 * 1024 * 1024 });
123
154
  } catch (e) {
124
- throw new Error(`omp config get ${key} --json failed: ${e.stderr || e.message}`);
155
+ throw new Error(`${cmd} failed: ${e.stderr || e.message}`);
125
156
  }
157
+ let parsed;
126
158
  try {
127
- return JSON.parse(out).value;
159
+ parsed = JSON.parse(out);
128
160
  } catch (e) {
129
- throw new Error(`omp config get ${key} --json exited 0 but printed non-JSON: ${e.message}`);
161
+ throw new Error(`${cmd} exited 0 but printed non-JSON: ${e.message}`);
162
+ }
163
+ if (!isJsonObject(parsed)) {
164
+ throw new Error(`${cmd} exited 0 but printed ${parsed === null ? "null" : Array.isArray(parsed) ? "an array" : `a ${typeof parsed}`}, not a JSON object`);
130
165
  }
166
+ return parsed;
167
+ }
168
+
169
+ // The catalog entry a `modelRoles` target names — `selector` exactly, else
170
+ // the first entry `modelsEqual` accepts (a role value written without its
171
+ // provider prefix). `null` when the catalog does not list it.
172
+ export function catalogEntry(model, catalog) {
173
+ const models = Array.isArray(catalog?.models) ? catalog.models : [];
174
+ return models.find((m) => m?.selector === model)
175
+ ?? models.find((m) => typeof m?.selector === "string" && modelsEqual(model, m.selector))
176
+ ?? null;
131
177
  }
132
178
 
133
179
  // Every fleet definition's `name:` carries this prefix — enforced in CI by
@@ -144,17 +190,29 @@ function shellQuote(s) {
144
190
  return `'${s.replace(/'/g, `'\\''`)}'`;
145
191
  }
146
192
 
147
- // The operator's install (`modelRoles`, and any leftover
148
- // `task.agentModelOverrides`) against what the fleet's own definitions need.
149
- // `violations` stop a run (ADR 0014); `notices` never do — they flag a
150
- // hazard (the alt-tier pairing controlling nothing) that is legal
193
+ // What a malformed catalog value is, for a refusal that names the shape.
194
+ function kindOf(v) {
195
+ if (v === undefined) return "absent";
196
+ if (v === null) return "null";
197
+ return Array.isArray(v) ? "array" : typeof v;
198
+ }
199
+
200
+ // The operator's install (`modelRoles`, omp's model `catalog`, and any
201
+ // leftover `task.agentModelOverrides`) against what the fleet's own
202
+ // definitions need. `violations` stop a run (ADR 0014); `notices` never do —
203
+ // they flag a hazard (two roles' cells measuring one model) that is legal
151
204
  // configuration, just probably not what the operator meant.
152
- export function checkRoutes({ agentsDir, modelRoles, overrides }) {
205
+ export function checkRoutes({ agentsDir, modelRoles, overrides, catalog }) {
153
206
  const violations = [];
154
207
  const notices = [];
155
- const usedBy = {}; // role -> [file, ...]
208
+ const usedBy = {}; // role -> [{ file, level }, ...]
156
209
 
157
210
  const files = readdirSync(agentsDir).filter((f) => f.endsWith(".agent.md")).sort();
211
+ // A directory with no definitions has nothing to route, so every
212
+ // per-definition check below (the model route, the role's resolution, the
213
+ // level) would pass over it — a wrong `--agents` path that exists must not
214
+ // read as a healthy install.
215
+ if (files.length === 0) violations.push(`no *.agent.md definition found in ${agentsDir}`);
158
216
  for (const file of files) {
159
217
  const text = readFileSync(join(agentsDir, file), "utf8");
160
218
  const fm = parseFrontmatter(text);
@@ -162,13 +220,40 @@ export function checkRoutes({ agentsDir, modelRoles, overrides }) {
162
220
  violations.push(`${file}: model ${JSON.stringify(fm.model)} is not @<role>:<level>`);
163
221
  continue;
164
222
  }
165
- (usedBy[fm.role] ??= []).push(file);
223
+ (usedBy[fm.role] ??= []).push({ file, level: fm.level });
224
+ }
225
+
226
+ // A catalog with no `models` list, or an entry whose `thinking` is neither a
227
+ // list nor `null`, is a shape omp did not print — name it, rather than read
228
+ // it as "not listed" or "runs at no level" and blame the operator's roles.
229
+ const listed = Array.isArray(catalog?.models);
230
+ if (!listed && Object.keys(usedBy).length > 0) {
231
+ violations.push(`omp's model catalog has no models list (models: ${kindOf(catalog?.models)}) — cannot check the level of any definition`);
166
232
  }
167
233
 
168
234
  for (const role of ROLE_ORDER) {
169
235
  if (!usedBy[role]) continue;
170
- if (resolveRole(role, modelRoles) === null) {
171
- violations.push(`modelRoles.${role} is unset — needed by ${usedBy[role].join(", ")}`);
236
+ const users = usedBy[role].map((u) => u.file).join(", ");
237
+ const model = resolveRole(role, modelRoles);
238
+ if (model === null) {
239
+ violations.push(`modelRoles.${role} is unset — needed by ${users}`);
240
+ continue;
241
+ }
242
+ if (!listed) continue;
243
+ const entry = catalogEntry(model, catalog);
244
+ if (entry === null) {
245
+ violations.push(`modelRoles.${role} resolves to ${model}, which omp's model catalog does not list — cannot check the level of ${users}`);
246
+ continue;
247
+ }
248
+ if (entry.thinking !== null && !Array.isArray(entry.thinking)) {
249
+ violations.push(`omp's model catalog lists ${model} with thinking that is neither a list nor null (${kindOf(entry.thinking)}) — cannot check the level of ${users}`);
250
+ continue;
251
+ }
252
+ const efforts = entry.thinking ?? [];
253
+ for (const { file, level } of usedBy[role]) {
254
+ if (!efforts.includes(level)) {
255
+ violations.push(`${file}: level ${level} is not one modelRoles.${role}'s target ${model} runs at (thinking: ${efforts.join(", ") || "none"}) — omp would clamp it silently`);
256
+ }
172
257
  }
173
258
  }
174
259
 
@@ -189,7 +274,7 @@ export function checkRoutes({ agentsDir, modelRoles, overrides }) {
189
274
  const slowModel = resolveRole("slow", modelRoles);
190
275
  const taskModel = resolveRole("task", modelRoles);
191
276
  if (slowModel !== null && taskModel !== null && modelsEqual(slowModel, taskModel)) {
192
- notices.push(`modelRoles.slow and modelRoles.task both resolve to ${slowModel} — the alternate-tier comparison controls nothing`);
277
+ notices.push(`modelRoles.slow and modelRoles.task both resolve to ${slowModel} — every slow-* cell measures the same model as its task-* twin`);
193
278
  }
194
279
 
195
280
  return { violations, notices };
@@ -206,19 +291,21 @@ const { arg, has, sweep, stray } = defineFlags(die, {
206
291
  agents: "value",
207
292
  overrides: "value",
208
293
  "model-roles": "value",
294
+ catalog: "value",
209
295
  },
210
296
  });
211
297
 
212
- function loadJsonObject(path, configKey, flagName) {
298
+ // `--<flagName> <path>` when given, else the live install read `readLive()`
299
+ // performs (`source` names it in a refusal).
300
+ function loadJsonObject(path, readLive, source, flagName) {
213
301
  let value;
214
302
  try {
215
- value = path ? JSON.parse(readFileSync(path, "utf8")) : readOmpConfigValue(configKey);
303
+ value = path ? JSON.parse(readFileSync(path, "utf8")) : readLive();
216
304
  } catch (e) {
217
305
  die(e.message);
218
306
  }
219
- if (typeof value !== "object" || value === null || Array.isArray(value)) {
220
- const source = path ? `--${flagName} ${path}` : `omp config get ${configKey} --json`;
221
- die(`${source} must be a JSON object, got ${Array.isArray(value) ? "an array" : typeof value}`);
307
+ if (!isJsonObject(value)) {
308
+ die(`${path ? `--${flagName} ${path}` : source} must be a JSON object, got ${value === null ? "null" : Array.isArray(value) ? "an array" : typeof value}`);
222
309
  }
223
310
  return value;
224
311
  }
@@ -228,16 +315,18 @@ function main() {
228
315
  stray();
229
316
 
230
317
  const check = has("check");
231
- if (!check) die("--check is required (usage: tier-roles.mjs --check [--agents <dir>] [--model-roles <path>] [--overrides <path>])");
318
+ if (!check) die("--check is required (usage: tier-roles.mjs --check [--agents <dir>] [--model-roles <path>] [--overrides <path>] [--catalog <path>])");
232
319
 
233
320
  const agentsDir = arg("agents") ?? join(SCRIPT_DIR, "..", "agents");
234
- const overrides = loadJsonObject(arg("overrides"), "task.agentModelOverrides", "overrides");
235
- const modelRoles = loadJsonObject(arg("model-roles"), "modelRoles", "model-roles");
321
+ const config = (key) => [() => readOmpConfigValue(key), `omp config get ${key} --json`];
322
+ const overrides = loadJsonObject(arg("overrides"), ...config("task.agentModelOverrides"), "overrides");
323
+ const modelRoles = loadJsonObject(arg("model-roles"), ...config("modelRoles"), "model-roles");
324
+ const catalog = loadJsonObject(arg("catalog"), readOmpModelCatalog, "omp models --json", "catalog");
236
325
 
237
- const { violations, notices } = checkRoutes({ agentsDir, modelRoles, overrides });
326
+ const { violations, notices } = checkRoutes({ agentsDir, modelRoles, overrides, catalog });
238
327
 
239
328
  if (violations.length === 0) {
240
- console.log("tier-roles: every definition routes to a resolvable model, no shadowing override");
329
+ console.log("tier-roles: every definition routes to a resolvable model at a level it runs, no shadowing override");
241
330
  for (const n of notices) console.log(`tier-roles: notice: ${n}`);
242
331
  process.exit(0);
243
332
  }
@@ -247,4 +336,4 @@ function main() {
247
336
  process.exit(1);
248
337
  }
249
338
 
250
- if (import.meta.url === pathToFileURL(process.argv[1] || "").href) main();
339
+ if (isCLI(import.meta.url)) main();