@feigi/fleet-ctl 3.17.6 → 3.18.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feigi/fleet-ctl",
-  "version": "3.17.6",
+  "version": "3.18.0",
   "description": "Agent fleet: run-team controller, merge bot, PR reviewer, and the ticket pipeline they share",
   "license": "Apache-2.0",
   "repository": {

package/scripts/compute-board.mjs

@@ -25,7 +25,7 @@
 // here, main() being guarded on argv[1].
 import { assessBeat, isStalled, stallReport } from "./fleet-state.mjs";
 import { parseToken, HALT_CAUSES } from "./ledger-grammar.mjs";
-import { PR_MENTION, REVIEWED, unlabelledFinishers } from "./fleet-tick.mjs";
+import { PR_MENTION, REVIEWED, latestFinisherAttempts } from "./fleet-tick.mjs";
 
 // A ledger row is freeform, controller-authored text. Two real examples:
 //   #332 impl-332=PR#344 → PR#344 → MERGED 73b356de
@@ -61,9 +61,10 @@ import { PR_MENTION, REVIEWED, unlabelledFinishers } from "./fleet-tick.mjs";
 // that outcome as a severity-4 flag while the PR sits in REVIEW. The halt is
 // the finisher working correctly — it refused to label — so the PR has no
 // `ready-to-merge` until the controller resolves the cause; the flag is what
-// puts it in front of a human. "Latest" is laterAttempt's reading, among the
-// finisher tokens bound to the row's own PR: a live `-b` after the halt
-// clears it.
+// puts it in front of a human. "Latest" is the reading `unlabelled` and
+// `finisher:*` below share (latestFinisherAttempts, off the same rows and
+// `## Dispatched` tokens): a live `-b` after the halt clears it, and a later
+// attempt settled anywhere replaces it.
 //
 // #2331: a PR whose finisher settled `labelled` while the open list shows no
 // `ready-to-merge` on it carries a severity-4 `unlabelled` flag in REVIEW —
@@ -72,6 +73,13 @@ import { PR_MENTION, REVIEWED, unlabelledFinishers } from "./fleet-tick.mjs";
 // reads, so the card is flagged whenever the tick is repairing it or escalated
 // it — including after a whole-line `row` rewrite dropped the settled token.
 //
+// A PR in REVIEW whose latest finisher attempt settled `failed` or `killed`
+// while it is open and lacks `ready-to-merge` carries `finisher:failed` or
+// `finisher:killed`, severity 4 — read off the same latest attempt and stretch
+// as `unlabelled`, so the tick and the cockpit never read it two ways. The tick
+// dispatches nothing for these; the flag is what puts the PR in front of a
+// human. The names are not the bare `killed` flag, which is the implementer's.
+//
 // A PR's review is not a member (#1773 §7): `review=wf:<runId>` is a Workflow
 // with nobody to name, while `review=member:<name>` and
 // `review=fallback:<name>` name a runner member, live until the token is
@@ -288,14 +296,15 @@ export function deriveFlags(parsed, ctx) {
   // escalated halt by labelling or merging it, the flag has nothing to ask.
   // `past-pin`'s own automatic resolution (SKILL.md "Resolving a finisher
   // halt") re-reviews the PR through fleet-tick.mjs, but a returned
-  // `reviewed=` does not itself clear `finisherOutcome` here — the halt is
+  // `reviewed=` does not itself clear the latest finisher outcome here — the halt is
   // answered only once "its result reaches a fresh finisher through the
   // same gate" (ibid.), so the card stays flagged, same as any other halt
   // cause, until that finisher settles. reviewBacklog below reads the same
   // gap the other way: still counted due for a fresh review while this flag
   // is up and the head has not caught up to what was reviewed.
-  if (parsed.finisherOutcome?.startsWith("halted:") && ctx.column === "REVIEW") flags.push(parsed.finisherOutcome);
-  if (ctx.unlabelled && ctx.column === "REVIEW") flags.push("unlabelled");
+  if (ctx.finisher?.startsWith("halted:") && ctx.column === "REVIEW") flags.push(ctx.finisher);
+  if (ctx.finisher === "labelled" && ctx.column === "REVIEW") flags.push("unlabelled");
+  if ((ctx.finisher === "failed" || ctx.finisher === "killed") && ctx.column === "REVIEW") flags.push(`finisher:${ctx.finisher}`);
   const limit = STALE_MS[ctx.column];
   if (limit != null && ctx.sinceEnteredStage != null && ctx.now - ctx.sinceEnteredStage > limit) {
     flags.push("stale");
@@ -325,6 +334,7 @@ function titleFor(issue, pr, issues) {
 
 const FLAG_SEVERITY = {
   "red-ci": 5, "ledger-error": 5, killed: 4, "tier-mismatch": 4, blocked: 4, "sha-off-branch": 4, unlabelled: 4,
+  "finisher:failed": 4, "finisher:killed": 4,
   ...Object.fromEntries(HALT_CAUSES.map((c) => [`halted:${c}`, 4])),
   stale: 1,
 };
@@ -453,7 +463,7 @@ export function computeBoard(inputs) {
 
   const parsed = (ledger.rows || []).map(parseRow).filter(Boolean);
   const unqueued = new Set(prs.filter((p) => p.state === "OPEN" && !(p.labels || []).includes("ready-to-merge")).map((p) => p.number));
-  const unlabelled = new Set(unlabelledFinishers({ rows: ledger.rows || [], dispatched: ledger.dispatched || [] }, unqueued).map((u) => u.pr));
+  const latestFinishers = new Map(latestFinisherAttempts({ rows: ledger.rows || [], dispatched: ledger.dispatched || [] }, unqueued).map((u) => [u.pr, u.outcome]));
   const rowIssues = new Set();
   const tickets = [];
 
@@ -468,7 +478,7 @@ export function computeBoard(inputs) {
     rowIssues.add(p.issue);
     const sinceEnteredStage = stageEntry(prevTicket, column, now);
     const ciState = p.pr != null ? (ci[p.pr] ?? "unknown") : null;
-    const flags = deriveFlags(p, { ci: ciState, column, sinceEnteredStage, now, unlabelled: unlabelled.has(p.pr) });
+    const flags = deriveFlags(p, { ci: ciState, column, sinceEnteredStage, now, finisher: latestFinishers.get(p.pr) });
     tickets.push({
       issue: p.issue,
       title: titleFor(p.issue, pr, issues),

package/scripts/compute-board.test.mjs

@@ -683,6 +683,97 @@ test("#2331: the cockpit reads `## Dispatched` as the tick does — a settled fi
   }
 });
 
+// A finisher that settled `failed` or `killed` leaves its PR unlabelled with
+// nothing dispatched for it, so the cockpit is what surfaces it — one token per
+// outcome, distinct from each other, from `unlabelled`, and from the
+// implementer's bare `killed`.
+test("a PR whose latest finisher settled failed or killed flags finisher:failed / finisher:killed, ranked exactly at severity 4", () => {
+  for (const outcome of ["failed", "killed"]) {
+    const b = computeBoard(reproInputs({
+      rows: ["#904 impl-904=killed", `#941 impl-941=PR#931 · finisher-pr-931=${outcome}`, "#942 impl-942=killed",
+        "#943 impl-943=PR#932 · held-behind:#931"],
+      prev: { tickets: [] },
+    }));
+    assert.equal(card(b, 941).column, "REVIEW", outcome);
+    assert.deepEqual(card(b, 941).flags, [`finisher:${outcome}`], outcome);
+    assert.deepEqual(b.attention.map((t) => t.issue), [904, 941, 942, 943], outcome);
+  }
+});
+
+test("the finisher's latest attempt decides failed / killed — by suffix order, never token position", () => {
+  for (const [row, flags] of [
+    ["#941 impl-941=PR#931 · finisher-pr-931=labelled · finisher-pr-931-b=failed", ["finisher:failed"]],
+    ["#941 impl-941=PR#931 · finisher-pr-931-b=killed · finisher-pr-931=labelled", ["finisher:killed"]],
+    ["#941 impl-941=PR#931 · finisher-pr-931=halted:rebase · finisher-pr-931-b=failed", ["finisher:failed"]],
+    // A later attempt settles it: a labelled one on a PR still without the label is `unlabelled`'s,
+    // a halt is its own flag, a live repair flags nothing.
+    ["#941 impl-941=PR#931 · finisher-pr-931=failed · finisher-pr-931-b=labelled", ["unlabelled"]],
+    ["#941 impl-941=PR#931 · finisher-pr-931-b=labelled · finisher-pr-931=killed", ["unlabelled"]],
+    ["#941 impl-941=PR#931 · finisher-pr-931=killed · finisher-pr-931-b=halted:other", ["halted:other"]],
+    ["#941 impl-941=PR#931 · finisher-pr-931=failed · finisher-pr-931-b", []],
+    ["#941 impl-941=PR#931 · finisher-pr-931-b · finisher-pr-931=failed", []],
+    // An unsettled bare copy beside the settled token is what a whole-line rewrite leaves.
+    ["#941 impl-941=PR#931 · finisher-pr-931 · finisher-pr-931=failed", ["finisher:failed"]],
+  ]) {
+    const { card: c } = cardFor(row, 941);
+    assert.equal(c.column, "REVIEW", row);
+    assert.deepEqual(c.flags, flags, row);
+  }
+});
+
+test("a failed or killed finisher flags nothing once the label is on, the PR is shut, or a label-off accounts for it", () => {
+  for (const outcome of ["failed", "killed"]) {
+    const row = `#941 impl-941=PR#931 · finisher-pr-931=${outcome}`;
+    const ready = computeBoard(reproInputs({ rows: [row], prs: [openPr(931, ["ready-to-merge"])], prev: { tickets: [] } }));
+    assert.equal(card(ready, 941).column, "READY");
+    assert.deepEqual(card(ready, 941).flags, [], outcome);
+    const merged = computeBoard(reproInputs({ rows: [row], prs: [], merged: [931], prev: { tickets: [] } }));
+    assert.deepEqual(card(merged, 941).flags, [], outcome);
+    for (const state of ["CLOSED", "MERGED"]) {
+      const shut = computeBoard(reproInputs({ rows: [row], prs: [{ ...openPr(931), state }], prev: { tickets: [] } }));
+      assert.deepEqual(card(shut, 941).flags, [], `${outcome} ${state}`);
+    }
+    // REVIEW only: a card the row already carries past review flags nothing.
+    const past = cardFor(`#941 impl-941=PR#931 → MERGED abc1234 · finisher-pr-931=${outcome}`, 941).card;
+    assert.equal(past.column, "MERGED");
+    assert.deepEqual(past.flags, [], outcome);
+  }
+  // A label-off naming a later attempt than the failed one cancels it; a failed attempt after the latest label-off flags.
+  for (const [row, flags] of [
+    ["#941 impl-941=PR#931 · finisher-pr-931=failed label-off=finisher-pr-931", []],
+    ["#941 impl-941=PR#931 · finisher-pr-931=failed · finisher-pr-931-b=failed label-off=finisher-pr-931-b", []],
+    ["#941 impl-941=PR#931 · finisher-pr-931=labelled label-off=finisher-pr-931 · finisher-pr-931-b=failed", ["finisher:failed"]],
+    ["#941 impl-941=PR#931 · finisher-pr-931=failed label-off=finisher-pr-931 · finisher-pr-931-b=killed", ["finisher:killed"]],
+    ["#941 impl-941=PR#931 · finisher-pr-931-b=failed label-off=finisher-pr-931", ["finisher:failed"]],
+  ]) {
+    assert.deepEqual(cardFor(row, 941).card.flags, flags, row);
+  }
+});
+
+test("a failed or killed finisher token on another PR's row is a stray; one settled only in `## Dispatched` still flags", () => {
+  const stray = ["#941 impl-941=PR#931", "#942 impl-942=PR#932 · finisher-pr-931=failed"];
+  const b = computeBoard(reproInputs({ rows: stray, prev: { tickets: [] } }));
+  assert.deepEqual(card(b, 941).flags, []);
+  const masked = ["#941 impl-941=PR#931 · finisher-pr-931=failed", "#942 impl-942=PR#932 · finisher-pr-931-b"];
+  const m = computeBoard(reproInputs({ rows: masked, prev: { tickets: [] } }));
+  assert.deepEqual(card(m, 941).flags, ["finisher:failed"]);
+  const rows = ["#941 impl-941=PR#931 → PR#931 · reviewed=abc1234:0/0/0"];
+  const d = computeBoard(reproInputs({ rows, ledger: { rows, dispatched: ["impl-941=PR#931", "finisher-pr-931=killed"], filed: [], ruled: [] }, prev: { tickets: [] } }));
+  assert.deepEqual(card(d, 941).flags, ["finisher:killed"]);
+});
+
+test("the latest finisher attempt decides the halt flag too, whichever of the row and `## Dispatched` holds it", () => {
+  const flagsFor = (rowToken, dispatchedToken) => {
+    const rows = [`#941 impl-941=PR#931 → PR#931 · reviewed=abc1234:0/0/0 · ${rowToken}`];
+    const b = computeBoard(reproInputs({ rows, ledger: { rows, dispatched: ["impl-941=PR#931", dispatchedToken], filed: [], ruled: [] }, prev: { tickets: [] } }));
+    return card(b, 941).flags;
+  };
+  assert.deepEqual(flagsFor("finisher-pr-931=halted:rebase", "finisher-pr-931-b=failed"), ["finisher:failed"]);
+  assert.deepEqual(flagsFor("finisher-pr-931=failed", "finisher-pr-931-b=halted:rebase"), ["halted:rebase"]);
+  // ACCEPT side: the halt still flags when it is the latest attempt in the row alone.
+  assert.deepEqual(flagsFor("finisher-pr-931=halted:rebase", "finisher-pr-931=halted:rebase"), ["halted:rebase"]);
+});
+
 test("#1820: a live implementer is IMPLEMENTING, and still earns stale", () => {
   const b = computeBoard(reproInputs());
   assert.equal(card(b, 906).column, "IMPLEMENTING");

package/scripts/finisher-died-prose.test.mjs

@@ -0,0 +1,31 @@
+// A finisher settled `failed` or `killed` leaves its PR without
+// `ready-to-merge` and nothing dispatches for it. The ruling: flag only — the
+// cockpit raises one severity-4 token per outcome and the controller resolves
+// it by hand, never an automatic retry and never the label itself. Each pin is
+// sliced to the paragraph or table row carrying its claim.
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { paragraph } from "./prose-pin.mjs";
+
+const RUN_TEAM = readFileSync(join(import.meta.dirname, "..", "skills", "run-team", "SKILL.md"), "utf8");
+const flat = (s) => s.replace(/\s+/g, " ");
+
+test("a finisher that died is flagged by the cockpit and resolved by hand — no automatic retry, no label from the controller", () => {
+  const para = flat(paragraph(RUN_TEAM, "**Resolving a finisher that died.**", "died-finisher resolution"));
+  assert.match(para, /`finisher:failed` or `finisher:killed` at severity 4/);
+  assert.match(para, /neither the implementer's bare `killed` flag nor `unlabelled`/);
+  assert.match(para, /the tick dispatches no finisher for it/);
+  assert.match(para, /You resolve it by hand: investigate why the finisher died, or dispatch the next-suffix finisher after the finisher gate/);
+  assert.match(para, /You never add the label yourself, and the tick does not retry\./);
+});
+
+test("the finisher-report wake row records failed and killed and points at the by-hand resolution", () => {
+  const row = flat(RUN_TEAM.split("\n").find((l) => l.startsWith("| Finisher report (failed / killed) |")) ?? "");
+  assert.match(row, /settle finisher-pr-<M>=failed` for a finisher that crashed or gave up, `=killed` for one that was killed/);
+  assert.match(row, /the tick prints no dispatch for it/);
+  assert.match(row, /`finisher:failed` \/ `finisher:killed` at severity 4/);
+  assert.match(row, /\*\*Resolving a finisher that died\*\*/);
+});

package/scripts/fleet-tick-prose.test.mjs

@@ -84,6 +84,7 @@ test("the record-before-tick table names every wake, each with what it records",
     ["Finisher report", ["settle finisher-pr-<M>=labelled"]],
     // #2083: a halt is its own outcome, escalated on the PR itself, then resolved by cause.
     ["Finisher report (halted)", ["settle finisher-pr-<M>=halted:<cause>", "gh pr comment <M>", "**Resolving a finisher halt**"]],
+    ["Finisher report (failed / killed)", ["settle finisher-pr-<M>=failed", "**Resolving a finisher that died**"]],
     ["Label seen (persistent Monitor)", ["nothing to record"]],
     ["CI run terminal", ["ci=<run-id>:<attempt>:<conclusion>", "finisher gate"]],
     ["Merge-bot pass report", ["held-behind:#<lower>", "settle merge-bot-<n>=done", "reap.sh --apply"]],

package/scripts/fleet-tick.mjs

@@ -370,23 +370,23 @@ export function rowNums(text) {
   return { keyNum, pr: mention ? Number(mention[1]) : keyNum };
 }
 
-// #2331: PRs whose finisher settled `labelled` while the open list shows no
-// `ready-to-merge` on them — `unqueued` is the set of open PR numbers without
+// PRs whose finisher attempts are read off the ledger while the open list shows
+// no `ready-to-merge` on them — `unqueued` is the set of open PR numbers without
 // it, so the label shape stays the caller's (gh's `{name}` here, plain names in
 // compute-board.mjs). A PR's attempts are its `finisher-pr-M` tokens in
 // `## Dispatched` and on PR M's own rows; a copy on another PR's row is a
 // stray (#2329) that neither makes nor masks a miss. Settled anywhere among
 // those is settled. The STRETCH is the attempts whose retry suffix sorts after
 // the highest one a `label-off=` names — suffix order ("" < "b" < …), never
-// row position (#2083). A miss is the stretch's latest attempt settled
-// `labelled`; `labelled` lists every attempt in the stretch that settled so,
-// the count the tick splits one repair from an escalation on.
+// row position (#2083). `attempts` is the stretch in suffix order and `outcome`
+// the outcome of its latest attempt (null while that one is live); a PR whose
+// stretch is empty is left out.
 //
 // Lenient on purpose — a malformed token is skipped, never thrown — because
 // the cockpit reads the same ledger and flags rather than refuses; deriveRun
 // has already refused anything malformed before it calls this.
-/** @returns {{pr: number, labelled: string[]}[]} ascending by PR */
-export function unlabelledFinishers({ rows, dispatched }, unqueued) {
+/** @returns {{pr: number, outcome: string|null, attempts: {name: string, retry: string, outcome: string|null}[]}[]} ascending by PR */
+export function latestFinisherAttempts({ rows, dispatched }, unqueued) {
   const attempts = new Map();
   const offs = new Map();
   const add = (t) => {
@@ -417,12 +417,23 @@ export function unlabelledFinishers({ rows, dispatched }, unqueued) {
     const stretch = [...byName.values()]
       .filter((a) => !offs.has(n) || a.retry > offs.get(n))
       .sort((a, b) => (a.retry < b.retry ? -1 : a.retry > b.retry ? 1 : 0));
-    if (stretch.at(-1)?.outcome !== "labelled") continue;
-    out.push({ pr: n, labelled: stretch.filter((a) => a.outcome === "labelled").map((a) => a.name) });
+    if (stretch.length === 0) continue;
+    out.push({ pr: n, outcome: stretch.at(-1).outcome, attempts: stretch });
   }
   return out.sort((a, b) => a.pr - b.pr);
 }
 
+// #2331: PRs whose latest finisher attempt settled `labelled` while the open
+// list shows no `ready-to-merge` on them. `labelled` lists every attempt in the
+// stretch that settled so, the count the tick splits one repair from an
+// escalation on.
+/** @returns {{pr: number, labelled: string[]}[]} ascending by PR */
+export function unlabelledFinishers(ledger, unqueued) {
+  return latestFinisherAttempts(ledger, unqueued)
+    .filter((u) => u.outcome === "labelled")
+    .map((u) => ({ pr: u.pr, labelled: u.attempts.filter((a) => a.outcome === "labelled").map((a) => a.name) }));
+}
+
 export function deriveRun({ rows, dispatched, drain }, prs) {
   // One entry per member name across `## Dispatched` and every row. A member
   // settled ANYWHERE is settled: `settle` is the only writer of an outcome, and

package/scripts/fleet-tick.test.mjs

@@ -561,7 +561,9 @@ test("deriveRun: only a settled-labelled LATEST attempt is a miss — live, halt
     `${base} · finisher-pr-40-b · finisher-pr-40=labelled`, // the same, by suffix not position
     `${base} · finisher-pr-40=halted:rebase`, // a halt refused to label, and says so
     `${base} · finisher-pr-40=labelled · finisher-pr-40-b=halted:past-pin`,
-    `${base} · finisher-pr-40=failed`, // #2359's, not this check's
+    `${base} · finisher-pr-40=failed`, // died, no label: the cockpit's to flag, no repair here
+    `${base} · finisher-pr-40=killed`,
+    `${base} · finisher-pr-40=labelled · finisher-pr-40-b=failed`,
     `${base} · fix-pr-40=applied:def5678`, // no finisher at all
   ]) {
     assert.deepEqual(run({ rows: [row] }, open).unlabelled, [], row);
@@ -569,6 +571,15 @@ test("deriveRun: only a settled-labelled LATEST attempt is a miss — live, halt
   assert.deepEqual(run({ rows: [LABELLED] }, []).unlabelled, [], "a PR off the open list is nobody's work");
 });
 
+test("deriveRun: a failed or killed latest finisher prints no DISPATCH or ESCALATE finisher line", () => {
+  const open = [pr(40, ["minor"], [10])];
+  for (const outcome of ["failed", "killed"]) {
+    const r = run({ rows: [`${LABELLED} · finisher-pr-40-b=${outcome}`] }, open);
+    assert.deepEqual(r.unlabelled, [], outcome);
+    assert.deepEqual(reconcile({ ...state(), ...r }).filter((x) => x.role === "reviewers").map((x) => x.action), ["IDLE OK"], outcome);
+  }
+});
+
 test("deriveRun: a label-off'd attempt is the controller's own removal — nothing until a later attempt labels again", () => {
   const open = [pr(40, ["minor"], [10])];
   for (const row of [

package/scripts/review-core.mjs

@@ -755,7 +755,7 @@ export async function runReview(host, args) {
     mkdir -p "${runRootParent}" || { echo SNAPSHOT_RUNROOT_FAILED; exit 1; }
     RUN=$(mktemp -d "${runRootPrefix}XXXXXXXX") || { echo SNAPSHOT_RUNROOT_FAILED; exit 1; }
     echo SNAPSHOT_RUN_ROOT="$RUN"
-    find "${runRootParent}" -maxdepth 1 -type d -name 'run-*' -mtime +7 -exec sh -c 'rc=0; for d; do chmod -R u+rwx "$d" 2>/dev/null; if command -v chflags >/dev/null 2>&1; then chflags -R nouchg "$d" 2>/dev/null; chmod -R u+rwx "$d" 2>/dev/null; fi; rm -rf "$d" || rc=1; done; exit $rc' sh {} + || echo SNAPSHOT_PRUNE_FAILED
+    find "${runRootParent}" -maxdepth 1 -type d -name 'run-*' -mtime +7 -exec sh -c 'rc=0; ph="{""}"; for d; do chmod -R u+rwx "$d" 2>/dev/null; m=$?; if command -v chflags >/dev/null 2>&1; then [ $m -eq 0 ] || { chflags nouchg,nouappnd "$d"; chmod u+rwx "$d"; find "$d" -type d -exec chflags nouchg,nouappnd "$ph" ";" -exec chmod u+rwx "$ph" ";"; }; chflags -R nouchg,nouappnd "$d" 2>/dev/null; fi; rm -rf "$d" || rc=1; done; exit $rc' sh {} + || echo SNAPSHOT_PRUNE_FAILED
     SHA=$(git -C ${worktree} rev-parse --short HEAD) || { echo SNAPSHOT_REVPARSE_FAILED; exit 1; }
     SNAP="$RUN/snapshot-$SHA"
     echo SNAPSHOT_DEST="$SNAP"

package/scripts/snapshot-repo.test.mjs

@@ -164,31 +164,48 @@ const renderPrune = (path, runRootParent) =>
 
 /**
  * A `run-` root under `parent` holding a `locked` directory with no permission
- * bits at all, aged `days`. On darwin, optionally, the file inside it carries
- * `uchg` (`immutableFile`), or `locked` itself does (`immutableDir`) — set
- * after the mode, so the directory's mode cannot be changed back until the
- * flag is gone. Aged last: creating children sets the root's mtime to now.
+ * bits at all, aged `days`. `nested` puts a second such directory, `deeper`,
+ * inside `locked`, and the file moves into it. On darwin, optionally, the file
+ * carries `fileFlag` (`uchg` or `uappnd`), and every locked directory carries
+ * `dirFlag` — set after the mode, so a directory's mode cannot be changed back
+ * until the flag is gone. `rootFlag` locks the run root itself the same way,
+ * with no permission bits and that flag. `fileMode` is the file's own mode:
+ * one that already grants owner rwx leaves `chmod -R u+rwx` nothing to change
+ * on the file, so that pass exits 0 over a flagged file. Aged before the root
+ * is locked: creating children sets the root's mtime to now, and a flagged
+ * root refuses `utimes`.
  */
-function lockedRunRoot(parent, name, days, { immutableFile = false, immutableDir = false } = {}) {
+function lockedRunRoot(parent, name, days, { fileFlag = null, fileMode = 0o644, dirFlag = null, rootFlag = null, nested = false } = {}) {
   const root = join(parent, name);
   const locked = join(root, "locked");
-  mkdirSync(locked, { recursive: true });
-  writeFileSync(join(locked, "inner.txt"), "");
-  if (immutableFile) execFileSync("chflags", ["uchg", join(locked, "inner.txt")]);
-  chmodSync(locked, 0);
-  if (immutableDir) execFileSync("chflags", ["uchg", locked]);
+  const dirs = nested ? [join(locked, "deeper"), locked] : [locked];
+  mkdirSync(dirs[0], { recursive: true });
+  const file = join(dirs[0], "inner.txt");
+  writeFileSync(file, "");
+  chmodSync(file, fileMode);
+  if (fileFlag) execFileSync("chflags", [fileFlag, file]);
+  // Deepest first: once a directory is 000, nothing below it can be reached.
+  for (const dir of dirs) {
+    chmodSync(dir, 0);
+    if (dirFlag) execFileSync("chflags", [dirFlag, dir]);
+  }
   const when = (Date.now() - days * DAY) / 1000;
   utimesSync(root, when, when);
+  if (rootFlag) {
+    chmodSync(root, 0);
+    execFileSync("chflags", [rootFlag, root]);
+  }
   return root;
 }
 
 /**
  * Undoes `lockedRunRoot` so the scratch teardown can remove what a prune left.
- * The second `chmod` is for `immutableDir`: its mode is only changeable once
- * `chflags` has cleared the flag.
+ * A flagged directory's mode is only changeable once its flag is gone, and a
+ * directory below it only reachable once that mode is back, so each directory
+ * is unlocked before `find` descends into it.
  */
 function unlock(parent) {
-  spawnSync("sh", ["-c", 'chmod -R u+rwx "$1"; if command -v chflags >/dev/null; then chflags -R nouchg "$1"; chmod -R u+rwx "$1"; fi; :', "sh", parent], { env: ENV });
+  spawnSync("sh", ["-c", 'chmod -R u+rwx "$1"; if command -v chflags >/dev/null; then find "$1" -type d -exec chflags nouchg,nouappnd {} ";" -exec chmod u+rwx {} ";"; chflags -R nouchg,nouappnd "$1"; fi; :', "sh", parent], { env: ENV });
 }
 
 /**
@@ -596,8 +613,8 @@ for (const [name, path] of SOURCES) {
   // #2322. A fixture that drops its own read bit, or a darwin file carrying the
   // user-immutable flag, made the plain `rm -rf` fail on that root — so it
   // stayed forever and every later review of the PR printed
-  // SNAPSHOT_PRUNE_FAILED. The prune now restores owner rwx and clears `uchg`
-  // on the roots it SELECTED, then removes them.
+  // SNAPSHOT_PRUNE_FAILED. The prune now restores owner rwx and clears darwin's
+  // user flags on the roots it SELECTED, then removes them.
   test(`${name}: the prune removes an aged run root a chmod 000 directory used to wedge`, (t) => {
     const parent = join(scratch(t, "snapshot-repo-prune-perm-"), "pr7");
     const stale = lockedRunRoot(parent, "run-staleaa", 9);
@@ -640,31 +657,75 @@ for (const [name, path] of SOURCES) {
     }
   });
 
-  test(`${name}: on darwin the prune removes an aged run root holding a uchg file inside a chmod 000 directory`, { skip: process.platform !== "darwin" && "chflags/uchg is darwin's" }, (t) => {
-    const parent = join(scratch(t, "snapshot-repo-prune-uchg-"), "pr7");
-    const stale = lockedRunRoot(parent, "run-staleaa", 9, { immutableFile: true });
-    try {
-      const r = spawnSync("sh", ["-c", renderPrune(path, parent)], { env: ENV, encoding: "utf8" });
-
-      assert.doesNotMatch(r.stdout ?? "", /SNAPSHOT_PRUNE_FAILED/, `the prune named a failure on a root it can normalise: ${r.stderr}`);
-      assert.ok(!existsSync(stale), "an aged run root holding a uchg file survived the prune — the immutable flag wedges `rm -rf` (EPERM) and the root stays forever");
-    } finally {
-      unlock(parent);
-    }
-  });
+  // #2322, #2370. On darwin a user flag wedges the removal too: `uchg` refuses
+  // every change and `uappnd` refuses the unlink, and on a directory either
+  // one also refuses the `chmod` that makes a `000` directory readable again —
+  // while `chflags -R` cannot read a `000` directory it has just unflagged. So
+  // a directory's flag and mode have to come off together, before anything
+  // inside it is read, at every depth and on the run root itself. The `uappnd`
+  // file whose mode already grants owner rwx is the shape where `chmod -R`
+  // exits 0 over a flag.
+  for (const [shape, opts] of [
+    ["a uchg file inside a chmod 000 directory", { fileFlag: "uchg" }],
+    ["a uappnd file whose own mode leaves chmod nothing to change", { fileFlag: "uappnd", fileMode: 0o700 }],
+    ["a directory that is both uchg and chmod 000", { dirFlag: "uchg" }],
+    ["a directory that is both uappnd and chmod 000", { dirFlag: "uappnd" }],
+    ["a uchg file inside a directory that is both uchg and chmod 000", { fileFlag: "uchg", dirFlag: "uchg" }],
+    ["a uchg, chmod 000 directory inside another", { dirFlag: "uchg", nested: true }],
+    ["its own uchg and chmod 000 bits", { rootFlag: "uchg" }],
+    ["its own uappnd and chmod 000 bits", { rootFlag: "uappnd" }],
+    ["its own uchg and chmod 000 bits over a uchg, chmod 000 directory", { rootFlag: "uchg", dirFlag: "uchg" }],
+  ]) {
+    test(`${name}: on darwin the prune removes an aged run root with ${shape}`, { skip: process.platform !== "darwin" && "chflags/uchg/uappnd are darwin's" }, (t) => {
+      const parent = join(scratch(t, "snapshot-repo-prune-flags-"), "pr7");
+      const stale = lockedRunRoot(parent, "run-staleaa", 9, opts);
+      try {
+        const r = spawnSync("sh", ["-c", renderPrune(path, parent)], { env: ENV, encoding: "utf8" });
+
+        assert.doesNotMatch(r.stdout ?? "", /SNAPSHOT_PRUNE_FAILED/, `the prune named a failure on a root it can normalise: ${r.stderr}`);
+        assert.equal(r.status, 0, `the prune exited non-zero: ${r.stderr}`);
+        assert.ok(!existsSync(stale), `an aged run root with ${shape} survived the prune — it now stays forever and every later review prints SNAPSHOT_PRUNE_FAILED`);
+      } finally {
+        unlock(parent);
+      }
+    });
+  }
 
-  // The flag on the DIRECTORY rather than on a file in it: `chmod -R` cannot
-  // restore a `uchg` directory's mode, and `chflags -R` clears the flag but
-  // cannot read the `000` directory it just unflagged — so the mode is only
-  // restorable after the flag step, which is why the prune runs `chmod` again.
-  test(`${name}: on darwin the prune removes an aged run root holding a directory that is both uchg and chmod 000`, { skip: process.platform !== "darwin" && "chflags/uchg is darwin's" }, (t) => {
-    const parent = join(scratch(t, "snapshot-repo-prune-uchgdir-"), "pr7");
-    const stale = lockedRunRoot(parent, "run-staleaa", 9, { immutableDir: true });
+  // GNU find (Linux) refuses an `-exec ... {} +` whose argument list holds the
+  // `{}` placeholder more than once — and counts a `{}` written inside an
+  // `sh -c` script argument as one. BSD find (darwin) does not, so a dev
+  // machine passes a prune that fails every Linux CI run. This shim enforces
+  // GNU's rule over the real `find`, so the case holds on either host.
+  test(`${name}: the prune's find invocation keeps to GNU find's one-placeholder rule for -exec ... +`, (t) => {
+    const parent = join(scratch(t, "snapshot-repo-prune-gnu-"), "pr7");
+    const stale = lockedRunRoot(parent, "run-staleaa", 9);
+    const bin = scratch(t, "snapshot-repo-prune-gnu-bin-");
+    const realFind = execFileSync("sh", ["-c", "command -v find"], { env: ENV, encoding: "utf8" }).trim();
+    writeFileSync(
+      join(bin, "find"),
+      `#!/bin/sh
+# GNU find: with a terminating '+', only one argument may contain {}.
+n=0; plus=0; prev=
+for a in "$@"; do
+  case "$a" in *'{}'*) n=$((n + 1));; esac
+  [ "$a" = "+" ] && [ "$prev" = '{}' ] && plus=1
+  prev=$a
+done
+if [ "$plus" = 1 ] && [ "$n" -gt 1 ]; then
+  echo "find: Only one instance of {} is supported with -exec ... +" >&2
+  exit 1
+fi
+exec "${realFind}" "$@"
+`,
+      { mode: 0o755 },
+    );
+    const env = { ...ENV, PATH: `${bin}:${ENV.PATH}` };
     try {
-      const r = spawnSync("sh", ["-c", renderPrune(path, parent)], { env: ENV, encoding: "utf8" });
+      const r = spawnSync("sh", ["-c", renderPrune(path, parent)], { env, encoding: "utf8" });
 
-      assert.doesNotMatch(r.stdout ?? "", /SNAPSHOT_PRUNE_FAILED/, `the prune named a failure on a root it can normalise: ${r.stderr}`);
-      assert.ok(!existsSync(stale), "an aged run root holding a uchg, chmod 000 directory survived the prune — its mode was never restored after the flag was cleared, so `rm -rf` cannot read it");
+      assert.doesNotMatch(r.stdout ?? "", /SNAPSHOT_PRUNE_FAILED/, `a GNU-rule find rejected the prune: ${r.stderr}`);
+      assert.doesNotMatch(r.stderr ?? "", /Only one instance of \{\}/, "the prune's find tripped GNU's one-placeholder rule");
+      assert.ok(!existsSync(stale), "the aged run root survived a find that enforces GNU's placeholder rule");
     } finally {
       unlock(parent);
     }

package/skills/run-team/SKILL.md

@@ -1396,6 +1396,7 @@ for a token on a ticket's line — never a hand edit.
 | Fix-applier report | `ledger.mjs settle fix-pr-<M>=…`; for a fix-applier that answered a review — not one that cleared a conflict hold, which has no review file — `~/.fleet/bin/fleet-run dispositions-check.mjs --member <that member> --scratch <scratch>`, from the checkout root: it judges `<scratch>/dispositions-<M>.json` against `<scratch>/review-<M>.json`, writes `dispositions-ok=`, `dispositions-mismatch=` or `dispositions-escalate=<member>:<head>` onto the PR's row itself, and exits 1 on a mismatch or an escalation, naming each violating or escalated entry's bucket and index, a violation its rule too — **Then dispatch a fix-applier** says what a mismatch asks of you, the gate paragraph below what an escalation does; copy the refutations it reversed — the record's `refuted` entries — to `ruled`; `ledger.mjs filed <N> "<subject>"` for each `unrecorded:` line |
 | Finisher report | `ledger.mjs settle finisher-pr-<M>=labelled` as reported, even when its read-back lacks `ready-to-merge` — the tick's `DISPATCH finisher PR#<M>` catches that next. A **repair** finisher's (one sent on that line) read-back lacking it also gets `gh pr comment <M>` with both finishers' read-backs: the one-shot escalation, where halts comment. `ledger.mjs filed <N> "<subject>"` for each `unrecorded:` line |
 | Finisher report (halted) | `ledger.mjs settle finisher-pr-<M>=halted:<cause>`; `gh pr comment <M>` with the finisher's halt report, cause and evidence; `ledger.mjs filed <N> "<subject>"` for each `unrecorded:` line; then the per-cause rule (**Resolving a finisher halt**, below) |
+| Finisher report (failed / killed) | `ledger.mjs settle finisher-pr-<M>=failed` for a finisher that crashed or gave up, `=killed` for one that was killed; no label, and the tick prints no dispatch for it. The cockpit flags the PR `finisher:failed` / `finisher:killed` at severity 4 and you resolve it by hand (**Resolving a finisher that died**, below) |
 | Label seen (persistent Monitor) | nothing to record |
 | CI run terminal | `ci=<run-id>:<attempt>:<conclusion>` on the row; then the finisher gate (below) |
 | Merge-bot pass report | `held-behind:#<lower>` rows; `ledger.mjs settle merge-bot-<n>=done`; `reap.sh --apply`. (The bot may also have written `conflict-hold:#<pr>` onto a held PR's own row earlier in this same pass, before reporting — that token is the bot's, never `reap.sh`'s.) |
@@ -2801,6 +2802,19 @@ review that died before returning. A head that moved past `reviewed=` with no
 such halt — a fix-applier's push — stays not due: duty 2 verifies what it
 applied.
 
+**Resolving a finisher that died.** A finisher settled `failed` or `killed` left
+its PR without `ready-to-merge`, and nothing else notices: the PR is out of the
+merge queue, and the tick dispatches no finisher for it. The cockpit flags a
+REVIEW-column open PR without the label, whose latest finisher attempt settled
+so, `finisher:failed` or `finisher:killed` at severity 4 — one token per
+outcome, neither the implementer's bare `killed` flag nor `unlabelled`. You
+resolve it by hand: investigate why the finisher died, or dispatch the
+next-suffix finisher after the finisher gate, `ledger.mjs dispatch <M>
+finisher-pr-<M>-<x>`. The flag clears when a later attempt settles — `labelled`
+(then `unlabelled` takes over while the label is still off), `halted:<cause>`,
+or a live `-b` — or when the PR carries `ready-to-merge`, merges or closes.
+You never add the label yourself, and the tick does not retry.
+
 Gate on the `check` job, **not** on `ci-state --quiet` exit 0: a behind PR never
 reaches full green, so an exit-0 gate strands it unlabelled. The finisher reads
 per-job state (`ci-state.mjs` without `--quiet`, or its `jobs`), since `--quiet`