@feigi/fleet-ctl 3.76.9 → 3.77.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feigi/fleet-ctl",
-  "version": "3.76.9",
+  "version": "3.77.1",
   "description": "Agent fleet: run-team controller, merge bot, PR reviewer, and the ticket pipeline they share",
   "license": "Apache-2.0",
   "repository": {

package/scripts/fleet-event.mjs

@@ -134,6 +134,27 @@
 //       `closeout.mjs reopen --reason "<reason>"`, its output printed as it
 //       stands, then the tick. Refused once any `finish` step is ok, and when
 //       no drain marker is set; its exit 2 or 3 is passed through.
+//   member-died <member>
+//       Settles a member the controller has confirmed dead, then ticks. The
+//       outcome comes from the session directory's listing, never from an
+//       argument or a file's contents. The first marker that matches decides:
+//         <m>.jsonl.tombstone                              killed
+//         <m>.md, no tombstone, fix-pr or finisher-pr      failed
+//         <m>.md, no tombstone, merge-bot                  killed
+//         <m>.md, no tombstone, impl                       refused: settle it by hand
+//         <m>.jsonl, no marker                             refused: running or parked
+//         no transcript, no marker                         killed once the tick's
+//                                                          first-seen stamp in
+//                                                          heartbeat.json is over
+//                                                          five minutes old; refused
+//                                                          when younger or unstamped
+//       Ahead of the markers: a review runner refuses, its death being
+//       `review-failed`; a member the ledger already holds settled is noted,
+//       and the tick runs; a report on disk refuses, naming the kind that
+//       settles it, or its violation when that kind could not settle from
+//       it. The session directory is this run's phase-0 receipt's
+//       `session`, else `--session <dir>`: the tick's lookup, so another
+//       run's receipt names none.
 //
 // A write this script cannot make, and does not refuse, is printed as a
 // `NEED <what> …` line, at the left margin, for the controller to make by
@@ -141,7 +162,10 @@
 //
 // Exit: 0 every write made and the tick exited 0. 2 a usage error, refused
 // before anything is written or ticked: an unknown kind or flag, a `--reason`
-// that is empty or beside any kind but `drain` and `undrain`, a member of
+// that is empty or beside any kind but `drain` and `undrain`, a `--session`
+// beside any kind but `member-died`, any `member-died` refusal (the table's,
+// a name that is no ledger member, no session directory, a session directory
+// that cannot be listed, a heartbeat file that cannot be used), a member of
 // the wrong family, a review path outside the PR's partition or a file that
 // is no review result, a ledger or phase-0 receipt that cannot be read. A
 // ledger write's own refusal — `settle`'s 3 (the report's claim and the world
@@ -155,17 +179,18 @@
 // Zero deps: node builtins and sibling scripts only.
 
 import { spawnSync } from "node:child_process";
-import { readFileSync, realpathSync } from "node:fs";
+import { readdirSync, readFileSync, realpathSync } from "node:fs";
 import { basename, dirname, join, normalize, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { makeDie, defineFlags, isDigits, writeAll } from "./arg.mjs";
 import { FleetDirUnresolvable, fleetFile } from "./fleet-dir.mjs";
 import { gitEnv } from "./git-env.mjs";
 import { isCLI } from "./is-cli.mjs";
-import { BRIEFS, REVIEW, REVIEWED, memberTokens, nextReviewerName, parseDispatchedEntry, parseMember, parseToken, readRetired, rowNums } from "./ledger-grammar.mjs";
-import { dispositionsToken, foldLedger } from "./fleet-tick.mjs";
-import { readRunReceipt, tickCaps } from "./phase0-receipt.mjs";
+import { BRIEFS, REVIEW, REVIEWED, REVIEWER, memberTokens, nextReviewerName, parseDispatchedEntry, parseMember, parseToken, readRetired, rowNums } from "./ledger-grammar.mjs";
+import { NO_TRANSCRIPT_AFTER_DEFAULT, controllerState, dispositionsToken, foldLedger } from "./fleet-tick.mjs";
+import { phase0Gate, readRunReceipt, tickCaps } from "./phase0-receipt.mjs";
 import { defaultScratch, readReportIn, reportsDir } from "./report-files.mjs";
+import { readState, statePath } from "./fleet-state.mjs";
 
 const NAME = "fleet-event";
 const die = makeDie(NAME);
@@ -188,8 +213,9 @@ const KINDS = {
   heartbeat: { args: [] },
   drain: { args: [], flag: "--reason \"<reason>\"" },
   undrain: { args: [], flag: "--reason \"<reason>\"" },
+  "member-died": { args: ["<member>"] },
 };
-const FLAGS = { "in-flight": "value", reason: "value" };
+const FLAGS = { "in-flight": "value", reason: "value", session: "value" };
 const USAGE = `usage: ${NAME}.mjs <kind> <args> [--in-flight <n>] — kinds: `
   + Object.entries(KINDS).map(([k, { args, flag }]) => [k, ...args, ...(flag ? [flag] : [])].join(" ")).join(", ");
 
@@ -299,15 +325,18 @@ class Event {
     return readReportIn(reportsDir(this.scratch), member);
   }
 
-  // `settle <member> --from-report` unless the ledger holds it settled.
-  // Returns the outcome, or null when the settle refused.
-  settle(member) {
+  // `settle <member> --from-report` unless the ledger holds it settled, or
+  // `settle <member> <decided>` when a marker decided the outcome — never one
+  // a file's contents named. Returns the outcome, or null when the settle
+  // refused.
+  settle(member, decided = null) {
     const before = settledOutcome(this.ledger, member);
     if (before !== null) {
       this.note(`${member}=${before} already settled`);
       return before;
     }
-    const r = node("ledger.mjs", ["settle", member, "--from-report", "--reports", reportsDir(this.scratch)]);
+    const tail = decided === null ? ["--from-report", "--reports", reportsDir(this.scratch)] : [decided];
+    const r = node("ledger.mjs", ["settle", member, ...tail]);
     if (r.status !== 0) {
       this.refuse(`settle ${member}`, r);
       return null;
@@ -727,6 +756,81 @@ function undrain(ev, reason) {
   }
 }
 
+// ── member-died ─────────────────────────────────────────────────────────────
+
+// fleet-tick's default `--no-transcript-after`: a member with no transcript
+// whose first-seen stamp is older than this never started.
+const NO_TRANSCRIPT_MS = NO_TRANSCRIPT_AFTER_DEFAULT * 60_000;
+
+// The kind whose report settles a family — the KINDS entry naming that family
+// — named in the refusal that finds one on disk.
+export const reportKind = (family) => Object.keys(KINDS).find((k) => KINDS[k].family === family);
+
+// This run's phase-0 receipt `session`, or null: the receipt phase0Gate()
+// judges this run's against the controller record in heartbeat.json and that
+// record's process, as fleet-tick judges it.
+function runSession({ read, path }) {
+  const { controller } = readState(statePath(NAME), NAME);
+  return phase0Gate(read, { path, controllerAt: controller?.at, controller: controllerState(controller, NAME) }).session;
+}
+
+// The names in the session directory. A missing one reads as empty when the
+// receipt named it, as the tick reads it, and refuses otherwise.
+function sessionListing(session, receiptNamed) {
+  try {
+    return new Set(readdirSync(session));
+  } catch (e) {
+    if (e.code === "ENOENT" && receiptNamed) return new Set();
+    return die(`member-died: cannot list the session directory ${session} (${e.code ?? e.message}) — nothing written`);
+  }
+}
+
+// Settles a member the controller has confirmed dead, by the first marker
+// that matches; see the header. The settle is a ledger write, so its
+// refusal passes through to the exit as every other kind's does. Every other
+// refusal is a die() before anything is written or ticked.
+function memberDied(ev, member, receipt, flagSession) {
+  const review = REVIEWER.exec(member);
+  if (review !== null) die(`member-died ${member}: a review runner settles nothing; its death is review-failed ${review[1]} member:${member} — nothing written`);
+  const m = parseMember(member);
+  if (m === null) die(`member-died: '${member}' is not a ledger member — nothing written`);
+  const before = settledOutcome(ev.ledger, member);
+  if (before !== null) {
+    ev.note(`${member}=${before} already settled`);
+    return;
+  }
+  const report = ev.report(member);
+  const kind = reportKind(m.family);
+  if (report.state === "invalid") die(`member-died ${member}: its report ${report.path} is on disk and ${kind} cannot settle from it (${report.violation}): fix the report and run ${kind} ${member}, or settle ${member} by hand — nothing written`);
+  if (report.state !== "absent") die(`member-died ${member}: its report ${report.path} is on disk; ${kind} ${member} settles it — nothing written`);
+  // The session directory the tick reads: this run's receipt's `session`,
+  // else `--session`. A `--session` the receipt's overrides is said, not
+  // dropped in silence.
+  const receiptSession = runSession(receipt);
+  if (receiptSession !== null && flagSession !== null && resolve(flagSession) !== resolve(receiptSession)) {
+    writeAll(2, `${NAME}: member-died ${member} reads ${receiptSession}, this run's phase-0 receipt's session, not --session ${flagSession}\n`);
+  }
+  const session = receiptSession ?? flagSession;
+  if (session === null) die(`member-died ${member} needs the session directory: this run's phase-0 receipt names none, so pass --session <dir> — nothing written`);
+  const listed = sessionListing(session, receiptSession !== null);
+  const md = `${member}.md`;
+  const jsonl = `${member}.jsonl`;
+  if (listed.has(`${jsonl}.tombstone`)) return ev.settle(member, "killed");
+  if (listed.has(md)) {
+    if (m.family === "impl") die(`member-died ${member}: ${md} is in ${session} with no tombstone, and an implementer's yield is its own outcome: read agent://${member} and settle it by hand — nothing written`);
+    return ev.settle(member, m.family === "merge-bot" ? "killed" : "failed");
+  }
+  if (listed.has(jsonl)) die(`member-died ${member}: ${member} has a transcript and no marker in ${session}, so it is running or parked; census it, kill it, then rerun — nothing written`);
+  const heartbeat = statePath("fleet-event");
+  const state = readState(heartbeat, "fleet-event");
+  if (state.degraded) die(`member-died ${member}: ${heartbeat} ${state.degraded} — nothing written`);
+  const stamp = state.seen[member];
+  if (typeof stamp !== "number") die(`member-died ${member}: no transcript and no first-seen stamp in ${heartbeat}, so the tick never saw it live — nothing written`);
+  const age = Date.now() - stamp;
+  if (age <= NO_TRANSCRIPT_MS) die(`member-died ${member}: first seen ${Math.floor(age / 60_000)} min ago, not yet past the ${NO_TRANSCRIPT_MS / 60_000}-minute no-transcript bound — nothing written`);
+  return ev.settle(member, "killed");
+}
+
 // ── main ────────────────────────────────────────────────────────────────────
 
 function boundArg(F, name) {
@@ -764,6 +868,7 @@ function main() {
   } else if (reason !== null) {
     die(`--reason belongs to drain and undrain, not ${kind}`);
   }
+  if (kind !== "member-died" && F.arg("session") !== null) die(`--session belongs to member-died, not ${kind}`);
   let pr = null;
   if (spec.family !== undefined) {
     const m = parseMember(args[0]);
@@ -776,9 +881,11 @@ function main() {
 
   let fleetDir;
   let scratch;
+  let receipt;
   try {
     fleetDir = fleetFile(null);
-    const { read } = readRunReceipt();
+    receipt = readRunReceipt();
+    const { read } = receipt;
     const named = read.state === "ok" ? read.receipt.scratch : undefined;
     scratch = typeof named === "string" && named !== "" ? named : defaultScratch();
   } catch (e) {
@@ -788,7 +895,6 @@ function main() {
   const caps = tickCaps(fleetDir);
   if (caps.error) die(`${caps.error} — nothing written`);
   const tickArgs = ["--apply", ...(inFlight === null ? caps.args : ["--in-flight", inFlight]), ...(kind === "heartbeat" ? ["--fold-unchanged"] : [])];
-
   const ev = new Event({ fleetDir, scratch, ledger: readLedger() });
   if (kind === "impl-report") implReport(ev, args[0]);
   else if (kind === "fix-report") fixReport(ev, args[0]);
@@ -800,6 +906,7 @@ function main() {
   else if (kind === "conflict") conflict(ev, pr);
   else if (kind === "drain") drain(ev, reason);
   else if (kind === "undrain") undrain(ev, reason);
+  else if (kind === "member-died") memberDied(ev, args[0], receipt, F.arg("session"));
 
   writeAll(1, `${[...ev.lines, `event: ${kind}: ${ev.summary.join("; ") || "nothing to record"}`].join("\n")}\n`);
   const tick = spawnSync(process.execPath, [join(SCRIPTS, "fleet-tick.mjs"), ...tickArgs], { stdio: ["ignore", "inherit", "inherit"], env: gitEnv() });

package/scripts/fleet-tick.mjs

@@ -1295,6 +1295,10 @@ export function unrecordedLines({ stems, recorded }) {
   });
 }
 
+// The `--no-transcript-after` default, in minutes: its one definition, which
+// fleet-event.mjs's member-died reads as its own no-transcript bound.
+export const NO_TRANSCRIPT_AFTER_DEFAULT = 5;
+
 // The NO-TRANSCRIPT lines: one per live member token in `live` whose `seen`
 // stamp is more than `after` minutes old and whose `<member>.jsonl` is not
 // among `stems`, as `{kind, name, line, key}` — a ledger token whose `task` call
@@ -1481,7 +1485,7 @@ const OPTIONS = {
   session: { type: "string", default: "" },
   // Minutes a live member token may go with no transcript in the session
   // directory before NO-TRANSCRIPT names it.
-  "no-transcript-after": { type: "string", default: "5" },
+  "no-transcript-after": { type: "string", default: String(NO_TRANSCRIPT_AFTER_DEFAULT) },
   // Minutes a Pull's question may stand, its `pull.json` unwritten, before the
   // PULL row naming it says `(stale)`.
   "pull-stale-minutes": { type: "string", default: "120" },
@@ -1867,15 +1871,16 @@ function claimed() {
 
 // What the controller record's process is now, for the phase-0 gate: `none`
 // with no record, else judgeRecord()'s answer. A process table that cannot
-// be read is `unreadable`, said on stderr: a controller nobody could judge
-// is not one whose receipt this tick may take as this run's.
-function controllerState(controller) {
+// be read is `unreadable`, said on stderr under the caller's `name`: a
+// controller nobody could judge is not one whose receipt may be taken as
+// this run's.
+export function controllerState(controller, name = NAME) {
   if (controller === null) return "none";
   try {
     return judgeRecord(controller);
   } catch (e) {
     if (!(e instanceof ProcUnreadable)) throw e;
-    console.error(`${NAME}: WARNING ${e.message} — the recorded controller pid ${controller.pid} cannot be judged, so the phase-0 receipt is held`);
+    console.error(`${name}: WARNING ${e.message} — the recorded controller pid ${controller.pid} cannot be judged, so the phase-0 receipt is held`);
     return "unreadable";
   }
 }

package/scripts/metrics-refresh.mjs

@@ -16,6 +16,11 @@
 // rows the next drain finds already there, so a second run changes nothing.
 // A pending file that is empty, or blank lines only, is removed with its own
 // line: an append created it and never wrote.
+// The drain takes no lock by design: it assumes only the run's controller
+// appends to the pending file and drains it, serially, so nothing appends
+// between the read and either unlink. One controller per checkout is the
+// rule, but phase 0 refuses a second one at its ledger rotation, which runs
+// after its drain step, so a second controller's drain is not guarded.
 //
 // COST GUARD. `pr-cost.mjs --guard` rewrites `.fleet/cost-guard.json`, which
 // the router reads and fails closed without. It needs all three metrics TSVs;

package/scripts/ticket-router.mjs

@@ -601,6 +601,10 @@ function main() {
       issue: readJson(issuePath), sizing: readJson(F.arg("sizing")), guard: readGuard(readJson(guardPath)), table,
     });
     const pending = F.arg("pending") ?? join(dirname(guardPath), "ticket-features.pending.tsv");
+    // The header decision and the append are not atomic and take no lock. That is
+    // safe only while the run's controller is the only process that routes or
+    // drains on a checkout, doing both serially: one controller per checkout is
+    // the rule, and neither this append nor the drain checks it.
     try {
       mkdirSync(dirname(pending), { recursive: true });
       const header = statSync(pending, { throwIfNoEntry: false })?.size > 0 ? "" : `${COLUMNS.join("\t")}\n`;

package/skills/run-team/SKILL.md

@@ -203,6 +203,7 @@ Run each as `~/.fleet/bin/fleet-run fleet-event.mjs <kind> <args>`, from the mai
 | Heartbeat | `fleet-event.mjs heartbeat` |
 | Drain | `fleet-event.mjs drain --reason "<reason>"` |
 | Undrain | `fleet-event.mjs undrain --reason "<reason>"` |
+| Member confirmed dead or never dispatched | `fleet-event.mjs member-died <member>` |
 
 ### Nothing to do: arm the beat
 
@@ -280,7 +281,7 @@ depth** guard table applied in code. Act on each line as it reads:
 - `PHASE0-INCOMPLETE <step>…` — every dispatching row is held: run the `phase0.mjs` command the line names; advice lines hold nothing.
 - `CHECK-LIVENESS <member> (…)` — advice, never an action: take the census the **Member unaccounted** row of **Failure handling** gives before you touch the token.
 - `UNRECORDED <stem> (…)` — record it with `ledger.mjs dispatch`, or confirm it is a stranger's member.
-- `NO-TRANSCRIPT <member> (…)` — make the `task` call, or settle the token `killed`.
+- `NO-TRANSCRIPT <member> (…)` — make the `task` call, or run the **Member confirmed dead or never dispatched** wake.
 - `SETTLE <member> (report on disk …)` — a live token whose report is on disk: run that member's report wake from the wake table, which runs the `ledger.mjs settle <member> --from-report` the line prints plus the follow-on work its report calls for; `REPORT-MISMATCH`, `REPORT-UNREADABLE` and `REPORT-INVALID` name a report file that cannot be settled from, and the tick settles nothing itself.
 
 ## Phase 1 — Pull: judge, then claim or relabel
@@ -1296,11 +1297,7 @@ owes beyond its call:
   nothing the tick did not name.
 - **Finisher reports halted** → the per-cause rule (**Resolving a finisher
   halt**, above).
-- **A finisher failed or was killed** → `ledger.mjs settle finisher-pr-<M>=failed` for a
-  finisher that crashed or gave up, `=killed` for one that was killed; no label,
-  and the tick prints no dispatch for it. The cockpit flags the PR
-  `finisher:failed` / `finisher:killed` at severity 4 and you resolve it by hand
-  (**Resolving a finisher that died**, above), then run the tick.
+- **A finisher failed or was killed** → the **Member confirmed dead or never dispatched** wake (`member-died finisher-pr-<M>`), which settles it `failed` or `killed`; no label, and the tick prints no dispatch for it. The cockpit flags the PR `finisher:failed` / `finisher:killed` at severity 4 and you resolve it by hand (**Resolving a finisher that died**, above), then run the tick.
 - **Implementer bails before implementing** → `fleet-event.mjs impl-report impl-<N>`; the causes and the issue action each takes are under **Relabel causes**.
 - **A review result lands** → see the result file written (by you or by the
   runner — **Reviewers** says which, per harness), then
@@ -2609,8 +2606,8 @@ Match the failure to its row in the table below and run that response; where the
 | Merge bot's CI wait reaches its per-PR time bound with the run's `status` still not `completed` | The bot skips that PR and reports `ci-wait-timeout-#<pr>` with the run id and its status. PR stays queued, label untouched |
 | Member silent or truncated | Send to ping or resume — same unit of work; see the state machine below |
 | Member idle with work outstanding | Read the PR first, *then* ping. Idle ≠ done |
-| Member **killed** (spend limit, API error, crash) | Confirm it is dead first — a frozen transcript does not establish that; see below — then new member, new name, prompt carries inherited state |
-| Member **unaccounted** (a live token the tick flags `CHECK-LIVENESS`, or any live token nothing has reported on) | Registry census first: `read history://` once. A row other than `running` → `read agent://<member>` and the artifacts (PR head, worktree, labels), settle the real outcome with the existing vocabulary, and resume a `parked` member with `write agent://<member>` when its work is worth continuing. A `running` row → the mtime discriminator below, two polls: moved means BLOCKED and alive; frozen at both polls **and** no artifact movement → `write proc://<member>/kill`, `ledger.mjs settle <member> killed`, fresh name. Three independent alive-signals — the first poll, the second poll, the artifacts — must all be silent before a kill; the tick settles and kills nothing |
+| Member **killed** (spend limit, API error, crash) | Confirm it is dead first — a frozen transcript does not establish that; see below — then run the **Member confirmed dead or never dispatched** wake, then new member, new name, prompt carries inherited state |
+| Member **unaccounted** (a live token the tick flags `CHECK-LIVENESS`, or any live token nothing has reported on) | Registry census first: `read history://` once. A row other than `running` → `read agent://<member>` and the artifacts (PR head, worktree, labels), settle the real outcome with the existing vocabulary, and resume a `parked` member with `write agent://<member>` when its work is worth continuing. A `running` row → the mtime discriminator below, two polls: moved means BLOCKED and alive; frozen at both polls **and** no artifact movement → `write proc://<member>/kill`, the **Member confirmed dead or never dispatched** wake, fresh name. Three independent alive-signals — the first poll, the second poll, the artifacts — must all be silent before a kill; the tick settles and kills nothing |
 
 ### Recovery
 
@@ -2623,8 +2620,6 @@ spellings. Recovery is a fresh member, fresh name (`impl-<N>-b`,
 `fix-pr-<M>-b`, and for a reviewer the next letter PR M's rows do not carry
 yet) whose prompt states what it inherits.
 
-`hub cancel` leaves a peer hard-aborted and unmessageable, but a `failed` job's peer can stay `idle` and answer normally. No distinct truncated state exists.
-
 Reviewers that went idle on CI recover as a **finisher, not a re-review** once
 commits are pushed. See references/member-lifecycle.md.
 

package/skills/run-team/references/tick-rows.md

@@ -151,7 +151,9 @@ The full text of each row `fleet-tick.mjs` prints and what it asks of the contro
   above the rows — a live ledger token first seen more than
   `--no-transcript-after` minutes ago (default 5) has no transcript in the
   session directory: a dispatch whose `task` call was never made. Make the
-  `task` call, or settle the token `killed`. Advice the same way, folded as
+  `task` call, or run the **Member confirmed dead or never dispatched** wake
+  (`fleet-event.mjs member-died <member>`), which settles the token `killed`
+  once the first-seen stamp is past five minutes. Advice the same way, folded as
   `…; NO-TRANSCRIPT <member> persists — …`. Both lines read the session
   directory this run's phase-0 receipt names, else `--session <dir>`; with
   neither the tick skips them with one stderr line. A heartbeat state file