@feigi/fleet-ctl 3.91.3 → 3.92.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feigi/fleet-ctl",
-  "version": "3.91.3",
+  "version": "3.92.0",
   "description": "Agent fleet: run-team controller, merge bot, PR reviewer, and the ticket pipeline they share",
   "license": "Apache-2.0",
   "repository": {

package/scripts/fleet-event.mjs

@@ -237,7 +237,7 @@ export const BAILS = {
   fixed: { kind: "fixed", flag: "detail" },
 };
 // The commit a `fixed` bail's detail opens with.
-const FIXED_SHA = /^([0-9a-f]{7,40})(?![0-9A-Za-z])/;
+export const FIXED_SHA = /^([0-9a-f]{7,40})(?![0-9A-Za-z])/;
 // The causes a `hands` detail opens with — the Pull's answer grammar — and the
 // words each puts in the comment.
 const HANDS = { "external access": "external access", "manual testing": "manual testing", judgment: "judgment during the work" };

package/scripts/member-outcomes.mjs

@@ -15,6 +15,10 @@ import { join, basename } from "node:path";
 import { readOmpSession, normalizeModel, parseMemberName, isOmpSessionDirName } from "./member-record.mjs";
 import { reportFamily, reportStatus, defaultScratch } from "./report-files.mjs";
 import { FleetDirUnresolvable } from "./fleet-dir.mjs";
+import { spawnSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { FIXED_SHA } from "./fleet-event.mjs";
+import { parseReport, reportViolation } from "./report-schema.mjs";
 
 export { normalizeModel, parseMemberName };
 
@@ -61,7 +65,7 @@ export { normalizeModel, parseMemberName };
 // run_date comes from the newest transcript's mtime rather than a clock
 // read, so a backfill run in December still dates an August session in
 // August.
-export function rowsForSession(sessionDir, stats = {}) {
+export function rowsForSession(sessionDir, stats = {}, { verifyMain = verifyMainWithGit } = {}) {
   let names;
   try { names = readdirSync(sessionDir, { recursive: true }); }
   catch { names = []; }
@@ -76,7 +80,8 @@ export function rowsForSession(sessionDir, stats = {}) {
   stats.torn = 0;
   const session = basename(sessionDir);
   const run_date = newest ? new Date(newest).toISOString().slice(0, 10) : "";
-  return recs.map((r) => ({
+  const endings = endingsOf(recs, verifyMain);
+  return recs.map((r, i) => ({
     session, run_date,
     harness: r.harness, role: r.role, member: r.member, model: r.model,
     effort: r.thinking === "-" ? "" : r.thinking, ticket: r.ticket, pr: r.pr,
@@ -85,6 +90,7 @@ export function rowsForSession(sessionDir, stats = {}) {
     subagentType: r.subagent_type,
     tokensIn: r.tokens_in, tokensCacheRead: r.tokens_cache_read,
     tokensCacheWrite1h: r.tokens_cache_write_1h ?? "", cost: r.cost == null ? "" : Number(r.cost.toFixed(6)),
+    ending: endings[i],
   }));
 }
 
@@ -101,6 +107,7 @@ export const COLUMNS = [
   "session", "run_date", "role", "member", "model", "effort", "ticket", "pr",
   "tokens_cache_create", "tokens_out", "wall_s", "turns", "agent", "harness",
   "subagent_type", "report", "tokens_in", "tokens_cache_read", "tokens_cache_write_1h", "cost",
+  "ending",
 ];
 
 // `report`: what reportStatus() says of the row's member under `scratch` — a
@@ -116,6 +123,98 @@ export function withReports(rows, scratch) {
   }));
 }
 
+// The `ending` column: how an implementer's run ENDED, read off its own
+// transcript and the reports it wrote. Blank on every row that is not an
+// implementer (the member's leaf is `impl-<N>`, with an optional retry
+// suffix): blank there means "not an implementer row", not unknown. `unknown`
+// is where the transcript cannot say: a malformed line, no transcript, or a
+// fixed sha origin/main cannot answer. Precedence, first that applies wins:
+//   pr            the row's pr is named, the member's own `gh pr create`
+//                 opened one (more than one leaves the pr column blank), or
+//                 the last report claims a PR an earlier same-ticket
+//                 implementer opened in this session;
+//   bail:<cause>  the last report bails (`fixed` is checked against main);
+//   provider      a model fell back, or the last turn errored;
+//   budget        a budget notice reached the member;
+//   abort         the last turn was aborted;
+//   crash         the session never exited and wrote no yield sidecar;
+//   hollow        none of the above: the member stopped on its own.
+const IMPLEMENTER_LEAF = /^impl-\d+(?:-[a-z0-9]+)?$/;
+const isImplementer = (member) => IMPLEMENTER_LEAF.test(member.split("/").pop());
+
+export function endingOf({ member, pr, signals = null, earlierPrs = new Set(), verifyMain = verifyMainWithGit }) {
+  if (!isImplementer(member)) return "";
+  if (pr) return "pr";
+  if (!signals || signals.malformedNonLastLines > 0) return "unknown";
+  if (signals.openedPrs.length > 0) return "pr";
+  const last = lastReport(signals.reportWrites, member.split("/").pop());
+  if (last?.outcome === "pr" && earlierPrs.has(String(last.pr))) return "pr";
+  // lastReport passed the impl schema, whose bailCause is a BAILS key or null.
+  if (last?.bailCause) return bailEnding(last, verifyMain);
+  if (signals.providerFallback || signals.lastStopReason === "error") return "provider";
+  if (signals.budgetNotice) return "budget";
+  if (signals.lastStopReason === "aborted") return "abort";
+  if (!signals.sessionExit && !signals.hasYield) return "crash";
+  return "hollow";
+}
+
+// The member's last report, when the last one it wrote is a valid impl report
+// naming this member; null when it wrote none, or the last one is malformed.
+function lastReport(reportWrites, leaf) {
+  const last = reportWrites.filter((r) => r.file === `${leaf}.json`).at(-1);
+  if (!last) return null;
+  const parsed = parseReport(last.content);
+  if ("violation" in parsed) return null;
+  if (reportViolation("impl", parsed.value) !== null) return null;
+  return parsed.value.member === leaf ? parsed.value : null;
+}
+
+// A bail's ending: its cause, except `fixed`, which holds only while origin/main
+// reaches the sha its detail opens with.
+function bailEnding(report, verifyMain) {
+  if (report.bailCause !== "fixed") return `bail:${report.bailCause}`;
+  const m = FIXED_SHA.exec((report.bailDetail ?? "").trim());
+  if (!m) return "unknown";
+  const status = verifyMain(m[1]);
+  if (status === 0) return "bail:fixed";
+  if (status === 1) return "bail:fixed-refuted";
+  return "unknown";
+}
+
+// Each record's ending. `earlierPrs` is the set of PRs the same-ticket
+// implementers opened before this one started, so a continuation's report can
+// name its predecessor's PR and still read `pr`. A record never joins itself:
+// its own start is not before its own start. An `impl-<ticket>` record's `pr`
+// is blank or its one opened PR, so `openedPrs` alone holds every PR it opened.
+function endingsOf(recs, verifyMain) {
+  return recs.map((r) => {
+    const earlierPrs = new Set();
+    if (r.ticket && isImplementer(r.member)) {
+      for (const o of recs) {
+        if (o.ticket !== r.ticket || !isImplementer(o.member)) continue;
+        if (!(o.transcript.startedAt < r.transcript.startedAt)) continue;
+        for (const n of o.transcript.openedPrs) earlierPrs.add(String(n));
+      }
+    }
+    return endingOf({ member: r.member, pr: r.pr, signals: r.transcript, earlierPrs, verifyMain });
+  });
+}
+
+// The verifier the `fixed` bail calls: `verify-sha.sh main <sha>` answers
+// 0 when origin/main reaches the sha, 1 when it does not, and 2 when it could
+// not say (its exit status, or none when the script did not run at all). A
+// verifier that could not say leaves the bail `unknown`, so its own diagnosis
+// goes to stderr: without it an offline scrape and a broken script both read
+// as a bare `unknown`.
+const VERIFY_SHA = fileURLToPath(new URL("./verify-sha.sh", import.meta.url));
+function verifyMainWithGit(sha) {
+  const r = spawnSync("sh", [VERIFY_SHA, "main", sha], { encoding: "utf8" });
+  if (r.status === 0 || r.status === 1) return r.status;
+  const why = r.error ? r.error.message : (r.stderr ?? "").trim() || `exit ${r.status ?? r.signal}`;
+  process.stderr.write(`${NAME}: verify-sha.sh main ${sha} could not answer, so its fixed bail reads unknown: ${why}\n`);
+  return 2;
+}
+
 // Row objects use camelCase; the file uses snake_case. One map, one direction
 // each, so a rename cannot silently drop a column.
 const FIELD = {
@@ -134,9 +233,22 @@ const key = (r) => `${r.session}\0${r.agent}`;
 // Replace-by-key, not append. This is what makes a close-out re-run and a full
 // regeneration both safe, and it is the reason no backfill-only code path is
 // needed: the idempotency backfill wants is the idempotency a re-run wants.
-export function mergeRows(existing, incoming) {
+//
+// One cell is the exception: an incoming `unknown` ending keeps the ending the
+// replaced row recorded. `unknown` is this scrape failing to say (a verifier
+// that is offline, a transcript gone unreadable), not a reading, so it never
+// overwrites one: a recorded `bail:fixed` would otherwise flip to `unknown`
+// on any re-scrape run without network. `stats.endingsKept` counts them.
+export function mergeRows(existing, incoming, stats = {}) {
   const by = new Map((existing ?? []).map((r) => [key(r), r]));
-  for (const r of incoming ?? []) by.set(key(r), r);
+  stats.endingsKept = 0;
+  for (const r of incoming ?? []) {
+    const old = by.get(key(r));
+    if (r.ending === "unknown" && old?.ending && old.ending !== "unknown") {
+      by.set(key(r), { ...r, ending: old.ending });
+      stats.endingsKept++;
+    } else by.set(key(r), r);
+  }
   return [...by.values()].sort((a, b) => key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0);
 }
 
@@ -268,7 +380,7 @@ if (isCLI(import.meta.url)) {
   }
 
   const stats = {};
-  const merged = mergeRows(existing, withReports(rowsForSession(sessionDir, stats), scratch));
+  const merged = mergeRows(existing, withReports(rowsForSession(sessionDir, stats), scratch), stats);
   // Write-then-rename: writeFileSync truncates before it writes, so an interrupt
   // over the committed corpus left a torn file that the NEXT scrape read, merged
   // and rewrote — losing every other session's rows for good.
@@ -278,8 +390,9 @@ if (isCLI(import.meta.url)) {
   // so a session where every member dropped printed the same reassuring line as
   // a healthy one — 0-of-2 was indistinguishable from full success.
   const torn = stats.torn ? `, ${stats.torn} torn mid-write` : "";
+  const kept = stats.endingsKept ? `, ${stats.endingsKept} read unknown and kept the ending already recorded` : "";
   process.stderr.write(
     `${NAME}: scraped ${stats.seen - stats.dropped} of ${stats.seen} members ` +
-    `(${stats.dropped} dropped${torn}); ${file} now holds ${merged.length} rows\n`,
+    `(${stats.dropped} dropped${torn}${kept}); ${file} now holds ${merged.length} rows\n`,
   );
 }

package/scripts/member-record.mjs

@@ -22,7 +22,7 @@
 // the transcript carried no `session_init` agent to read (no such line, or
 // one without a string `agent`). An untyped `task` dispatch is not that
 // shape: the omp reader records the generic `task` for it.
-import { readdirSync, readFileSync, realpathSync } from "node:fs";
+import { existsSync, readdirSync, readFileSync, realpathSync } from "node:fs";
 import { join, basename, relative, isAbsolute, sep } from "node:path";
 
 import { classifyRole, canonicalMemberName, CANONICAL_MEMBER_NAME_PREFIXES } from "./compute-spend.mjs";
@@ -361,6 +361,12 @@ export function foldOmpTranscript(jsonlText, filePath) {
   let input = 0, cacheWrite = 0, cacheWrite1h = 0, cacheRead = 0, output = 0, cost = 0, turns = 0;
   let sawCost = false, unsplit = false;
   let malformedNonLastLines = 0;
+  // What the transcript's END says, read for the `ending` column: the last
+  // assistant turn's stopReason, whether a model fell back, whether the
+  // session exited cleanly, whether a budget notice reached the member, and
+  // the report files the member wrote (last write wins, read by the caller).
+  let lastStopReason = null, providerFallback = false, sessionExit = false, budgetNotice = false;
+  const reportWrites = [];
   const entries = [];
   const createCallIds = new Set();
   const openedPrs = new Set();
@@ -379,6 +385,8 @@ export function foldOmpTranscript(jsonlText, filePath) {
     assertOmpShaped(d, filePath);
     if (typeof d.timestamp === "string") { firstTs ??= d.timestamp; lastTs = d.timestamp; }
     if (d.type === "thinking_level_change" && typeof d.thinkingLevel === "string") thinking = d.thinkingLevel;
+    if (d.type === "model_change" && d.resolvedModelIsFallback === true) providerFallback = true;
+    if (d.type === "custom" && d.customType === "session_exit") sessionExit = true;
     if (d.type === "session_init") {
       if (typeof d.task === "string") task = d.task;
       if (typeof d.resolvedModelIdentity === "string") resolvedModelIdentity = d.resolvedModelIdentity;
@@ -388,9 +396,11 @@ export function foldOmpTranscript(jsonlText, filePath) {
     const m = d.message;
     if (d.type === "message" && m?.role === "assistant") {
       const blocks = Array.isArray(m.content) ? m.content : [];
+      lastStopReason = typeof m.stopReason === "string" ? m.stopReason : null;
       for (const c of blocks) {
         if (c?.type === "toolCall" && c.name === "bash" && typeof c.arguments?.command === "string"
           && GH_PR_CREATE_RE.test(c.arguments.command)) createCallIds.add(c.id);
+        reportWrites.push(...reportWritesOf(c));
       }
       if (m.usage) {
         const u = m.usage;
@@ -419,6 +429,7 @@ export function foldOmpTranscript(jsonlText, filePath) {
         for (const n of createdPrNumbers(text)) openedPrs.add(n);
       }
     }
+    if (d.type === "message" && m?.role === "user" && messageText(m).trimStart().startsWith("[budget notice]")) budgetNotice = true;
   }
   const span = firstTs && lastTs ? (Date.parse(lastTs) - Date.parse(firstTs)) / 1000 : 0;
   return {
@@ -436,9 +447,42 @@ export function foldOmpTranscript(jsonlText, filePath) {
     entries,
     malformedNonLastLines,
     openedPrs: [...openedPrs],
+    startedAt: firstTs,
+    reportWrites,
+    lastStopReason, providerFallback, sessionExit, budgetNotice,
   };
 }
 
+// The report files one transcript tool call writes, as { file, content } with
+// `file` the written path's basename: the `write` tool's arguments, or each
+// `cat > <path> <<'EOF'` heredoc a `bash` call carries. Only a path under a
+// `reports/` directory counts.
+const HEREDOC_RE = /cat\s*>\s*(\S+)\s*<<\s*['"]?([A-Za-z_][A-Za-z0-9_]*)['"]?[^\n]*\n([\s\S]*?)\n\2[ \t]*(?=\n|$)/g;
+const REPORT_PATH_RE = /\/reports\/[^/\s]+\.json$/;
+function reportWritesOf(c) {
+  if (c?.type !== "toolCall") return [];
+  const a = c.arguments ?? {};
+  if (c.name === "write" && typeof a.path === "string" && typeof a.content === "string") {
+    return REPORT_PATH_RE.test(a.path) ? [{ file: basename(a.path), content: a.content }] : [];
+  }
+  if (c.name === "bash" && typeof a.command === "string") {
+    const out = [];
+    for (const m of a.command.matchAll(HEREDOC_RE)) {
+      const path = m[1].replace(/^["']|["']$/g, "");
+      if (REPORT_PATH_RE.test(path)) out.push({ file: basename(path), content: m[3] });
+    }
+    return out;
+  }
+  return [];
+}
+
+// The text a transcript message carries: a bare string, or its text blocks.
+function messageText(m) {
+  if (typeof m?.content === "string") return m.content;
+  if (!Array.isArray(m?.content)) return "";
+  return m.content.map((b) => (b?.type === "text" && typeof b.text === "string" ? b.text : "")).join("\n");
+}
+
 // One member record from one omp transcript's fold. `member` is the AgentId
 // (the filename stem, e.g. `InstallVerifySearch`) — there is no separate
 // display name, so `ticket`/`pr` extraction runs against it directly — except
@@ -577,7 +621,7 @@ export function ompSessionTranscripts(sessionDir) {
 
 // Walks one omp session directory into records, via the walk above.
 //
-// Deliberately NOT wrapped in a blanket try/catch around readOmpMember: the
+// Deliberately NOT wrapped in a blanket try/catch around foldOmpTranscript: the
 // wrong-shape refusal (assertOmpShaped, inside foldOmpTranscript) must
 // propagate all the way out of readMembers, uncaught. Only the directory
 // listing and the raw file read are given the ordinary per-member tolerance.
@@ -591,9 +635,19 @@ export function readOmpSession(sessionDir) {
     let text;
     try { text = readFileSync(file, "utf8"); }
     catch { continue; }
-    const rec = readOmpMember(text, file, agent, spawnDepth); // may throw — see comment above
+    const folded = foldOmpTranscript(text, file); // may throw — see comment above
+    const rec = ompMemberRecord(folded, agent, spawnDepth);
     if (!rec) continue;
-    rows.push({ ...rec, session, agent });
+    // How the transcript ENDED, for the `ending` column (member-outcomes.mjs):
+    // the fold's end-of-transcript signals, and whether the member wrote its
+    // yield sidecar beside the transcript.
+    rows.push({ ...rec, session, agent, transcript: {
+      startedAt: folded.startedAt, reportWrites: folded.reportWrites, openedPrs: folded.openedPrs,
+      lastStopReason: folded.lastStopReason, providerFallback: folded.providerFallback,
+      sessionExit: folded.sessionExit, budgetNotice: folded.budgetNotice,
+      malformedNonLastLines: folded.malformedNonLastLines,
+      hasYield: existsSync(file.replace(/\.jsonl$/, ".md")),
+    } });
   }
   return rows;
 }