@cohortapp/agent-sdk 2.18.13 → 2.18.14

package/scripts/session/supervisor.mjs

@@ -42,7 +42,7 @@
  * @module scripts/session/supervisor
  */
 
-import { existsSync as fsExistsSync, readFileSync as fsReadFileSync, mkdirSync as fsMkdirSync, unlinkSync as fsUnlinkSync, writeFileSync as fsWriteFileSync, chmodSync as fsChmodSync } from "node:fs";
+import { existsSync as fsExistsSync, readFileSync as fsReadFileSync, mkdirSync as fsMkdirSync, unlinkSync as fsUnlinkSync, writeFileSync as fsWriteFileSync, chmodSync as fsChmodSync, readdirSync as fsReaddirSync } from "node:fs";
 import { randomUUID } from "node:crypto";
 import { join } from "node:path";
 import { spawn } from "node:child_process";
@@ -66,6 +66,11 @@ import {
   launchMode, recordLaunch, rotationDecision, rotateMainSession, BACKOFF_MS,
 } from "../../lib/session/identity.mjs";
 import { detectMux, chooseMux, buildMuxCommands, parseScreenList, parseExitFile, renderEnvFile } from "../../lib/session/launch-args.mjs";
+import { resumeTargetState } from "../../lib/session/resume-target.mjs";
+import {
+  classifyLaunchFailure, launchFailureSignature, parseLaunchFailures,
+  launchFailureStreak, escalationDecision, escalationHint, IDENTICAL_FAILURE_LIMIT,
+} from "../../lib/session/launch-failure.mjs";
 
 /** Write a private (0600) file that must not already exist. The engine env file carries the seat token. */
 export function writePrivateFile(path, text) {
@@ -83,6 +88,17 @@ export const LOCK_NAME = "session";
 export const DEFAULT_POLL_MS = 2_000;
 /** The signals the supervisor owns (launchd sends SIGTERM to stop a job). */
 export const OWNED_SIGNALS = Object.freeze(["SIGTERM", "SIGINT"]);
+/**
+ * How long after start the pane is captured ONCE, to keep whatever the runtime
+ * printed before it died. A resume against a missing conversation is over in
+ * about two seconds, taking its mux session (and its pane) with it — so the
+ * capture has to happen while it is still there. Best-effort by design: when
+ * it comes back empty, `resumeTargetState`'s preflight is what carries the
+ * verdict, and the post-mortem heuristics below carry the rest.
+ */
+export const EARLY_PANE_MS = 1_500;
+/** Consecutive identical launch failures before the seat escalates to the org. */
+export const LAUNCH_FAILURE_LIMIT = IDENTICAL_FAILURE_LIMIT;
 
 /** Real signal wiring: `fn(signal)` on each owned signal. */
 function installSignalHandlers(fn) {
@@ -198,6 +214,15 @@ export async function runSupervisor(deps = {}) {
     seatSpawn: deps.seatSpawn || ((root) => resolveSeatSpawn(root)),
     writePrivateFile: deps.writePrivateFile || writePrivateFile,
     envFileId: deps.envFileId || randomUUID,
+    readdirSync: deps.readdirSync || fsReaddirSync,
+    launchFailureLimit: deps.launchFailureLimit ?? LAUNCH_FAILURE_LIMIT,
+    earlyPaneMs: deps.earlyPaneMs ?? EARLY_PANE_MS,
+    // The one-shot pane probe has its OWN timer seam, separate from the
+    // watchdog's `after`. They are different clocks doing different jobs — one
+    // fires once at a second and a half, the other recurs at the grace period —
+    // and collapsing them would make every watchdog test reason about a timer
+    // that has nothing to do with the watchdog.
+    afterPaneProbe: deps.afterPaneProbe || deps.after || defaultAfter,
   };
   const fsDeps = { readFileSync: d.readFileSync, writeJsonAtomic: d.writeJsonAtomic };
   const agentRoot = d.agentRoot;
@@ -271,7 +296,7 @@ export async function runSupervisor(deps = {}) {
   // 4. Identity + argv.
   let record = loadMainSession(agentRoot, fsDeps);
   if (!record || cfg.resume === false) record = newMainSession({ now: d.now, uuid: d.uuid });
-  const mode = launchMode(record, cfg);
+  let mode = launchMode(record, cfg);
   let permissionArgs = d.permissionArgs;
   if (!permissionArgs) {
     permissionArgs = sessionPermissionArgs({ source: MAIN_SESSION_SOURCE, allowedTools: cfg.allowedTools });
@@ -288,6 +313,38 @@ export async function runSupervisor(deps = {}) {
   let seat = { engine: "claude", fields: {} };
   try { seat = (await d.seatSpawn(agentRoot)) || seat; } catch { seat = { engine: "claude", fields: {} }; }
   const engineCohort = seat.engine === "cohort";
+
+  // ── (a) A DEAD RESUME TARGET IS CAUGHT BEFORE THE LAUNCH, NOT AFTER IT ────
+  //
+  // James Kirkland's seat, measured 2026-09-25: `claude --resume <id>` for a
+  // conversation that no longer existed on that machine, exit 1 after 2 s,
+  // relaunched every ten minutes for DAYS. state/session/main-session.json held
+  // the dead id and nothing ever invalidated it.
+  //
+  // THE CHOICE, MADE EXPLICIT: invalidate the record AND fall back to a fresh
+  // id — both, in one write, here, before anything is spawned. Invalidating
+  // alone would leave the supervisor with no id (and `newMainSession` would
+  // mint one anyway, unrecorded); falling back alone would leave the dead id on
+  // disk for the next reader to resume again. `rotateMainSession` does both:
+  // the dead id survives as `rotatedFrom` — provenance, never a target.
+  //
+  // It runs on POSITIVE EVIDENCE ONLY (`resume-target.mjs` fails open): a
+  // wrong "missing" throws away a live transcript, a wrong "unknown" costs one
+  // launch and lands in the post-mortem that was already there.
+  if (mode === "resume") {
+    const target = resumeTargetState(
+      { homeDir: d.homeDir, projectDir: agentRoot, sessionId: record.sessionId, engine: seat.engine },
+      { readdirSync: d.readdirSync },
+    );
+    if (target.state === "missing") {
+      const dead = record.sessionId;
+      record = rotateMainSession(record, { now: d.now, uuid: d.uuid, reason: `resume target ${dead} is not on this machine` });
+      mode = launchMode(record, cfg);
+      const res = saveMainSession(agentRoot, record, fsDeps);
+      d.log(`resume target ${dead} is gone (${target.reason}) — invalidated it and starting a fresh session ${record.sessionId}${res.ok ? "" : ` (not saved: ${res.error})`}`);
+    }
+  }
+
   const spec = buildSpawn({ lane: "main-session", bin: d.claudeBin || undefined, first, sessionId: record.sessionId, resumeMode: mode, permissions: permissionArgs, env: d.env, ...seat.fields });
   if (!spec.ok && engineCohort) {
     // An engine-cohort seat never falls back to claude (it would spend the seat's
@@ -348,8 +405,34 @@ export async function runSupervisor(deps = {}) {
   const restartsSoFar = carriedRestarts({ attention: priorAttention, heartbeat: priorHeartbeat });
   if (restartsSoFar > 0) d.log(`this launch carries ${restartsSoFar} silent restart(s) from the previous run`);
 
+  // ── (b) THE CARRIED LAUNCH-FAILURE STREAK ────────────────────────────────
+  //
+  // Read BEFORE the unlink below, for the same reason `carriedRestarts` is: a
+  // count held in memory is zero on every launch, because every launch is a
+  // fresh process. This one lives in its own file rather than on the attention
+  // record, because it has to survive the unlink that clears that record.
+  let priorFailures = null;
+  try { priorFailures = parseLaunchFailures(d.readFileSync(paths.launchFailuresFile, "utf8")); } catch { priorFailures = null; }
+
   try { d.unlinkSync(paths.lastExitFile); } catch { /* none from a previous run */ }
   try { d.unlinkSync(paths.attentionFile); } catch { /* none outstanding */ }
+
+  // A seat already past the limit re-states the escalation on EVERY launch.
+  // The note rides `machine.sessionNote` on the presence beat, and the unlink
+  // above just cleared it: without this the org would see the alert blink out
+  // each time the seat tried again, which reads as a seat that healed.
+  if (priorFailures) {
+    const carried = escalationDecision({ streak: priorFailures.streak, limit: d.launchFailureLimit, escalatedAt: priorFailures.escalatedAt });
+    if (carried.escalate) {
+      writeLaunchAttention({
+        d, paths, mux, muxName, now: d.now(),
+        streak: carried.streak, limit: carried.limit, signature: priorFailures.signature,
+        detail: `${priorFailures.signature} since ${priorFailures.firstAt || "an earlier launch"}`,
+        sessionId: record.sessionId, phase: "retrying",
+      });
+      d.log(`launch-failure escalation still standing: ${carried.reason}`);
+    }
+  }
   // 4c. An upgrade notice asking for a restart onto the version THIS launch
   //     runs is honoured by this launch: retire it, so `maestro session
   //     status` stops flagging a hop the session has completed (and a notice
@@ -380,6 +463,9 @@ export async function runSupervisor(deps = {}) {
   //    almost certainly waiting on a dialog or a prompt nobody can see.
   const startedAt = Number(d.now());
   let launchFailed = false;
+  /** Whatever the mux client itself printed — `screen -D -m` reports a missing
+   *  command here, and it is the one piece of evidence no pane capture can race. */
+  let startStdout = "";
   stopCmd = cmds.stop;
   // ── THE WATCHDOG ────────────────────────────────────────────────────────
   //
@@ -400,6 +486,10 @@ export async function runSupervisor(deps = {}) {
   let clears = 0;
   let restarts = restartsSoFar;
   let watchdogStopped = false;
+  /** The last non-empty pane the watchdog saw — evidence for the post-mortem. */
+  let lastPane = "";
+  /** Did the WATCHDOG end this session? A death we caused is never rotated on. */
+  let watchdogRestarted = false;
   let cancelPass = () => {};
   const paneFile = join(agentRoot, "state", "session", "pane-capture.txt");
 
@@ -412,6 +502,17 @@ export async function runSupervisor(deps = {}) {
     } catch { return ""; }
   };
 
+  // The pane is captured ONCE, early, and kept. A resume whose conversation is
+  // gone prints "No conversation found with session ID: <id>" and exits inside
+  // a couple of seconds, taking the mux session and its pane with it — so the
+  // only moment this evidence exists is while the doomed session is still up.
+  // Best-effort: an empty capture is not an absence of fault, it is an absence
+  // of evidence, and `classifyLaunchFailure` is written to say so.
+  let earlyPane = "";
+  d.afterPaneProbe(d.earlyPaneMs, () => {
+    capturePane().then((text) => { if (!earlyPane) earlyPane = paneTail(text); }).catch(() => {});
+  });
+
   const sendKeys = async (keys) => {
     for (const k of keys) {
       const cmd = sendKeyCommand(mux, muxName, k);
@@ -433,6 +534,7 @@ export async function runSupervisor(deps = {}) {
     if (!silence.silent) { schedulePass(); return; }
 
     const pane = paneTail(await capturePane());
+    if (pane) lastPane = pane;
     const seen = classifyPane(pane);
     const act = watchdogAction({ silent: true, kind: seen.kind, keys: seen.keys, clears, restarts });
 
@@ -459,6 +561,7 @@ export async function runSupervisor(deps = {}) {
       await sendKeys(seen.keys);
     } else if (act.act === "restart") {
       restarts = nextRestarts;
+      watchdogRestarted = true;
       d.log(`session ${muxName}: restarting (${restarts}) — the session is not answering and the screen cannot be cleared safely`);
       try { await d.execFile(cmds.stop.file, cmds.stop.args, { cwd: agentRoot, env: d.env }); } catch { /* the relaunch loop handles a dead mux */ }
       watchdogStopped = true; // the supervisor's own loop relaunches; do not fight it
@@ -485,6 +588,7 @@ export async function runSupervisor(deps = {}) {
     // env file; the mux client itself gets the supervisor's env, as for claude.
     if (envFile) d.writePrivateFile(envFile, renderEnvFile(spec.env));
     const r = await d.execFile(cmds.start.file, cmds.start.args, { cwd: agentRoot, env: d.env });
+    startStdout = `${String((r && r.stdout) || "")}${String((r && r.stderr) || "")}`.trim();
     if (r && r.code !== 0) { launchFailed = true; d.log(`${mux.kind} start exited ${r.code}${r.stderr ? `: ${String(r.stderr).trim()}` : ""}`); }
     else if (!cmds.blocking) await waitForProbe(cmds.waitProbe, d);
   } catch (err) {
@@ -506,11 +610,69 @@ export async function runSupervisor(deps = {}) {
     d.log(`session ${muxName} stopped on ${stopping} — id kept, exit ${EXIT_RELAUNCH}`);
     return EXIT_RELAUNCH;
   }
-  const decision = rotationDecision({ record: launched, mode, exitCode, startedAt, endedAt, now: d.now });
+
+  // Did THIS launch ever report in? The single most useful fact about a run,
+  // and the one the ten-minute relaunch loop never asked: James's seat looked
+  // up for days and had not beaten once.
+  let endHeartbeat = null;
+  try { endHeartbeat = parseHeartbeat(d.readFileSync(paths.heartbeatFile, "utf8")); } catch { endHeartbeat = null; }
+  const endBeatMs = endHeartbeat && typeof endHeartbeat.ts === "string" ? Date.parse(endHeartbeat.ts) : NaN;
+  const beatSeen = Number.isFinite(endBeatMs) && endBeatMs >= startedAt;
+
+  // What the runtime itself said, if we caught it. `earlyPane` is the capture
+  // taken while a fast-failing session was still up; `lastPane` is the
+  // watchdog's. `startStdout` is whatever the mux client printed to us.
+  const evidence = [startStdout, earlyPane, lastPane].filter(Boolean).join("\n");
+  const failure = classifyLaunchFailure({ mode, exitCode, output: evidence, beatSeen });
+  if (failure.kind !== "none") d.log(`launch verdict: ${failure.kind}${failure.proven ? " (proven)" : ""} — ${failure.detail}`);
+
+  // ── (b) N IDENTICAL FAILURES ARE A FAULT, NOT N RETRIES ──────────────────
+  //
+  // The ten-minute backoff was the right instinct and the wrong outcome: every
+  // attempt failed identically and the seat stayed quiet about it for days,
+  // because the only record was a log on a machine nobody can reach. So the
+  // streak is counted across supervisor lifetimes and, at the limit, written
+  // where it LEAVES the machine — `state/session/attention.json` rides the
+  // presence beat as `machine.sessionNote`, which is the org's view of this
+  // seat. A watchdog restart is not counted here: that path has its own
+  // bounded budget (`carriedRestarts`) and counting it twice would escalate a
+  // fault the seat is already handling.
+  let escalation = { escalate: false, fresh: false, streak: 0, limit: d.launchFailureLimit, reason: "" };
+  if (!watchdogRestarted) {
+    const signature = launchFailureSignature({ mode, kind: failure.kind, exitCode });
+    const next = launchFailureStreak(priorFailures, { signature, at: new Date(Number(d.now())).toISOString() });
+    if (!next) {
+      // The launch worked. Evidence of work clears the record — the same reset
+      // rule `carriedRestarts` uses, and for the same reason.
+      try { d.unlinkSync(paths.launchFailuresFile); } catch { /* nothing carried */ }
+    } else {
+      escalation = escalationDecision({ streak: next.streak, limit: d.launchFailureLimit, escalatedAt: next.escalatedAt });
+      if (escalation.escalate && !next.escalatedAt) next.escalatedAt = new Date(Number(d.now())).toISOString();
+      try { d.writeJsonAtomic(paths.launchFailuresFile, next); } catch { /* the log line still says it */ }
+      d.log(`launch failure ${next.streak}× in a row (${signature}) — ${escalation.reason}`);
+      if (escalation.escalate) {
+        writeLaunchAttention({
+          d, paths, mux, muxName, now: d.now(),
+          streak: next.streak, limit: escalation.limit, signature,
+          detail: failure.detail, sessionId: launched.sessionId, phase: "failed",
+        });
+        if (escalation.fresh) d.log(`escalating to the org: ${escalationHint({ signature, streak: next.streak, limit: escalation.limit, detail: failure.detail, sessionId: launched.sessionId })}`);
+      }
+    }
+  }
+
+  const decision = rotationDecision({
+    record: launched, mode, exitCode, startedAt, endedAt, now: d.now,
+    resumeTargetMissing: failure.kind === "resume-target-missing",
+    beatSeen,
+    streakAtLimit: escalation.escalate,
+    selfStopped: watchdogRestarted,
+  });
   if (decision.action === "rotate") {
-    const rotated = rotateMainSession(launched, { now: d.now, uuid: d.uuid });
+    const why = decision.why || "the resume failed";
+    const rotated = rotateMainSession(launched, { now: d.now, uuid: d.uuid, reason: why });
     const res = saveMainSession(agentRoot, rotated, fsDeps);
-    d.log(`resume of ${launched.sessionId} failed inside the failure window — rotated to ${rotated.sessionId}${res.ok ? "" : ` (not saved: ${res.error})`}`);
+    d.log(`resume of ${launched.sessionId}: ${why}${decision.proven ? " (proven, so not rationed by the rotation budget)" : ""} — invalidated it and rotated to ${rotated.sessionId}${res.ok ? "" : ` (not saved: ${res.error})`}`);
   } else if (decision.action === "backoff") {
     d.log(`rotation budget spent for this hour — sleeping ${decision.sleepMs / 60000} min before relaunch`);
     await d.sleep(decision.sleepMs);
@@ -518,6 +680,37 @@ export async function runSupervisor(deps = {}) {
   return EXIT_RELAUNCH;
 }
 
+/**
+ * Write the escalation where it LEAVES THE MACHINE.
+ *
+ * `state/session/attention.json` is read by `lib/telemetry/collect#sessionNote`
+ * and rides the presence beat to hq as `machine.sessionNote` — the only channel
+ * out of a seat that accepts no ssh. The record is shaped exactly like the
+ * watchdog's (`first-run#attentionRecord` fields) so the existing reader needs
+ * no special case; `restarts: 0` is honest and, per `carriedRestarts`, cannot
+ * steal budget from the watchdog.
+ *
+ * @param {object} a
+ */
+export function writeLaunchAttention(a) {
+  const { d, paths, mux, muxName } = a;
+  const attach = mux && mux.kind === "tmux" ? `tmux attach -t =${muxName}` : `screen -r ${muxName}`;
+  const record = {
+    reason: "launch-failing",
+    since: new Date(Number(a.now)).toISOString(),
+    runMs: 0,
+    restarts: 0,
+    action: a.phase === "retrying" ? "wait" : "give-up",
+    attach,
+    signature: a.signature,
+    streak: a.streak,
+    limit: a.limit,
+    hint: escalationHint({ signature: a.signature, streak: a.streak, limit: a.limit, detail: a.detail, sessionId: a.sessionId }),
+  };
+  try { d.writeJsonAtomic(paths.attentionFile, record); } catch { /* the log line still says it */ }
+  return record;
+}
+
 const isMain = (() => {
   try { return process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; } catch { return false; }
 })();