@cohortapp/agent-sdk 2.18.13 → 2.18.15

package/lib/session/launch-failure.mjs

@@ -0,0 +1,251 @@
+/**
+ * lib/session/launch-failure.mjs — WHY a main-session launch failed, and when
+ * the seat should stop trying quietly and say so. Pure; no clock, no fs.
+ *
+ * ── THE FAULT THIS MODULE EXISTS TO NOT HAVE ────────────────────────────────
+ * Measured on James Kirkland's seat, 2026-09-25. The supervisor relaunched
+ * every ten minutes for DAYS running
+ *
+ *     claude --resume e953a33e-97a0-4d13-9fd4-6a77cd7a68c9
+ *     No conversation found with session ID: e953a33e-…
+ *     exit 1, after ~2 s
+ *
+ * for a conversation that no longer existed on that machine. Every layer
+ * behaved "correctly": launchd relaunched a job that exited non-zero, the
+ * rotation budget throttled to one attempt per ten minutes, the log recorded
+ * each failure. The seat looked UP the whole time — a live launchd job, a
+ * supervisor process, a mux session flickering into existence — and never
+ * beat once. Nobody reads that log; nobody could ssh in to read it.
+ *
+ * Two things were missing and both are here:
+ *
+ *   1. THE FAILURE HAS A NAME AND THE NAME WAS ON THE SCREEN. "No conversation
+ *      found with session ID" is not a symptom to infer from timing — it is the
+ *      runtime telling us the resume target is gone. {@link classifyLaunchFailure}
+ *      reads it, so the repair (mint a fresh id) is taken on evidence rather
+ *      than on the 20-second heuristic in `identity#rotationDecision`, which
+ *      misses a failure that takes 21 seconds and cannot distinguish a dead
+ *      transcript from a missing binary.
+ *
+ *   2. N IDENTICAL FAILURES ARE NOT N CHANCES — they are one fault, repeated.
+ *      A launch that fails the same way three times in a row will fail the
+ *      fourth time too, and the only useful next step is to tell somebody who
+ *      is not on this machine. {@link launchFailureStreak} counts consecutive
+ *      failures WITH THE SAME SIGNATURE across supervisor lifetimes (each
+ *      launch is a fresh process, so an in-memory counter is always zero —
+ *      the same trap `first-run#carriedRestarts` exists for), and
+ *      {@link escalationDecision} says when it stops being a retry and becomes
+ *      an escalation.
+ *
+ * The signature deliberately does NOT include the session id: rotating to a
+ * fresh id and failing the same way again is the SAME fault (that is exactly
+ * what a missing binary does), and a signature that changed every rotation
+ * would reset the streak forever — which is how this failure stayed invisible.
+ *
+ * @module lib/session/launch-failure
+ */
+
+"use strict";
+
+/** Consecutive identical launch failures before the seat escalates. */
+export const IDENTICAL_FAILURE_LIMIT = 3;
+
+/**
+ * The runtime's own words for "the conversation you asked me to resume is not
+ * on this machine". Matched case-insensitively against the tail of the pane /
+ * the launcher's stderr. Kept as a list because the wording is the CLI's, not
+ * ours, and a second phrasing must be addable without touching the classifier.
+ */
+export const RESUME_MISSING_PATTERNS = Object.freeze([
+  /no conversation found with session id/i,
+  /no conversation found matching/i,
+  /session .{0,80}? not found/i,
+]);
+
+/** "the binary is not there" — a shell's 127, and the two spellings of ENOENT. */
+export const BINARY_MISSING_PATTERNS = Object.freeze([
+  /command not found/i,
+  /no such file or directory/i,
+  /\bENOENT\b/,
+]);
+
+const anyMatch = (patterns, text) => patterns.some((re) => re.test(text));
+
+/**
+ * Pure: name the failure of one launch.
+ *
+ * `output` is whatever text the launch left behind — a pane capture, the
+ * launcher's stderr, or "" when nothing was captured. It is EVIDENCE, and its
+ * absence is never treated as proof of anything: with no output the verdict
+ * falls back to what the exit code and the mode can carry on their own.
+ *
+ * @param {object} a
+ * @param {"session-id"|"resume"} a.mode      how this launch addressed the session
+ * @param {number|null} a.exitCode            the recorded exit (null = the exit file was never written)
+ * @param {string} [a.output]                 pane tail / stderr, may be ""
+ * @param {boolean} [a.beatSeen]              did this launch produce a heartbeat?
+ * @returns {{kind:"none"|"resume-target-missing"|"binary-missing"|"never-beaten"|"nonzero-exit", proven:boolean, detail:string}}
+ */
+export function classifyLaunchFailure(a = {}) {
+  const exitCode = a.exitCode === null || a.exitCode === undefined ? null : Number(a.exitCode);
+  const output = typeof a.output === "string" ? a.output : "";
+  const beatSeen = a.beatSeen === true;
+  const failed = exitCode === null || exitCode !== 0;
+
+  // A launch that beat and then exited 0 is a session that RAN. Nothing to name.
+  if (!failed && beatSeen) return { kind: "none", proven: true, detail: "the session ran and exited cleanly" };
+  if (!failed) return { kind: "none", proven: false, detail: "the session exited cleanly without beating" };
+
+  // PROVEN verdicts first: the runtime said what was wrong, in words.
+  if (a.mode === "resume" && anyMatch(RESUME_MISSING_PATTERNS, output)) {
+    return {
+      kind: "resume-target-missing",
+      proven: true,
+      detail: "the runtime reported that the conversation this launch tried to resume does not exist on this machine",
+    };
+  }
+  if (exitCode === 127 || anyMatch(BINARY_MISSING_PATTERNS, output)) {
+    return {
+      kind: "binary-missing",
+      proven: exitCode === 127 || anyMatch(BINARY_MISSING_PATTERNS, output),
+      detail: "the launcher could not find the binary it was told to run",
+    };
+  }
+
+  // INFERRED verdicts. `never-beaten` is the honest name for James's seat when
+  // the pane was not captured: the launch ended non-zero having never reported
+  // in, so whatever it was, it was never a working session.
+  if (!beatSeen) {
+    return { kind: "never-beaten", proven: false, detail: `the launch ended (exit ${exitCode === null ? "unknown" : exitCode}) without ever writing a heartbeat` };
+  }
+  return { kind: "nonzero-exit", proven: false, detail: `the session beat, then ended with exit ${exitCode === null ? "unknown" : exitCode}` };
+}
+
+/**
+ * Pure: the stable identity of a failure, for counting repeats.
+ *
+ * Mode and kind only. NOT the session id (see the module docblock) and not the
+ * exit code when the kind already names the fault — otherwise a flapping exit
+ * status would read as a different fault each time and never accumulate.
+ *
+ * @param {{mode:string, kind:string, exitCode?:number|null}} a
+ * @returns {string} e.g. `resume:resume-target-missing`
+ */
+export function launchFailureSignature(a = {}) {
+  const kind = String(a.kind || "unknown");
+  const mode = a.mode === "resume" ? "resume" : "session-id";
+  if (kind === "none") return "";
+  if (kind === "never-beaten" || kind === "nonzero-exit") {
+    const exit = a.exitCode === null || a.exitCode === undefined ? "unknown" : String(Number(a.exitCode));
+    return `${mode}:${kind}:${exit}`;
+  }
+  return `${mode}:${kind}`;
+}
+
+/**
+ * Pure: parse `state/session/launch-failures.json`. Anything unreadable is
+ * "no streak" — a corrupt counter must never be able to escalate on its own.
+ * @param {string|null|undefined} text
+ * @returns {{signature:string, streak:number, firstAt:string, lastAt:string, escalatedAt:string}|null}
+ */
+export function parseLaunchFailures(text) {
+  if (typeof text !== "string" || !text.trim()) return null;
+  let raw;
+  try { raw = JSON.parse(text); } catch { return null; }
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
+  const signature = typeof raw.signature === "string" ? raw.signature : "";
+  const streak = Number.isInteger(raw.streak) && raw.streak > 0 ? raw.streak : 0;
+  if (!signature || !streak) return null;
+  return {
+    signature,
+    streak,
+    firstAt: typeof raw.firstAt === "string" ? raw.firstAt : "",
+    lastAt: typeof raw.lastAt === "string" ? raw.lastAt : "",
+    escalatedAt: typeof raw.escalatedAt === "string" ? raw.escalatedAt : "",
+  };
+}
+
+/**
+ * Pure: the streak after one more launch outcome.
+ *
+ * A DIFFERENT signature restarts the count at 1 rather than adding to it: two
+ * different faults in a row are two problems, and escalating on their sum
+ * would cry wolf. A successful launch (`signature` empty) clears the record
+ * entirely — evidence of work, the same reset rule `carriedRestarts` uses.
+ *
+ * @param {object|null} prior      {@link parseLaunchFailures} output
+ * @param {{signature:string, at:string}} outcome
+ * @returns {{signature:string, streak:number, firstAt:string, lastAt:string, escalatedAt:string}|null} null = clear the file
+ */
+export function launchFailureStreak(prior, outcome = {}) {
+  const signature = typeof outcome.signature === "string" ? outcome.signature : "";
+  const at = typeof outcome.at === "string" ? outcome.at : "";
+  if (!signature) return null; // the launch worked: nothing to carry
+  if (prior && prior.signature === signature) {
+    return { signature, streak: prior.streak + 1, firstAt: prior.firstAt || at, lastAt: at, escalatedAt: prior.escalatedAt || "" };
+  }
+  return { signature, streak: 1, firstAt: at, lastAt: at, escalatedAt: "" };
+}
+
+/**
+ * Pure: has this stopped being a retry?
+ *
+ * `escalate` is true on the launch that REACHES the limit and on every launch
+ * after it, because the escalation channel is a beat field that the next
+ * supervisor run clears (`attention.json` is unlinked at launch): a one-shot
+ * escalation would flicker off and the org would see a seat that healed. The
+ * `fresh` flag distinguishes the first crossing, which is the one worth a log
+ * line and a distinct wording.
+ *
+ * @param {{streak:number, limit?:number, escalatedAt?:string}} a
+ * @returns {{escalate:boolean, fresh:boolean, streak:number, limit:number, reason:string}}
+ */
+export function escalationDecision(a = {}) {
+  const limit = Number.isInteger(a.limit) && a.limit > 0 ? a.limit : IDENTICAL_FAILURE_LIMIT;
+  const streak = Number.isInteger(a.streak) && a.streak > 0 ? a.streak : 0;
+  if (streak < limit) {
+    return { escalate: false, fresh: false, streak, limit, reason: `launch failure ${streak}/${limit} — retrying` };
+  }
+  const fresh = !a.escalatedAt;
+  return {
+    escalate: true,
+    fresh,
+    streak,
+    limit,
+    reason: `${streak} consecutive launches have failed the same way (limit ${limit}) — this is a fault, not a retry`,
+  };
+}
+
+/**
+ * Pure: the one sentence an operator in a browser gets, via the beat's
+ * `machine.sessionNote.detail`.
+ *
+ * THE VERDICT LEADS, THE EXPLANATION TRAILS — because the channel TRUNCATES.
+ * `telemetry/collect#sanitizeNoteDetail` caps the detail at 200 characters and
+ * appends an ellipsis, so anything after the first sentence may never leave the
+ * machine. The first draft of this line ended "…this needs a person" and that
+ * clause — the only part that asks for an action — was the part that got cut.
+ * So: how many, that retries are not working, then the diagnosis.
+ *
+ * @param {{signature:string, streak:number, limit?:number, detail:string, sessionId?:string}} a
+ * @returns {string}
+ */
+export function escalationHint(a = {}) {
+  const sig = String(a.signature || "unknown");
+  const streak = Number(a.streak) || 0;
+  const detail = String(a.detail || "").trim();
+  const id = typeof a.sessionId === "string" && a.sessionId ? ` Session ${a.sessionId}.` : "";
+  return `${streak}× in a row, identically — retries are not fixing it and this needs a person. ${detail || sig}.${id}`;
+}
+
+export default {
+  IDENTICAL_FAILURE_LIMIT,
+  RESUME_MISSING_PATTERNS,
+  BINARY_MISSING_PATTERNS,
+  classifyLaunchFailure,
+  launchFailureSignature,
+  parseLaunchFailures,
+  launchFailureStreak,
+  escalationDecision,
+  escalationHint,
+};

package/lib/session/resume-target.mjs

@@ -0,0 +1,86 @@
+/**
+ * lib/session/resume-target.mjs — is the conversation we are about to `--resume`
+ * actually on this machine?
+ *
+ * ── WHY THIS IS A PREFLIGHT AND NOT A POST-MORTEM ───────────────────────────
+ * The supervisor already had a post-mortem: a resume that died non-zero inside
+ * twenty seconds rotates to a fresh id (`identity#rotationDecision`). It is a
+ * good heuristic and it is not enough — it cannot tell a dead transcript from a
+ * missing binary, it misses a failure that takes twenty-one seconds, and every
+ * one of its verdicts costs a full launch, a mux session and a launchd cycle to
+ * reach. On James Kirkland's seat that cycle ran every ten minutes for days
+ * against a session id that had not existed for just as long.
+ *
+ * The answer is on disk before we launch. Claude Code keeps each project's
+ * transcripts as `~/.claude/projects/<project-slug>/<session-id>.jsonl`, where
+ * the slug is the project directory with every non-alphanumeric character
+ * replaced by `-`. `--resume` is project-scoped, so the seat's own slug is the
+ * only place the id could be.
+ *
+ * ── FAIL-OPEN, DELIBERATELY ─────────────────────────────────────────────────
+ * A wrong "missing" throws away a live transcript; a wrong "unknown" costs one
+ * launch and lands in the post-mortem that was already there. So `missing` is
+ * returned ONLY from positive evidence: the project directory exists, it holds
+ * transcripts, and this id is not among them. An absent directory, an empty
+ * one, an unreadable one, an engine that does not use this layout at all — all
+ * `unknown`, and the launch proceeds exactly as before.
+ *
+ * One I/O shell (`readdirSync`/`existsSync`), both injected.
+ *
+ * @module lib/session/resume-target
+ */
+
+"use strict";
+
+import { readdirSync as fsReaddirSync } from "node:fs";
+import { join } from "node:path";
+
+/** Where Claude Code keeps per-project transcripts, relative to the home dir. */
+export const TRANSCRIPT_ROOT = join(".claude", "projects");
+
+/**
+ * Pure: Claude Code's directory name for a project path — every character that
+ * is not a letter or a digit becomes `-` (`/Users/x/hq` → `-Users-x-hq`).
+ * @param {string} projectDir
+ * @returns {string}
+ */
+export function projectSlug(projectDir) {
+  return String(projectDir || "").replace(/[^A-Za-z0-9]/g, "-");
+}
+
+/**
+ * Is `sessionId`'s transcript present for `projectDir`?
+ *
+ * @param {object} a
+ * @param {string} a.homeDir              the user's home directory
+ * @param {string} a.projectDir           the directory the session runs in (the agent root)
+ * @param {string} a.sessionId            the id this launch would resume
+ * @param {"claude"|"cohort"} [a.engine]  a non-claude engine does not use this layout → always "unknown"
+ * @param {{readdirSync?:Function}} [deps]
+ * @returns {{state:"present"|"missing"|"unknown", dir:string, reason:string}}
+ */
+export function resumeTargetState(a = {}, deps = {}) {
+  const readdirSync = deps.readdirSync || fsReaddirSync;
+  const homeDir = typeof a.homeDir === "string" ? a.homeDir : "";
+  const sessionId = String(a.sessionId || "").toLowerCase();
+  if (a.engine && a.engine !== "claude") return { state: "unknown", dir: "", reason: `engine ${a.engine} does not keep transcripts here` };
+  if (!homeDir || !sessionId) return { state: "unknown", dir: "", reason: "no home directory or session id to check" };
+  const dir = join(homeDir, TRANSCRIPT_ROOT, projectSlug(a.projectDir));
+  let entries;
+  try { entries = readdirSync(dir); } catch { return { state: "unknown", dir, reason: "the project's transcript directory is absent or unreadable" }; }
+  const names = (Array.isArray(entries) ? entries : []).map((e) => String(e && e.name ? e.name : e));
+  const transcripts = names.filter((n) => n.toLowerCase().endsWith(".jsonl"));
+  if (transcripts.some((n) => n.toLowerCase() === `${sessionId}.jsonl`)) {
+    return { state: "present", dir, reason: "the transcript is on disk" };
+  }
+  // An EMPTY directory proves nothing: it is equally what a seat looks like the
+  // moment before its first session writes, and rotating on it would be a guess.
+  if (transcripts.length === 0) return { state: "unknown", dir, reason: "the project's transcript directory holds no transcripts at all" };
+  return {
+    state: "missing",
+    dir,
+    reason: `${transcripts.length} transcript(s) on disk for this project and none of them is ${sessionId}`,
+  };
+}
+
+export default { TRANSCRIPT_ROOT, projectSlug, resumeTargetState };

package/lib/telemetry/alerts.mjs

@@ -19,6 +19,10 @@
  *   - subAgentsRunning over the cap      → INFO (more children than expected).
  *   - machine.sessionNote present        → WARNING/CRITICAL (frontdoor) — the
  *                                          seat's front door is not answering.
+ *   - machine.upgrade.failStreak >= N    → WARNING/CRITICAL (rollout) — N
+ *                                          consecutive attempts to reach
+ *                                          @latest have failed; the seat is
+ *                                          stranded on old code.
  *
  * Output: a deterministic array `[{ id, severity, kind, detail }, ...]` (the
  * snapshot's alert shape) sorted critical→warning→info then by id, so identical
@@ -90,6 +94,30 @@ export const DEFAULT_THRESHOLDS = Object.freeze({
     warnMs: 15 * 60 * 1000,
     critMs: 2 * 60 * 60 * 1000,
   }),
+  /**
+   * How many CONSECUTIVE failed attempts to reach @latest make a seat STUCK.
+   *
+   * `warnStreak` 3, not 1: one failure is a bad release or a slow mirror and
+   * the hourly job is entitled to try again; the failed-target hold already
+   * spaces those out. Three separate, completed attempts have failed, which
+   * means every fix the seat can apply to itself has been applied three times
+   * and the seat is still where it was.
+   *
+   * `critStreak` 6: past this the seat has been refusing @latest for the best
+   * part of a week under the default 24 h hold — which is exactly how long
+   * three seats sat on 2.17.0 while every hourly log line was individually
+   * correct. A release that cannot reach a colleague is not a machine problem
+   * that resolves itself; it waits for a person, like a scope fault.
+   */
+  upgradeStuck: Object.freeze({
+    // THE ONE DEFINITION of the stuck threshold. scripts/fleet/rollout.mjs
+    // imports `warnStreak` as its STUCK_STREAK rather than restating it, and
+    // the shell copy (autoupdate.sh) is pinned against this value by a
+    // source-reading test — because three literals agreeing by comment is the
+    // prose-promise-instead-of-a-check pattern this repo bans.
+    warnStreak: 3,
+    critStreak: 6,
+  }),
   replyDebt: Object.freeze({
     // ONE withheld reply is already worth saying. This is not a resource gauge
     // where a low reading is normal noise — it counts people who asked this
@@ -175,6 +203,7 @@ export function deriveAlerts(status, thresholds, now = Date.now()) {
     push(alerts, frontDoorRule(machine, t.frontDoor, now));
     push(alerts, scopeFaultRule(machine));
     push(alerts, repliesWithheldRule(machine, t.replyDebt));
+    push(alerts, upgradeStuckRule(machine, t.upgradeStuck, now));
 
     alerts.sort((a, b) => {
       const r = (SEVERITY_RANK[a.severity] ?? 9) - (SEVERITY_RANK[b.severity] ?? 9);
@@ -282,6 +311,71 @@ function repliesWithheldRule(machine, cfg = {}) {
   };
 }
 
+/**
+ * THIS SEAT CANNOT GET TO @latest, AND HAS PROVED IT N TIMES.
+ *
+ * Every other rule in this file reads a gauge. This one reads a COUNT of
+ * completed failures, because the failure it names has no gauge: on 2026-09-25
+ * three seats were found sitting on 2.17.0 while the fleet ran 2.18.13, and
+ * nothing anywhere was in an error state. The launchd job fired hourly, npm
+ * answered, the install ran, the health gate did its job, the rollback worked,
+ * `last.json` recorded the outcome honestly and the beat carried it. Each hour
+ * was a correct, self-contained failure — and the org has no way to see the
+ * difference between the first one and the fortieth, which is the only thing
+ * that distinguishes "a release is settling" from "a colleague is stranded on
+ * code from three weeks ago and will never leave it unaided".
+ *
+ * WHY THIS RIDES THE EXISTING BEAT AND NOTHING ELSE. The seats this rule is
+ * about are, by construction, running the OLDEST code in the fleet — so any
+ * new channel it might use is the one channel they do not have. `machine.upgrade`
+ * is a field their beat already carries; the count is added to it, and this
+ * rule turns it into an alert on the seats new enough to run this file.
+ *
+ * FOR THE ONES THAT ARE NOT, THIS RULE IS STRUCTURALLY BLIND AND SAYING SO IS
+ * PART OF THE DESIGN. A seat too stale to install the SDK runs a collector that
+ * never writes `failStreak` and an alerts file that has never heard of this
+ * rule — so the alert cannot reach exactly the seats it was written for. The
+ * fleet-side derivation covers them from outside: `seatStuck()` in
+ * scripts/fleet/rollout.mjs reads the same fact out of a 2.17-era beat, and
+ * `propagationOutcome()` there returns reason `"stuck"` so that stage 3 of
+ * every publish exits non-zero and names the machine. That path is automatic
+ * because publishing is; nobody has to decide to go looking.
+ *
+ * Both answer (d) the same way: a streak counts ATTEMPTS, and a seat that has
+ * not been asked has made none.
+ *
+ * DISTINCT FROM DRIFT. A `stranded` seat (framework paths pinned in
+ * `.maestroignore`, `machine.stranded`) cannot RECEIVE parts of a release it
+ * did install; a STUCK seat never installs the release at all. Different
+ * cause, different fix, different alert id — they can be true at once.
+ */
+function upgradeStuckRule(machine, cfg = {}, now = Date.now()) {
+  const u = machine && machine.upgrade;
+  if (!u || typeof u !== "object") return null;
+  // Strict: the producer (autoupdate.sh via collect.upgradeSummary) writes an
+  // integer, and both validate it. A string here means something else wrote
+  // this record, and a rule that coerces would be reporting on a shape it does
+  // not understand.
+  const n = Number.isInteger(u.failStreak) ? u.failStreak : 0;
+  if (!(n > 0)) return null;
+  const sev = severityFor(n, num(cfg.warnStreak, Infinity), num(cfg.critStreak, Infinity));
+  if (!sev) return null;
+  const target = typeof u.streakTarget === "string" && u.streakTarget ? u.streakTarget : (typeof u.to === "string" ? u.to : "@latest");
+  // A DURATION, not only a tally: four attempts since Monday and four attempts
+  // in the last hour are different seats with different urgency, and the count
+  // alone cannot tell them apart.
+  const sinceMs = typeof u.stuckSince === "string" ? Date.parse(u.stuckSince) : NaN;
+  const days = Number.isFinite(sinceMs) ? Math.floor((now - sinceMs) / 86400000) : null;
+  const forHow = days === null ? "" : days >= 1 ? ` over ${days} ${days === 1 ? "day" : "days"}` : " today";
+  const why = typeof u.reason === "string" && u.reason ? ` Last failure: ${u.reason.split(":")[0]}.` : "";
+  return {
+    id: "upgrade_stuck",
+    severity: sev,
+    kind: "rollout",
+    detail: `${n} consecutive upgrade attempts to ${target} have failed${forHow} — this seat is running ${typeof u.from === "string" && u.from ? u.from : "older code"} and will not arrive on its own.${why} It needs a person at the machine.`,
+  };
+}
+
 function subAgentsRule(status, cfg = {}) {
   const v = num(status.subAgentsRunning);
   const cap = num(cfg.infoCap, Infinity);