@cohortapp/agent-sdk 2.18.8 → 2.18.11

package/lib/session/first-run.mjs

@@ -2,16 +2,36 @@
  * lib/session/first-run.mjs — the dialogs an unattended session cannot answer.
  *
  * The supervisor launches an INTERACTIVE `claude` inside a multiplexer with
- * nobody attached. On a fresh seat that first launch shows two one-time
- * dialogs and waits for a keypress that never comes:
+ * nobody attached. That launch can stop on a one-time screen and wait for a
+ * keypress that never comes. Each such screen is recorded in `~/.claude.json`
+ * once it is answered, and the supervisor SEEDS those records before launch:
  *
  *   - the bypass-permissions acknowledgement (`--dangerously-skip-permissions`),
- *     recorded in `~/.claude.json` as `bypassPermissionsModeAccepted: true`;
+ *     recorded as `bypassPermissionsModeAccepted: true`;
  *   - the per-directory trust dialog, recorded as
- *     `projects[<cwd>].hasTrustDialogAccepted: true`.
+ *     `projects[<cwd>].hasTrustDialogAccepted: true`;
+ *   - onboarding, recorded as `hasCompletedOnboarding: true` AND
+ *     `lastOnboardingVersion: "<cli version>"`;
+ *   - the release notes screen, recorded as `lastReleaseNotesSeen: "<cli version>"`.
  *
- * Neither is a secret and both are exactly what the operator answered when
- * they set the seat up by hand, so the supervisor SEEDS them before launch:
+ * ── WHY THE LAST TWO ARE NOT OPTIONAL: A SEEDED SEAT CAN GO BACK TO BLOCKED ──
+ * The first two are answered ONCE per seat and stay answered. The last two are
+ * VERSION-GATED: they are compared against the CLI version that is running, so
+ * an UPGRADE re-opens a screen a seeded seat had already passed. That is the
+ * difference between "the seat was set up wrong" and "the seat was fine until
+ * Tuesday", and it is the second that produced three seats which had been
+ * answering for weeks and then stopped inside an upgrade window.
+ *
+ * A seeder that hardcodes the screens it knows about is the autoupdater's
+ * lesson a second time: the check passed because it only ever asked the
+ * questions it already knew the answers to. {@link seedClaudeConfig} is
+ * therefore version-aware, and the supervisor's watchdog
+ * (`lib/session/pane#watchdogAction`) exists precisely because it CANNOT be
+ * complete — a screen this build has never heard of must still end in a running
+ * session, not in a three-day silence.
+ *
+ * Nothing here is a secret and every value is exactly what the operator
+ * answered when they set the seat up by hand:
  * {@link seedClaudeConfig} is the pure merge (adds only the missing flags,
  * preserves every other key, refuses to touch a malformed file), and
  * {@link ensureClaudeConfig} is its I/O shell (writes only when something
@@ -25,6 +45,22 @@
  * it and writes `state/session/attention.json` ({@link attentionRecord}) so
  * `doctor` can report it and an operator knows which command attaches.
  *
+ * ── AND THE ATTENTION RECORD IS NOT A FIX ────────────────────────────────────
+ * Writing that file used to be the whole of the response: the watchdog fired
+ * once, at the grace mark, wrote the record, logged a line telling a human to
+ * `tmux attach` — and then the supervisor sat in `waitForProbe` for as long as
+ * the blocked mux session existed. Days, on three seats. The record said
+ * "attach and answer it" to nobody, because the seats are unattended by
+ * design and there is no ssh into them.
+ *
+ * `lib/session/pane#watchdogAction` is the answer the seat can carry out
+ * itself: it reads the pane, answers a modal whose safe key is known, and
+ * otherwise RESTARTS — a relaunch re-runs {@link ensureClaudeConfig} against
+ * the version now installed and so has a real chance of clearing a
+ * version-gated screen. The restarts are BOUNDED, and {@link carriedRestarts}
+ * is what makes the bound bind: each restart ENDS this process, so a counter
+ * held in memory would be zero again on every relaunch.
+ *
  * Pure except the one I/O shell; no ambient clock, fs or home.
  *
  * @module lib/session/first-run
@@ -38,12 +74,58 @@ import { join } from "node:path";
 export const CLAUDE_CONFIG_FILE = ".claude.json";
 /** A session alive this long without a beat is flagged for attention. */
 export const HEARTBEAT_GRACE_MS = 120_000;
+/**
+ * The `~/.claude.json` keys the CLI compares against the version it is running.
+ * A seat that has answered these once is asked again after an upgrade, which
+ * is why they are advanced on every launch rather than seeded once.
+ */
+export const VERSION_GATED_KEYS = Object.freeze(["lastOnboardingVersion", "lastReleaseNotesSeen"]);
+
+/**
+ * Pure: compare two dotted version strings. Anything unparseable is `null` —
+ * "cannot tell", which every caller treats as "do not touch the value".
+ *
+ * @param {unknown} a
+ * @param {unknown} b
+ * @returns {number|null} <0 if a<b, 0 if equal, >0 if a>b, null if either is unreadable
+ */
+export function compareVersions(a, b) {
+  const parse = (v) => {
+    if (typeof v !== "string") return null;
+    const m = v.trim().match(/^(\d+)(?:\.(\d+))?(?:\.(\d+))?/);
+    if (!m) return null;
+    return [Number(m[1]), Number(m[2] || 0), Number(m[3] || 0)];
+  };
+  const x = parse(a);
+  const y = parse(b);
+  if (!x || !y) return null;
+  for (let i = 0; i < 3; i += 1) if (x[i] !== y[i]) return x[i] - y[i];
+  return 0;
+}
 
 /**
- * Pure: merge the two acknowledgements into a `~/.claude.json` body.
+ * Pure: merge every one-time acknowledgement an unattended launch cannot give
+ * into a `~/.claude.json` body.
+ *
+ * TWO CLASSES OF KEY, and the difference is the whole point:
+ *
+ *  - ONCE-EVER (`bypassPermissionsModeAccepted`, `hasCompletedOnboarding`,
+ *    `projects[root].hasTrustDialogAccepted`) — written when absent, never
+ *    rewritten. A seat passes these on its first launch and keeps them.
+ *  - VERSION-GATED (`lastOnboardingVersion`, `lastReleaseNotesSeen`) — the CLI
+ *    compares them against the version it is running, so they go STALE on
+ *    upgrade and the screen comes back. These are advanced to `cliVersion`,
+ *    and only forwards: a recorded version AHEAD of the installed one is left
+ *    alone, because winding it back would re-open the very screen this exists
+ *    to close.
+ *
+ * `cliVersion` is a PARAMETER. Without it the version-gated keys are not
+ * touched at all — a caller that cannot read the installed version must not
+ * guess one, and writing an invented version is strictly worse than leaving
+ * the screen to the watchdog.
  *
  * @param {string|null|undefined} text current file body (absent → empty)
- * @param {{agentRoot:string, bypass?:boolean}} a `bypass` — seed the bypass-permissions flag (only needed when launching with --dangerously-skip-permissions)
+ * @param {{agentRoot:string, bypass?:boolean, cliVersion?:string|null}} a `bypass` — seed the bypass-permissions flag (only needed when launching with --dangerously-skip-permissions); `cliVersion` — the installed Claude Code version, for the version-gated screens
  * @returns {{ok:true, changed:boolean, applied:string[], config:object}|{ok:false, changed:false, applied:[], error:string}}
  */
 export function seedClaudeConfig(text, a) {
@@ -58,6 +140,26 @@ export function seedClaudeConfig(text, a) {
     next.bypassPermissionsModeAccepted = true;
     applied.push("bypassPermissionsModeAccepted");
   }
+  if (next.hasCompletedOnboarding !== true) {
+    next.hasCompletedOnboarding = true;
+    applied.push("hasCompletedOnboarding");
+  }
+  // The version-gated pair. `compareVersions` returning null means one side is
+  // unreadable — treated as "leave it", never as "overwrite".
+  const cliVersion = typeof a.cliVersion === "string" && a.cliVersion.trim() ? a.cliVersion.trim() : null;
+  if (cliVersion) {
+    for (const key of VERSION_GATED_KEYS) {
+      const cur = next[key];
+      const cmp = compareVersions(cur, cliVersion);
+      // Absent (cmp === null with no current value) → seed it. Behind → advance
+      // it. Equal, ahead, or unreadable-but-present → leave it alone.
+      const absent = cur === undefined || cur === null || cur === "";
+      if (absent || (cmp !== null && cmp < 0)) {
+        next[key] = cliVersion;
+        applied.push(key);
+      }
+    }
+  }
   const projects = next.projects && typeof next.projects === "object" && !Array.isArray(next.projects) ? next.projects : {};
   const proj = projects[a.agentRoot] && typeof projects[a.agentRoot] === "object" && !Array.isArray(projects[a.agentRoot]) ? projects[a.agentRoot] : {};
   if (proj.hasTrustDialogAccepted !== true) {
@@ -72,7 +174,7 @@ export function seedClaudeConfig(text, a) {
  * Result frame; never throws (a seat must still launch when this fails —
  * the dialog then blocks and {@link heartbeatSilence} reports it).
  *
- * @param {{homeDir:string, agentRoot:string, bypass?:boolean, readFileSync:Function, writeFileAtomic:Function}} a
+ * @param {{homeDir:string, agentRoot:string, bypass?:boolean, cliVersion?:string|null, readFileSync:Function, writeFileAtomic:Function}} a
  * @returns {{ok:boolean, changed:boolean, applied:string[], path:string|null, error?:string}}
  */
 export function ensureClaudeConfig(a) {
@@ -83,7 +185,7 @@ export function ensureClaudeConfig(a) {
     if (!err || err.code !== "ENOENT") return { ok: false, changed: false, applied: [], path, error: err && err.message ? err.message : String(err) };
     text = null; // first launch on this account
   }
-  const seeded = seedClaudeConfig(text, { agentRoot: a.agentRoot, bypass: a.bypass });
+  const seeded = seedClaudeConfig(text, { agentRoot: a.agentRoot, bypass: a.bypass, cliVersion: a.cliVersion });
   if (!seeded.ok) return { ok: false, changed: false, applied: [], path, error: seeded.error };
   if (!seeded.changed) return { ok: true, changed: false, applied: [], path };
   try {
@@ -95,17 +197,45 @@ export function ensureClaudeConfig(a) {
 }
 
 /**
- * Pure: has the session run past the grace period without beating?
+ * Pure: is the front door silent right NOW?
+ *
+ * ── THE BUG THIS SHAPE EXISTS TO NOT HAVE ─────────────────────────────────
+ * This used to ask `ts >= startedAt` — "has it beaten since this launch?" — and
+ * answer `{silent:false, reason:"beating"}` for any beat at all, however old. A
+ * session that beat once thirty seconds after launch and then died was
+ * therefore "beating" for the rest of its life. Running the real module on
+ * Hannah's measured seat made the size of that plain:
+ * `{startedAt:T0, heartbeat:{ts:T0+30s}, now:T0+65h}` returned
+ * `{"action":"wait","reason":"beating"}` — 65 hours of silence read as health,
+ * no restart ever issued, no attention record ever written. That is the
+ * `heartbeat-stale` shape the fleet surface was reporting the whole time, and
+ * it is why making the watchdog RUN more often changed nothing: the predicate
+ * it ran never flipped.
+ *
+ * So the beat is compared to `now`, not to `startedAt`. `startedAt` still has
+ * one job — telling a beat from a PREVIOUS run (`stale-heartbeat`) apart from
+ * one this run produced and then stopped (`beat-stopped`) — because those two
+ * are fixed differently: the first means the session never got going, the
+ * second means it got going and wedged.
+ *
  * @param {{startedAt:number, heartbeat:{ts:string}|null, now:number, graceMs?:number}} a
- * @returns {{silent:boolean, reason:"within-grace"|"beating"|"no-heartbeat"|"stale-heartbeat", runMs:number}}
+ * @returns {{silent:boolean, reason:"within-grace"|"beating"|"no-heartbeat"|"stale-heartbeat"|"beat-stopped", runMs:number, sinceBeatMs?:number}}
  */
 export function heartbeatSilence(a) {
   const graceMs = a.graceMs ?? HEARTBEAT_GRACE_MS;
   const runMs = Number(a.now) - Number(a.startedAt);
   if (runMs < graceMs) return { silent: false, reason: "within-grace", runMs };
   const ts = a.heartbeat && typeof a.heartbeat.ts === "string" ? Date.parse(a.heartbeat.ts) : NaN;
-  if (Number.isFinite(ts) && ts >= Number(a.startedAt)) return { silent: false, reason: "beating", runMs };
-  return { silent: true, reason: Number.isFinite(ts) ? "stale-heartbeat" : "no-heartbeat", runMs };
+  if (!Number.isFinite(ts)) return { silent: true, reason: "no-heartbeat", runMs };
+  // A beat stamped BEFORE this launch belongs to a previous run: this session
+  // has never reported in, which is the same fault as no file at all.
+  if (ts < Number(a.startedAt)) return { silent: true, reason: "stale-heartbeat", runMs };
+  const sinceBeatMs = Number(a.now) - ts;
+  // A beat from the FUTURE is clock skew, not liveness. Treated as fresh (the
+  // bias is towards not restarting a session on the strength of a bad clock),
+  // which `Math.max` below makes explicit rather than accidental.
+  if (Math.max(0, sinceBeatMs) > graceMs) return { silent: true, reason: "beat-stopped", runMs, sinceBeatMs };
+  return { silent: false, reason: "beating", runMs, sinceBeatMs };
 }
 
 /**
@@ -114,13 +244,80 @@ export function heartbeatSilence(a) {
  */
 export function attentionRecord(a) {
   const attach = a.mux && a.mux.kind === "tmux" ? `tmux attach -t =${a.muxName}` : `screen -r ${a.muxName}`;
+  const restarts = Number.isFinite(Number(a.restarts)) && Number(a.restarts) > 0 ? Math.floor(Number(a.restarts)) : 0;
+  const action = a.action === "restart" || a.action === "give-up" || a.action === "clear" ? a.action : "wait";
+  const secs = Math.round(a.runMs / 1000);
+  // THE HINT IS READ BY SOMEBODY WHO CANNOT REACH THE MACHINE. It rides the
+  // presence beat to hq (`machine.sessionNote.detail`) and is the only sentence
+  // an operator in a browser gets. "Attach and answer it" was the entire text
+  // for three days on a seat nobody could ssh into; what it owes the reader now
+  // is what the seat has ALREADY tried, so the reader knows whether a restart
+  // is still coming or whether this is where it stops.
+  const tried = restarts > 0 ? ` Restarted ${restarts}× already and it still has not reported in.` : "";
+  const next = action === "restart"
+    ? " Restarting it now."
+    : action === "give-up"
+      ? " Out of automatic restarts — this needs a person at the machine."
+      : action === "clear"
+        ? " Answering the screen now."
+        : "";
   return {
     reason: a.reason,
     since: new Date(Number(a.since)).toISOString(),
     runMs: a.runMs,
+    restarts,
+    action,
     attach,
-    hint: `The session has been up ${Math.round(a.runMs / 1000)} s without a heartbeat — it is probably waiting on a first-run dialog or a permission prompt. Attach with \`${attach}\` and answer it.`,
+    // WHAT THE SCREEN ACTUALLY SAYS IS NOT GUESSED HERE. The old text asserted
+    // "probably waiting on a first-run dialog or a permission prompt" and was
+    // wrong on every seat it was read off: the real modal was the
+    // subscription-limit chooser, and the trust dialog it blamed is
+    // pre-answered at launch by `ensureClaudeConfig`. The diagnosis belongs to
+    // the watchdog, which reads the pane and writes `modal`/`modalWhy` beside
+    // this record; the hint states only what this function can prove.
+    hint: `The session has been up ${secs} s without a heartbeat (${a.reason}).${tried}${next} Attach with \`${attach}\` and answer it.`,
   };
 }
 
-export default { CLAUDE_CONFIG_FILE, HEARTBEAT_GRACE_MS, seedClaudeConfig, ensureClaudeConfig, heartbeatSilence, attentionRecord };
+/**
+ * Pure: how many silent restarts this launch inherits.
+ *
+ * The counter has to survive the restart it authorises — each restart ENDS the
+ * supervisor process and launchd starts a new one, so a count held in memory
+ * would reset to zero every time and the bound in `pane#watchdogAction` would
+ * never bind. It rides `state/session/attention.json`, the record the previous
+ * run already wrote for exactly this session.
+ *
+ * It resets on EVIDENCE OF WORK and on nothing else: a heartbeat stamped after
+ * the attention record means the session that record describes went on to run,
+ * so the next blockage is a new fault and gets its own budget. An attention
+ * record with no usable `since` cannot be ordered against a beat, and a real
+ * beat is the harder evidence, so that also resets — the bias is towards
+ * trying again, never towards giving up early on a seat that recovered once.
+ *
+ * @param {{attention:object|null, heartbeat:{ts?:string}|null}} a
+ * @returns {number}
+ */
+export function carriedRestarts(a = {}) {
+  const att = a.attention && typeof a.attention === "object" && !Array.isArray(a.attention) ? a.attention : null;
+  if (!att) return 0;
+  const n = Number(att.restarts);
+  const carried = Number.isFinite(n) && n > 0 ? Math.floor(n) : 0;
+  if (carried === 0) return 0;
+  const attMs = Date.parse(typeof att.since === "string" ? att.since : "");
+  const hbMs = Date.parse(a.heartbeat && typeof a.heartbeat.ts === "string" ? a.heartbeat.ts : "");
+  if (Number.isFinite(hbMs) && (!Number.isFinite(attMs) || hbMs > attMs)) return 0;
+  return carried;
+}
+
+export default {
+  CLAUDE_CONFIG_FILE,
+  HEARTBEAT_GRACE_MS,
+  VERSION_GATED_KEYS,
+  compareVersions,
+  seedClaudeConfig,
+  ensureClaudeConfig,
+  heartbeatSilence,
+  carriedRestarts,
+  attentionRecord,
+};

package/lib/session/revive.mjs

@@ -0,0 +1,135 @@
+/**
+ * lib/session/revive.mjs — the decision to restart a front door that has
+ * stopped answering, made from OUTSIDE the process that is stuck.
+ *
+ * ── WHY THIS IS NOT IN THE SUPERVISOR ──────────────────────────────────────
+ *
+ * The supervisor already watches its own session: since 2.18.7 it captures the
+ * pane, names the modal and answers one whose answer is free. That covers a
+ * session which wedges while the supervisor is healthy, and it is the right
+ * place for it.
+ *
+ * It cannot cover the case that actually held this fleet. James Kirkland's
+ * front door had been shut for 3.5 days and Isla Roselli's for 15, and their
+ * supervisors never relaunched once in that time: each was parked in its own
+ * launch probe, waiting on a mux session that would never report ready.
+ * Installing a newer SDK does not restart a process that is already stuck, so
+ * every fix shipped into the supervisor arrived somewhere it could not run.
+ *
+ * The hourly autoupdate job can restart it, and does since 2.18.8. An hour is
+ * a long time to be unable to answer anybody, and it only happens on the runs
+ * that get that far.
+ *
+ * The daemon is the honest home for this. It is a separate process, it is
+ * already alive on every seat that beats, it already reads the front-door
+ * state every poll to decide who owns the inbox, and it runs every thirty
+ * seconds. A seat whose door shuts is then measured in a minute rather than a
+ * day, by something the shut door cannot take down with it.
+ *
+ * Pure: the decision only. The daemon owns the kickstart.
+ */
+
+/** Default silence before the daemon restarts the front door. */
+export const DEFAULT_REVIVE_AFTER_MS = 10 * 60 * 1000;
+
+/** Default quiet period between attempts. */
+export const DEFAULT_REVIVE_BACKOFF_MS = 15 * 60 * 1000;
+
+/** How many restarts before the daemon stops and leaves it to a person. */
+export const DEFAULT_REVIVE_MAX = 3;
+
+/**
+ * Should the daemon restart the front door right now?
+ *
+ * The caller supplies the front-door state it already reads for dispatch, the
+ * age of the session heartbeat, and what this daemon has already tried. Every
+ * bound is explicit because the failure mode of getting this wrong is a seat
+ * that restarts its own session every thirty seconds forever — which is worse
+ * than the shut door, and is the reason the ladder ends in "stop and say so"
+ * rather than in another attempt.
+ *
+ * @param {{frontDoor?:string, sessionLive?:boolean, silentMs?:number|null,
+ *          attempts?:number, lastAttemptAt?:number|null, now:number,
+ *          reviveAfterMs?:number, backoffMs?:number, maxAttempts?:number}} a
+ * @returns {{revive:boolean, reason:string}}
+ */
+export function shouldReviveFrontDoor(a) {
+  const x = a && typeof a === "object" ? a : {};
+  const now = Number(x.now);
+  const after = Number.isFinite(x.reviveAfterMs) && x.reviveAfterMs > 0 ? x.reviveAfterMs : DEFAULT_REVIVE_AFTER_MS;
+  const backoff = Number.isFinite(x.backoffMs) && x.backoffMs > 0 ? x.backoffMs : DEFAULT_REVIVE_BACKOFF_MS;
+  const max = Number.isFinite(x.maxAttempts) && x.maxAttempts >= 0 ? x.maxAttempts : DEFAULT_REVIVE_MAX;
+
+  // A seat whose lane is the daemon has no front door to revive: the daemon
+  // itself is answering, and restarting a session job it does not depend on
+  // would be a gratuitous interruption.
+  if (x.frontDoor !== "session") return { revive: false, reason: "front-door-daemon" };
+
+  // Working. This is the common case and it must be cheap and obviously safe:
+  // the heartbeat rides every tool use, so a session doing anything at all is
+  // fresh.
+  if (x.sessionLive === true) return { revive: false, reason: "answering" };
+
+  // Not live, but we cannot say for how long — a missing or unreadable
+  // heartbeat is not evidence of a wedge, and a daemon that restarts a session
+  // it knows nothing about is a daemon that fights its own launch.
+  // `Number(null)` is 0, and 0 is finite — so a MISSING heartbeat, which is
+  // exactly what `sessionLiveFromHeartbeat` reports as `ageMs: null`, would
+  // otherwise read as "silent for no time at all" and be reported as
+  // within-grace. Harmless at today's ten-minute grace and a lie at any grace
+  // of zero; either way the reason would name the wrong state.
+  const silentMs = x.silentMs === null || x.silentMs === undefined ? NaN : Number(x.silentMs);
+  if (!Number.isFinite(silentMs)) return { revive: false, reason: "silence-unknown" };
+  if (silentMs < after) return { revive: false, reason: "within-grace" };
+
+  const attempts = Number(x.attempts) || 0;
+  if (attempts >= max) return { revive: false, reason: "budget-spent" };
+
+  // One attempt, then wait. A front door takes time to come up, and a restart
+  // loop that re-fires before the new session can beat never lets it.
+  const last = Number(x.lastAttemptAt);
+  if (Number.isFinite(last) && now - last < backoff) return { revive: false, reason: "backing-off" };
+
+  return { revive: true, reason: `front door silent ${Math.round(silentMs / 1000)}s (attempt ${attempts + 1}/${max})` };
+}
+
+/**
+ * The command that restarts the front-door job. Pure — the caller runs it.
+ *
+ * `kickstart -k` rather than a polite request: the session this exists for is
+ * one that cannot act on a request, and `maestro session restart` without
+ * `--force` asks the session to stand itself down at an idle moment it will
+ * never reach.
+ *
+ * @param {string} label launchd label, e.g. "ai.maestro.james-session"
+ * @param {number} uid
+ * @returns {{file:string, args:string[]}}
+ */
+export function reviveCommand(label, uid) {
+  return { file: "launchctl", args: ["kickstart", "-k", `gui/${uid}/${label}`] };
+}
+
+/**
+ * This seat's front-door launchd label, read off disk rather than derived.
+ *
+ * The label is `ai.maestro.<first>-session`, but `<first>` is whatever the
+ * generator wrote — `~/Maya-ai` yields `Maya`, and case is preserved. Guessing
+ * it from a directory name is how a healthy seat gets accused of having no job
+ * (collect.mjs#sessionJobLabel carries the same warning). A readdir is a hard
+ * fact, and there is exactly one such plist on a seat.
+ *
+ * @param {(dir:string)=>string[]} readdir
+ * @param {string} homeDir
+ * @returns {string|null}
+ */
+export function sessionJobLabelOnDisk(readdir, homeDir) {
+  if (!homeDir || typeof readdir !== "function") return null;
+  let names = [];
+  try { names = readdir(`${homeDir}/Library/LaunchAgents`) || []; } catch { return null; }
+  const hits = names
+    .filter((n) => /^ai\.maestro\..+-session\.plist$/.test(String(n)))
+    .map((n) => String(n).replace(/\.plist$/, ""));
+  // Two would mean two seats share a home directory, which is not a thing this
+  // may guess its way through.
+  return hits.length === 1 ? hits[0] : null;
+}

package/lib/telemetry/alerts.mjs

@@ -17,6 +17,8 @@
  *   - machine.memUsedPct high            → WARNING/CRITICAL (memory pressure).
  *   - machine.diskUsedPct high           → WARNING/CRITICAL (disk pressure).
  *   - subAgentsRunning over the cap      → INFO (more children than expected).
+ *   - machine.sessionNote present        → WARNING/CRITICAL (frontdoor) — the
+ *                                          seat's front door is not answering.
  *
  * Output: a deterministic array `[{ id, severity, kind, detail }, ...]` (the
  * snapshot's alert shape) sorted critical→warning→info then by id, so identical
@@ -56,6 +58,46 @@ export const DEFAULT_THRESHOLDS = Object.freeze({
   subAgents: Object.freeze({
     infoCap: 10, // more live claude children than this is worth surfacing
   }),
+  /**
+   * How long the seat's FRONT DOOR may be unable to answer before the fleet is
+   * told. The daemon lane keeps running underneath, so nothing here is about
+   * the machine being down — it is about the seat being unable to hold a
+   * conversation.
+   *
+   * ── WHY THIS RULE EXISTS AT ALL ──────────────────────────────────────────
+   * Every other rule in this file is about the MACHINE: how hot it is, how
+   * full its disk is, how much memory it has left, whether it can still
+   * authenticate. In the week three seats had their front door blocked — for
+   * 3, 15 and 2.7 days — every alert the fleet opened was one of those, and
+   * not one was about a seat nobody could reach. THE SHAPE IS THE POINT AND NO
+   * TALLY IS GIVEN: a comment that states a count goes stale the next time the
+   * fleet opens an alert, and this one already had — it read "twelve alerts,
+   * five disk and seven thermal" while the loudest seat in the fleet was
+   * carrying eleven open critical memory rows. A conversational seat that
+   * cannot converse is at least as broken as a warm one.
+   *
+   * ── WHY THE WINDOWS ARE THESE ────────────────────────────────────────────
+   * `warnMs` 15 min: the supervisor's own watchdog is bounded (2 modal clears,
+   * 3 restarts, one pass per 2-minute grace window), so anything past a
+   * quarter of an hour has already outlived every fix the seat can apply to
+   * itself. Below that the seat is still trying and an alert would be noise.
+   *
+   * `critMs` 2 h: past this the silence has spanned a working block. A message
+   * sent at the start of it has gone unanswered long enough that the sender
+   * has drawn a conclusion about the colleague, which is the actual damage.
+   */
+  frontDoor: Object.freeze({
+    warnMs: 15 * 60 * 1000,
+    critMs: 2 * 60 * 60 * 1000,
+  }),
+  replyDebt: Object.freeze({
+    // ONE withheld reply is already worth saying. This is not a resource gauge
+    // where a low reading is normal noise — it counts people who asked this
+    // seat something and got nothing. The threshold exists only so the severity
+    // can climb, not so the first few can be ignored.
+    warnCount: 1,
+    critCount: 5,
+  }),
 });
 
 const SEVERITY_RANK = { critical: 0, warning: 1, info: 2 };
@@ -109,11 +151,16 @@ function round1(n) { return Math.round(Number(n) * 10) / 10; }
 /**
  * Derive the health alerts for a status snapshot.
  *
+ * `now` is a PARAMETER (defaulted, doctrine §2) because one rule — the front
+ * door — is about a DURATION, and a rule that reads the clock itself cannot be
+ * tested across the boundary it exists to draw.
+ *
  * @param {object} status      the SHARED STATUS SNAPSHOT (collect.collectStatus)
  * @param {object} [thresholds] operator override merged over DEFAULT_THRESHOLDS
+ * @param {number} [now]       epoch ms
  * @returns {Array<{id:string, severity:"info"|"warning"|"critical", kind:string, detail:string}>}
  */
-export function deriveAlerts(status, thresholds) {
+export function deriveAlerts(status, thresholds, now = Date.now()) {
   const alerts = [];
   try {
     const s = status && typeof status === "object" ? status : {};
@@ -125,6 +172,9 @@ export function deriveAlerts(status, thresholds) {
     push(alerts, memRule(machine, t.mem));
     push(alerts, diskRule(machine, t.disk));
     push(alerts, subAgentsRule(s, t.subAgents));
+    push(alerts, frontDoorRule(machine, t.frontDoor, now));
+    push(alerts, scopeFaultRule(machine));
+    push(alerts, repliesWithheldRule(machine, t.replyDebt));
 
     alerts.sort((a, b) => {
       const r = (SEVERITY_RANK[a.severity] ?? 9) - (SEVERITY_RANK[b.severity] ?? 9);
@@ -187,6 +237,51 @@ function diskRule(machine, cfg = {}) {
   };
 }
 
+/**
+ * The seat is missing a scope, which nothing on this machine can grant.
+ *
+ * Critical, and deliberately louder than a hot CPU: twelve alerts opened on
+ * this fleet in a week were all disk and thermal, and none for a seat that
+ * could not answer anybody. A thermal warning resolves itself; a scope fault
+ * waits for a person forever.
+ */
+function scopeFaultRule(machine) {
+  const d = machine && machine.replyDebt;
+  const n = num(d && d.scopeRefused);
+  if (!(n > 0)) return null;
+  return {
+    id: "seat_missing_scope",
+    severity: "critical",
+    kind: "scope",
+    detail: `${n} ${n === 1 ? "reply was" : "replies were"} refused today for a scope this seat does not hold — a person asked and got nothing. Grant the scope; the seat cannot.`,
+  };
+}
+
+/**
+ * This seat withheld N replies today — the number the whole reply-debt spine
+ * exists to make sayable. Scope faults have their own critical rule above, so
+ * this one carries the rest (an unparseable CLI envelope, a session-id
+ * collision the single retry did not recover).
+ */
+function repliesWithheldRule(machine, cfg = {}) {
+  const d = machine && machine.replyDebt;
+  if (!d) return null;
+  const v = num(d.withheld) - num(d.scopeRefused);
+  if (!(v > 0)) return null;
+  const sev = severityFor(v, num(cfg.warnCount, Infinity), num(cfg.critCount, Infinity));
+  if (!sev) return null;
+  const parts = [];
+  if (num(d.unparseable) > 0) parts.push(`${num(d.unparseable)} unreadable CLI envelope`);
+  const lost = Math.max(0, num(d.sessionCollisions) - num(d.recovered));
+  if (lost > 0) parts.push(`${lost} session-id collision`);
+  return {
+    id: "replies_withheld",
+    severity: sev,
+    kind: "reply",
+    detail: `${v} ${v === 1 ? "reply" : "replies"} withheld today${parts.length ? ` (${parts.join(", ")})` : ""} — someone asked and got silence.`,
+  };
+}
+
 function subAgentsRule(status, cfg = {}) {
   const v = num(status.subAgentsRunning);
   const cap = num(cfg.infoCap, Infinity);
@@ -199,6 +294,68 @@ function subAgentsRule(status, cfg = {}) {
   };
 }
 
+/**
+ * The front door is not answering, and has not been for long enough that the
+ * seat's own recovery has already been tried and failed.
+ *
+ * READS `machine.sessionNote` — the exception field `collect.sessionNote()`
+ * emits, which is present ONLY when the front door is not live and carries the
+ * REASON (`awaiting-input`, `heartbeat-stale`, `never-beaten`, `job-absent`,
+ * …). The note is the seat's own diagnosis and it is passed through verbatim in
+ * the detail, because the reason is exactly what tells an operator whether this
+ * is a dialog nobody answered, a supervisor that died, or a seat deliberately
+ * configured without a session job.
+ *
+ * `job-absent` is the one reason that raises NOTHING. A seat whose front door
+ * is the daemon by configuration is not broken, and alerting on it would train
+ * the fleet to ignore this rule — which is how the disk and thermal alerts got
+ * their current signal-to-noise and this rule must not spend it.
+ *
+ * A note with no usable `since` cannot be aged, so it cannot clear the warn
+ * window and raises nothing. That is deliberate: an alert is a claim about
+ * duration, and an undatable note does not support one. The note still rides
+ * the beat and the fleet surface still shows it.
+ *
+ * ── THE DETAIL NAMES AN INSTANT, NEVER A DURATION ────────────────────────────
+ * hq opens one AgentAlert row per DISTINCT `severity\0kind\0detail`
+ * (`alertClientKey`, src/server/methods/presence/beat.ts). A detail carrying a
+ * ticking age — "has not answered for 15m… 16m… 17m…" — is therefore a NEW row
+ * on every beat: running this rule against a 30-second beat cadence for eight
+ * hours on one seat produced 83 distinct rows, and a `critical` row does not
+ * self-clear. That is exactly how a seat in this fleet accumulated eleven open
+ * critical `memory` rows whose details differed only in the percentage. So the
+ * detail anchors on `note.since`, which is FIXED for as long as one blockage
+ * lasts, and the reader does the subtraction. At most two rows per episode —
+ * the warn one, then the critical one — and the warn row clears when the door
+ * opens.
+ *
+ * For the same reason `note.detail` is NOT appended: it carries the seat's own
+ * ticking text ("no beat for 431 s"), and pasting it here would put the
+ * counter straight back into the key. It rides the beat as
+ * `machine.sessionNote.detail` and the fleet surface shows it there.
+ */
+function frontDoorRule(machine, cfg = {}, now = Date.now()) {
+  const note = machine && typeof machine.sessionNote === "object" && machine.sessionNote !== null && !Array.isArray(machine.sessionNote)
+    ? machine.sessionNote
+    : null;
+  if (!note) return null;
+  const reason = typeof note.reason === "string" ? note.reason.trim() : "";
+  if (!reason || reason === "job-absent") return null;
+  const since = Date.parse(typeof note.since === "string" ? note.since : "");
+  if (!Number.isFinite(since)) return null;
+  const ageMs = num(now) - since;
+  if (!(ageMs > 0)) return null; // a future stamp is skew, not a duration
+  const sev = severityFor(ageMs, num(cfg.warnMs, Infinity), num(cfg.critMs, Infinity));
+  if (!sev) return null;
+  return {
+    id: "front_door_blocked",
+    severity: sev,
+    kind: "frontdoor",
+    // An INSTANT, never a duration — see `alertClientKey` above.
+    detail: `the seat's front door has not answered since ${new Date(since).toISOString()} (${reason}) — messages to this colleague are going unanswered.`,
+  };
+}
+
 /**
  * Does this alert list contain a critical alert? Convenience for the mesh's
  * local-escalation decision. Never throws.
@@ -211,6 +368,6 @@ export function hasCritical(alerts) {
   }
 }
 
-export const _internals = { severityFor, reloginRule, tempRule, memRule, diskRule, subAgentsRule };
+export const _internals = { severityFor, reloginRule, tempRule, memRule, diskRule, subAgentsRule, frontDoorRule };
 
 export default { deriveAlerts, hasCritical, mergeThresholds, DEFAULT_THRESHOLDS };