@cohortapp/agent-sdk 2.18.10 → 2.18.12

package/lib/session/revive.mjs

@@ -0,0 +1,135 @@
+/**
+ * lib/session/revive.mjs — the decision to restart a front door that has
+ * stopped answering, made from OUTSIDE the process that is stuck.
+ *
+ * ── WHY THIS IS NOT IN THE SUPERVISOR ──────────────────────────────────────
+ *
+ * The supervisor already watches its own session: since 2.18.7 it captures the
+ * pane, names the modal and answers one whose answer is free. That covers a
+ * session which wedges while the supervisor is healthy, and it is the right
+ * place for it.
+ *
+ * It cannot cover the case that actually held this fleet. James Kirkland's
+ * front door had been shut for 3.5 days and Isla Roselli's for 15, and their
+ * supervisors never relaunched once in that time: each was parked in its own
+ * launch probe, waiting on a mux session that would never report ready.
+ * Installing a newer SDK does not restart a process that is already stuck, so
+ * every fix shipped into the supervisor arrived somewhere it could not run.
+ *
+ * The hourly autoupdate job can restart it, and does since 2.18.8. An hour is
+ * a long time to be unable to answer anybody, and it only happens on the runs
+ * that get that far.
+ *
+ * The daemon is the honest home for this. It is a separate process, it is
+ * already alive on every seat that beats, it already reads the front-door
+ * state every poll to decide who owns the inbox, and it runs every thirty
+ * seconds. A seat whose door shuts is then measured in a minute rather than a
+ * day, by something the shut door cannot take down with it.
+ *
+ * Pure: the decision only. The daemon owns the kickstart.
+ */
+
+/** Default silence before the daemon restarts the front door. */
+export const DEFAULT_REVIVE_AFTER_MS = 10 * 60 * 1000;
+
+/** Default quiet period between attempts. */
+export const DEFAULT_REVIVE_BACKOFF_MS = 15 * 60 * 1000;
+
+/** How many restarts before the daemon stops and leaves it to a person. */
+export const DEFAULT_REVIVE_MAX = 3;
+
+/**
+ * Should the daemon restart the front door right now?
+ *
+ * The caller supplies the front-door state it already reads for dispatch, the
+ * age of the session heartbeat, and what this daemon has already tried. Every
+ * bound is explicit because the failure mode of getting this wrong is a seat
+ * that restarts its own session every thirty seconds forever — which is worse
+ * than the shut door, and is the reason the ladder ends in "stop and say so"
+ * rather than in another attempt.
+ *
+ * @param {{frontDoor?:string, sessionLive?:boolean, silentMs?:number|null,
+ *          attempts?:number, lastAttemptAt?:number|null, now:number,
+ *          reviveAfterMs?:number, backoffMs?:number, maxAttempts?:number}} a
+ * @returns {{revive:boolean, reason:string}}
+ */
+export function shouldReviveFrontDoor(a) {
+  const x = a && typeof a === "object" ? a : {};
+  const now = Number(x.now);
+  const after = Number.isFinite(x.reviveAfterMs) && x.reviveAfterMs > 0 ? x.reviveAfterMs : DEFAULT_REVIVE_AFTER_MS;
+  const backoff = Number.isFinite(x.backoffMs) && x.backoffMs > 0 ? x.backoffMs : DEFAULT_REVIVE_BACKOFF_MS;
+  const max = Number.isFinite(x.maxAttempts) && x.maxAttempts >= 0 ? x.maxAttempts : DEFAULT_REVIVE_MAX;
+
+  // A seat whose lane is the daemon has no front door to revive: the daemon
+  // itself is answering, and restarting a session job it does not depend on
+  // would be a gratuitous interruption.
+  if (x.frontDoor !== "session") return { revive: false, reason: "front-door-daemon" };
+
+  // Working. This is the common case and it must be cheap and obviously safe:
+  // the heartbeat rides every tool use, so a session doing anything at all is
+  // fresh.
+  if (x.sessionLive === true) return { revive: false, reason: "answering" };
+
+  // Not live, but we cannot say for how long — a missing or unreadable
+  // heartbeat is not evidence of a wedge, and a daemon that restarts a session
+  // it knows nothing about is a daemon that fights its own launch.
+  // `Number(null)` is 0, and 0 is finite — so a MISSING heartbeat, which is
+  // exactly what `sessionLiveFromHeartbeat` reports as `ageMs: null`, would
+  // otherwise read as "silent for no time at all" and be reported as
+  // within-grace. Harmless at today's ten-minute grace and a lie at any grace
+  // of zero; either way the reason would name the wrong state.
+  const silentMs = x.silentMs === null || x.silentMs === undefined ? NaN : Number(x.silentMs);
+  if (!Number.isFinite(silentMs)) return { revive: false, reason: "silence-unknown" };
+  if (silentMs < after) return { revive: false, reason: "within-grace" };
+
+  const attempts = Number(x.attempts) || 0;
+  if (attempts >= max) return { revive: false, reason: "budget-spent" };
+
+  // One attempt, then wait. A front door takes time to come up, and a restart
+  // loop that re-fires before the new session can beat never lets it.
+  const last = Number(x.lastAttemptAt);
+  if (Number.isFinite(last) && now - last < backoff) return { revive: false, reason: "backing-off" };
+
+  return { revive: true, reason: `front door silent ${Math.round(silentMs / 1000)}s (attempt ${attempts + 1}/${max})` };
+}
+
+/**
+ * The command that restarts the front-door job. Pure — the caller runs it.
+ *
+ * `kickstart -k` rather than a polite request: the session this exists for is
+ * one that cannot act on a request, and `maestro session restart` without
+ * `--force` asks the session to stand itself down at an idle moment it will
+ * never reach.
+ *
+ * @param {string} label launchd label, e.g. "ai.maestro.james-session"
+ * @param {number} uid
+ * @returns {{file:string, args:string[]}}
+ */
+export function reviveCommand(label, uid) {
+  return { file: "launchctl", args: ["kickstart", "-k", `gui/${uid}/${label}`] };
+}
+
+/**
+ * This seat's front-door launchd label, read off disk rather than derived.
+ *
+ * The label is `ai.maestro.<first>-session`, but `<first>` is whatever the
+ * generator wrote — `~/Maya-ai` yields `Maya`, and case is preserved. Guessing
+ * it from a directory name is how a healthy seat gets accused of having no job
+ * (collect.mjs#sessionJobLabel carries the same warning). A readdir is a hard
+ * fact, and there is exactly one such plist on a seat.
+ *
+ * @param {(dir:string)=>string[]} readdir
+ * @param {string} homeDir
+ * @returns {string|null}
+ */
+export function sessionJobLabelOnDisk(readdir, homeDir) {
+  if (!homeDir || typeof readdir !== "function") return null;
+  let names = [];
+  try { names = readdir(`${homeDir}/Library/LaunchAgents`) || []; } catch { return null; }
+  const hits = names
+    .filter((n) => /^ai\.maestro\..+-session\.plist$/.test(String(n)))
+    .map((n) => String(n).replace(/\.plist$/, ""));
+  // Two would mean two seats share a home directory, which is not a thing this
+  // may guess its way through.
+  return hits.length === 1 ? hits[0] : null;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cohortapp/agent-sdk",
-  "version": "2.18.10",
+  "version": "2.18.12",
   "description": "Cohort Agent SDK — autonomous AI colleague runtime. Deploy senior AI colleagues on dedicated Mac minis, wired to the Cohort operating surface.",
   "type": "module",
   "bin": {

package/scripts/daemon/agent-daemon.mjs

@@ -17,6 +17,10 @@
 // =============================================================================
 
 import { resolve, join } from "path";
+import { homedir } from "os";
+import { execFile as _execFileCb } from "child_process";
+import { promisify } from "util";
+const execFileAsync = promisify(_execFileCb);
 import { readdirSync, readFileSync, renameSync, mkdirSync, appendFileSync, writeFileSync, unlinkSync } from "fs";
 import { createRequire } from "module";
 
@@ -155,7 +159,8 @@ import { processOne } from "../../lib/execution/pipeline.mjs";
 // of spawning `claude --print`; not live → today's dispatch, unchanged. The
 // gate is pure (lib/session/frontdoor.mjs); the assurance sweep reopens a
 // session claim that was neither replied nor done within 20 min.
-import { makeFrontDoorGate } from "../../lib/session/frontdoor.mjs";
+import { shouldReviveFrontDoor, reviveCommand, sessionJobLabelOnDisk } from "../../lib/session/revive.mjs";
+import { makeFrontDoorGate, sessionLiveFromHeartbeat, readFrontDoorState } from "../../lib/session/frontdoor.mjs";
 import { CADENCE_REGISTRY } from "./cadence-handlers.mjs";
 import { sweepSessionInboxForDaemon } from "../../lib/session/inbox-claims.mjs";
 import { defaultEffects, scheduleToQueue } from "../../lib/execution/effects.mjs";
@@ -269,6 +274,54 @@ async function poll() {
  * on the 60s full poll. Returns the count of new items processed. Never throws.
  */
 const _frontDoorGate = makeFrontDoorGate({ agentRoot: AGENT_REPO_DIR });
+
+// ── REVIVING A SHUT FRONT DOOR ───────────────────────────────────────────────
+//
+// The daemon already reads the front-door state every poll to decide who owns
+// the inbox. It is therefore the one process that watches a session it does not
+// live inside — and that is exactly what a wedged front door needs.
+//
+// The supervisor's own watchdog (2.18.7) cannot help when the supervisor is
+// itself parked in its launch probe waiting on a session that will never report
+// ready: James Kirkland sat that way for 3.5 days and Isla Roselli for 15,
+// through several SDK upgrades, because installing new code does not restart a
+// stuck process. The hourly autoupdate job restarts it (2.18.8) but only once
+// an hour and only on the runs that reach that far. This closes it to a minute.
+//
+// State is per-process and deliberately not persisted: a daemon restart is
+// itself a change of circumstances, and a fresh budget after one is the answer
+// we want rather than a stale count carried across it.
+/** How often the daemon asks whether the front door has gone quiet. */
+const REVIVE_CHECK_INTERVAL_MS = 60 * 1000;
+const _revive = { attempts: 0, lastAt: null };
+async function reviveFrontDoorIfShut(state, deps = {}) {
+  const now = deps.now ? deps.now() : Date.now();
+  const live = sessionLiveFromHeartbeat(state && state.heartbeat, { now });
+  const verdict = shouldReviveFrontDoor({
+    frontDoor: state && state.frontDoor,
+    sessionLive: state && state.sessionLive,
+    silentMs: live.ageMs,
+    attempts: _revive.attempts,
+    lastAttemptAt: _revive.lastAt,
+    now,
+    reviveAfterMs: Number(process.env.MAESTRO_SESSION_STALE_S || 0) * 1000 || undefined,
+  });
+  if (!verdict.revive) return verdict;
+  const label = sessionJobLabelOnDisk(readdirSync, homedir());
+  if (!label) return { revive: false, reason: "no-session-job" };
+  _revive.attempts += 1;
+  _revive.lastAt = now;
+  const cmd = reviveCommand(label, typeof process.getuid === "function" ? process.getuid() : 0);
+  console.error(`[daemon] front door shut — ${verdict.reason}; restarting ${label}`);
+  try {
+    await (deps.execFile || execFileAsync)(cmd.file, cmd.args);
+    console.error(`[daemon] ${label} restarted; a fresh session clears whatever the old one was sitting on`);
+  } catch (err) {
+    console.error(`[daemon] could not restart ${label}: ${err && err.message ? err.message : err} — this seat needs a person`);
+  }
+  return verdict;
+}
+
 async function pollService(svc) {
   try {
     // Front door: a live main session owns this service's inbox — leave it.
@@ -3096,6 +3149,22 @@ async function main() {
     pollCohortFast().catch((err) => console.error("[daemon] cohort fast-poll error:", err.message));
   }, COHORT_FAST_INTERVAL);
 
+  // ── THE SHUT-DOOR CHECK GETS ITS OWN TICK ─────────────────────────────────
+  //
+  // It lived inside pollService for one release, which was the wrong place for
+  // exactly the reason it exists: `pollCohortFast` guards itself with
+  // `_cohortFastRunning`, so a poll that hangs starves every later call — and a
+  // seat whose front door is wedged is precisely a seat where things hang. The
+  // remedy must not share a lane with the thing it is rescuing.
+  //
+  // Its own interval, reading the front-door state directly, so nothing between
+  // here and the kickstart can be blocked by the wedge.
+  setInterval(() => {
+    let state = null;
+    try { state = readFrontDoorState(AGENT_REPO_DIR, { now: Date.now() }); } catch { state = null; }
+    if (state) reviveFrontDoorIfShut(state).catch(() => {});
+  }, REVIVE_CHECK_INTERVAL_MS);
+
   // Backlog sweep
   setInterval(async () => {
     try {