@cohortapp/agent-sdk 2.11.5 → 2.11.6

package/lib/resource-governor.mjs

@@ -79,19 +79,21 @@ export const FREEMEM_FLOOR_FRACTION = frac(process.env.GOV_FREEMEM_FLOOR, 0.15);
 /** Load-average multiplier over cpu count that triggers a DEFER. */
 export const LOAD_FACTOR = num(process.env.GOV_LOAD_FACTOR, 1.5);
 /**
- * Hard ceiling on dynamicMax regardless of RAM. The binding constraint on a
- * subscription-auth seat is NOT host RAM — it is the ONE Max subscription, which
- * cannot serve many parallel `claude --print` calls without throttling. Ten
- * concurrent backlog sessions starved the reactive quick-reply's claude call
- * into a 60s timeout (observed on A016/isla). Default is subscription-safe;
- * API/multi-key deployments raise it with GOV_HARD_MAX.
+ * Hard ceiling on dynamicMax regardless of RAM. Generous by design: the runtime
+ * guards below (free-memory floor, load, projected RSS) throttle DOWN under real
+ * pressure, and stale sessions no longer count (see liveClaudeStats + STALE_*),
+ * so the ceiling can use available capacity and back off when it isn't there.
+ * The earlier 60s quick-reply timeout "at 10 sessions" was really TWO permanent
+ * zombie `claude --resume` procs (counted forever) + a log flood, not 8 healthy
+ * sessions. With those fixed, more real lanes are safe. GOV_HARD_MAX overrides.
  */
-export const HARD_MAX = num(process.env.GOV_HARD_MAX, 4);
+export const HARD_MAX = num(process.env.GOV_HARD_MAX, 7);
 /**
  * Slots the governor holds back from self-directed (backlog/cadence) work so a
  * human/agent reply always has subscription + host headroom to GENERATE, not
  * merely a dispatch slot. The dispatcher's RESERVED_INBOX_SLOTS reserves a
  * dispatch slot; this reserves the scarce resource that actually timed out.
+ * With HARD_MAX 7 this leaves ~5 backlog lanes + 2 reactive.
  */
 export const REACTIVE_RESERVE = num(process.env.GOV_REACTIVE_RESERVE, 2);
 /**
@@ -101,9 +103,36 @@ export const REACTIVE_RESERVE = num(process.env.GOV_REACTIVE_RESERVE, 2);
  * gets near-all capacity.
  */
 export const DEGRADED_BACKLOG_MAX = num(process.env.GOV_DEGRADED_BACKLOG_MAX, 1);
+/**
+ * A `claude` session older than this is treated as STALE and does NOT count
+ * toward the concurrency ceiling — an idle/wedged lane must not hold a slot
+ * forever. No healthy cadence/reply session runs anywhere near this long. Two
+ * 8-day-old `--resume` zombies once pinned the whole box (owner: "idle lanes
+ * shouldn't count"). GOV_STALE_SESSION_MS overrides.
+ */
+export const STALE_SESSION_MS = num(process.env.GOV_STALE_SESSION_MS, 30 * 60_000);
+/**
+ * A `claude` session older than this is REAPED (killed) by reapStaleClaude —
+ * the self-cleaning half. Higher than STALE so a merely-slow session is only
+ * discounted, not killed, until it is unambiguously wedged. GOV_REAP_SESSION_MS
+ * overrides; set to 0 to disable reaping (discount-only).
+ */
+export const REAP_SESSION_MS = num(process.env.GOV_REAP_SESSION_MS, 45 * 60_000);
 /** Minimum dynamicMax (always allow at least one session). */
 const MIN_MAX = 1;
 
+/**
+ * Parse a `ps` etime string ("[[DD-]HH:]MM:SS") to milliseconds. Returns 0 on
+ * anything unparseable (fail-safe: an unparseable age is treated as young, so we
+ * never discount/kill a session we cannot age).
+ */
+export function etimeToMs(etime) {
+  const m = String(etime || "").trim().match(/^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)$/);
+  if (!m) return 0;
+  const [, d, h, mm, ss] = m;
+  return ((((Number(d || 0) * 24) + Number(h || 0)) * 60 + Number(mm || 0)) * 60 + Number(ss || 0)) * 1000;
+}
+
 function num(v, dflt) {
   const n = Number(v);
   return Number.isFinite(n) && n > 0 ? n : dflt;
@@ -176,20 +205,26 @@ export function liveClaudeStats(opts = {}) {
   try {
     // pid, rss(KB), comm — mirror the watchdog's filter set so the governor's
     // view of "live claude" matches what the watchdog would act on.
-    const out = execFileSync("/bin/ps", ["-eo", "pid,rss,comm"], { encoding: "utf-8", timeout: 4000 });
+    const out = execFileSync("/bin/ps", ["-eo", "pid,etime,rss,comm"], { encoding: "utf-8", timeout: 4000 });
     let count = 0;
     let rssKb = 0;
+    let stale = 0;
     for (const line of out.split("\n")) {
       const l = line.trim();
       if (!l) continue;
       if (!/claude/i.test(l)) continue;
       if (/Claude\.app|Claude Helper|watchdog|grep|ps -eo/.test(l)) continue;
       const parts = l.split(/\s+/);
-      const rss = Number(parts[1]);
+      // Columns: pid etime rss comm...  A STALE session (idle/wedged, older than
+      // STALE_SESSION_MS) does NOT count toward the ceiling and its RSS is
+      // excluded — an abandoned lane must not hold a slot or RAM headroom forever.
+      const ageMs = etimeToMs(parts[1]);
+      if (STALE_SESSION_MS > 0 && ageMs >= STALE_SESSION_MS) { stale += 1; continue; }
+      const rss = Number(parts[2]);
       if (Number.isFinite(rss)) rssKb += rss;
       count += 1;
     }
-    value = { count, rssMB: Math.round(rssKb / 1024) };
+    value = { count, rssMB: Math.round(rssKb / 1024), stale };
   } catch {
     value = { count: 0, rssMB: 0 };
   }
@@ -200,6 +235,43 @@ export function liveClaudeStats(opts = {}) {
 /** Reset the live-stats cache (tests). */
 export function _resetLiveCache() { _liveCache = { at: 0, value: null }; }
 
+/**
+ * Self-cleaning: KILL `claude` sessions older than REAP_SESSION_MS. A session
+ * that has run this long is unambiguously wedged — no healthy cadence or reply
+ * session runs anywhere near this long — and a wedged session holds a dispatch
+ * slot, RAM, and (worst) a share of the single Max subscription. Two 8-day-old
+ * `--resume` zombies once deferred every cadence on a seat, silencing its inbox.
+ *
+ * Side-effecting BY DESIGN, so it is NOT called from the pure admit() path — the
+ * daemon calls it on a timer. Best-effort: a ps/kill failure never throws.
+ * `deps` ({ psImpl, killImpl, reapMs }) is injected by tests.
+ *
+ * @returns {{ reaped:number[], errors:number }}
+ */
+export function reapStaleClaude(deps = {}) {
+  const reapMs = Number.isFinite(deps.reapMs) ? deps.reapMs : REAP_SESSION_MS;
+  if (!(reapMs > 0)) return { reaped: [], errors: 0 };
+  const reaped = [];
+  let errors = 0;
+  try {
+    const out = typeof deps.psImpl === "function"
+      ? deps.psImpl()
+      : execFileSync("/bin/ps", ["-eo", "pid,etime,comm"], { encoding: "utf-8", timeout: 4000 });
+    const kill = typeof deps.killImpl === "function" ? deps.killImpl : (pid) => process.kill(pid, "SIGKILL");
+    for (const line of String(out).split("\n")) {
+      const l = line.trim();
+      if (!l || !/claude/i.test(l)) continue;
+      if (/Claude\.app|Claude Helper|watchdog|grep|ps -eo/.test(l)) continue;
+      const parts = l.split(/\s+/);
+      const pid = Number(parts[0]);
+      if (!Number.isInteger(pid) || pid <= 1) continue;      // never signal pid 0/1
+      if (etimeToMs(parts[1]) < reapMs) continue;
+      try { kill(pid); reaped.push(pid); } catch { errors += 1; }
+    }
+  } catch { /* best-effort — a ps failure must never crash the daemon */ }
+  return { reaped, errors };
+}
+
 // ---------------------------------------------------------------------------
 // Default deps snapshot
 // ---------------------------------------------------------------------------

package/lib/resource-governor.test.mjs

@@ -23,6 +23,8 @@ import {
   DECISIONS,
   availableMemoryBytes,
   FREEMEM_FLOOR_FRACTION,
+  etimeToMs,
+  reapStaleClaude,
 } from "./resource-governor.mjs";
 
 test("availableMemoryBytes counts reclaimable pages, not just free ones", () => {
@@ -68,15 +70,14 @@ function deps(over = {}) {
 // ---------------------------------------------------------------------------
 
 test("dynamicMax sizes by RAM, clamped to min(HARD_MAX, cpus)", () => {
-  // HARD_MAX is 4 (subscription-safe): the single Max subscription, not host RAM,
-  // is the binding constraint, so anything with a few GB clamps to 4.
+  // HARD_MAX is 7 (generous; runtime guards + stale-reaping keep it safe).
   // 4GB: byRam=floor(4*0.6/0.75)=3 → RAM-bound, below HARD_MAX → 3.
   assert.equal(dynamicMax({ totalmem: 4 * GB, cpus: 8 }), 3);
-  // 8GB: byRam=floor(8*0.6/0.75)=6 → clamp to min(4,8)=4 → HARD_MAX-bound.
-  assert.equal(dynamicMax({ totalmem: 8 * GB, cpus: 8 }), 4);
-  // 16GB with 16 cpus: byRam=12 → clamp to min(4,16)=4.
-  assert.equal(dynamicMax({ totalmem: 16 * GB, cpus: 16 }), 4);
-  // cpu-bound below HARD_MAX: 16GB with 2 cpus → min(4,2)=2.
+  // 8GB: byRam=floor(8*0.6/0.75)=6 → RAM-bound, below HARD_MAX → 6.
+  assert.equal(dynamicMax({ totalmem: 8 * GB, cpus: 8 }), 6);
+  // 16GB with 16 cpus: byRam=12 → clamp to min(7,16)=7 (HARD_MAX-bound).
+  assert.equal(dynamicMax({ totalmem: 16 * GB, cpus: 16 }), 7);
+  // cpu-bound below HARD_MAX: 16GB with 2 cpus → min(7,2)=2.
   assert.equal(dynamicMax({ totalmem: 16 * GB, cpus: 2 }), 2);
   // Tiny box always yields at least 1.
   assert.equal(dynamicMax({ totalmem: 1 * GB, cpus: 1 }), 1);
@@ -93,8 +94,8 @@ test("dynamicMax never returns 0 even on a starved box", () => {
 test("admit: healthy box with spare capacity → ADMIT", () => {
   const r = admit({ source: "backlog" }, deps());
   assert.equal(r.decision, DECISIONS.ADMIT);
-  assert.equal(r.snapshot.dynamicMax, 4);
-  assert.equal(r.snapshot.effectiveMax, 4);
+  assert.equal(r.snapshot.dynamicMax, 7);
+  assert.equal(r.snapshot.effectiveMax, 7);
 });
 
 test("admit: inbox on a healthy box → ADMIT", () => {
@@ -107,22 +108,22 @@ test("admit: inbox on a healthy box → ADMIT", () => {
 // ---------------------------------------------------------------------------
 
 test("admit: backlog stops REACTIVE_RESERVE slots short of the ceiling → QUEUE", () => {
-  // effectiveMax = 4; self-directed backlog may use effectiveMax - REACTIVE_RESERVE(2)
-  // = 2. With 2 already live, backlog QUEUEs — holding 2 slots for the inbox so a
+  // effectiveMax = 7; self-directed backlog may use effectiveMax - REACTIVE_RESERVE(2)
+  // = 5. With 5 already live, backlog QUEUEs — holding 2 slots for the inbox so a
   // reply can always be GENERATED (subscription + host headroom, not just a slot).
-  const r = admit({ source: "backlog" }, deps({ liveClaude: { count: 2, rssMB: 500 } }));
+  const r = admit({ source: "backlog" }, deps({ liveClaude: { count: 5, rssMB: 1200 } }));
   assert.equal(r.decision, DECISIONS.QUEUE);
   assert.match(r.reason, /yields 2 slots to the inbox/);
 });
 
 test("admit: inbox may use the whole envelope; the reactive reserve is for it", () => {
-  // 2 live (backlog's reserved cap) but INBOX still ADMITs — the reserve is its.
+  // 5 live (backlog's reserved cap) but INBOX still ADMITs — the reserve is its.
   assert.equal(
-    admit({ source: "inbox" }, deps({ liveClaude: { count: 2, rssMB: 500 } })).decision,
+    admit({ source: "inbox" }, deps({ liveClaude: { count: 5, rssMB: 1200 } })).decision,
     DECISIONS.ADMIT,
   );
-  // Only at the full envelope (effectiveMax=4 live) does even the inbox QUEUE.
-  const r = admit({ source: "inbox" }, deps({ liveClaude: { count: 4, rssMB: 1000 } }));
+  // Only at the full envelope (effectiveMax=7 live) does even the inbox QUEUE.
+  const r = admit({ source: "inbox" }, deps({ liveClaude: { count: 7, rssMB: 1800 } }));
   assert.equal(r.decision, DECISIONS.QUEUE);
   assert.match(r.reason, /concurrency ceiling/);
 });
@@ -347,3 +348,46 @@ test("an unrecognised budget mode is treated as normal, never as something stric
   assert.equal(admit({ source: "backlog", mode: "catastrophic" }, deps()).decision, "ADMIT");
   assert.equal(admit({ source: "backlog", mode: "normal" }, deps()).decision, "ADMIT");
 });
+
+// ---------------------------------------------------------------------------
+// Stale-session accounting + reaping (idle lanes must not count / self-clean)
+// ---------------------------------------------------------------------------
+
+test("etimeToMs parses ps elapsed-time in all three shapes", () => {
+  assert.equal(etimeToMs("05:20"), (5 * 60 + 20) * 1000);            // MM:SS
+  assert.equal(etimeToMs("07:05:20"), (7 * 3600 + 5 * 60 + 20) * 1000); // HH:MM:SS
+  assert.equal(etimeToMs("08-07:05:20"), (8 * 86400 + 7 * 3600 + 5 * 60 + 20) * 1000); // DD-HH:MM:SS
+  assert.equal(etimeToMs("garbage"), 0); // fail-safe: unparseable → young
+  assert.equal(etimeToMs(""), 0);
+});
+
+test("reapStaleClaude kills only claude sessions older than the reap threshold", () => {
+  const ps = [
+    "  100 08-07:05:20 claude --dangerously-skip-permissions --resume old", // 8 days → reap
+    "  200 00:30 claude --print young",                                     // 30s → keep
+    "  300 50:00 claude --resume wedged",                                   // 50 min → reap
+    "  400 90:00 node scripts/daemon/maestro-daemon.mjs",                   // not claude → skip
+    "  500 46:00 Claude Helper (Renderer)",                                 // helper → skip
+    "    1 99-00:00:00 launchd",                                            // pid 1 / not claude → skip
+  ].join("\n");
+  const killed = [];
+  const r = reapStaleClaude({ psImpl: () => ps, killImpl: (p) => killed.push(p), reapMs: 45 * 60_000 });
+  assert.deepEqual(r.reaped.sort((a, b) => a - b), [100, 300]);
+  assert.deepEqual(killed.sort((a, b) => a - b), [100, 300]);
+  assert.equal(r.errors, 0);
+});
+
+test("reapStaleClaude is a no-op when reaping is disabled (reapMs=0)", () => {
+  const ps = "  100 08-07:05:20 claude --resume old\n";
+  const killed = [];
+  const r = reapStaleClaude({ psImpl: () => ps, killImpl: (p) => killed.push(p), reapMs: 0 });
+  assert.deepEqual(r.reaped, []);
+  assert.deepEqual(killed, []);
+});
+
+test("reapStaleClaude counts kill failures without throwing", () => {
+  const ps = "  100 08-07:05:20 claude --resume old\n";
+  const r = reapStaleClaude({ psImpl: () => ps, killImpl: () => { throw new Error("ESRCH"); }, reapMs: 60_000 });
+  assert.deepEqual(r.reaped, []);
+  assert.equal(r.errors, 1);
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cohortapp/agent-sdk",
-  "version": "2.11.5",
+  "version": "2.11.6",
   "description": "Cohort Agent SDK — autonomous AI colleague runtime. Deploy senior AI colleagues on dedicated Mac minis, wired to the Cohort operating surface.",
   "type": "module",
   "bin": {

package/scripts/daemon/maestro-daemon.mjs

@@ -113,6 +113,31 @@ try {
   // Reactive daemon continues. Doctor / healthcheck will surface this.
 }
 
+// ---------------------------------------------------------------------------
+// 3a. Stale-session reaper (self-cleaning)
+// ---------------------------------------------------------------------------
+// A wedged `claude` session (crash mid-stream, an orphaned `--resume`) otherwise
+// lingers for DAYS. The governor counts it as "live" via ps, so it holds a
+// dispatch slot, RAM, and — worst — a share of the single Max subscription, until
+// the seat defers every cadence and its inbox goes silent (observed on A016: two
+// 8-day-old zombies deferred 6704 ticks). Reap anything older than
+// REAP_SESSION_MS: once at boot (any old session is a leftover) and on a timer so
+// idle lanes self-clean. Best-effort — never takes the daemon down.
+try {
+  const { reapStaleClaude } = await import("../../lib/resource-governor.mjs");
+  const sweep = () => {
+    try {
+      const r = reapStaleClaude();
+      if (r.reaped.length) console.log(`[DAEMON] reaped ${r.reaped.length} stale claude session(s): ${r.reaped.join(", ")}`);
+    } catch { /* best-effort */ }
+  };
+  sweep();
+  const reapTimer = setInterval(sweep, 5 * 60_000);
+  if (reapTimer && typeof reapTimer.unref === "function") reapTimer.unref();
+} catch (err) {
+  console.error(`[DAEMON] stale-session reaper failed to start: ${err && err.message}`);
+}
+
 // ---------------------------------------------------------------------------
 // 3b. Channel bus (1.12.0+; generic platform loop in 1.15.0/WS2)
 // ---------------------------------------------------------------------------