@commonlyai/cli 0.1.198 → 0.1.200

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@commonlyai/cli",
-  "version": "0.1.198",
+  "version": "0.1.200",
   "license": "Apache-2.0",
   "description": "The Commonly CLI — connect agents, manage pods, iterate fast",
   "type": "module",

package/src/lib/adapters/claude.js

@@ -1,6 +1,11 @@
 import { claudeRuntimeInventory } from '../native-runtime-inventory.js';
 import { observeSpawn, controlsFromArguments } from '../runtime-observation.js';
-import { spawnInOwnGroup, wantsOwnProcessGroup } from '@commonlyai/seats/process-group';
+import {
+  firstOutputWindowMs, killStallSurvivors, physicalPath, spawnStalledError, STALL_KILL_GRACE_MS, watchFirstOutput,
+} from '../first-output-watchdog.js';
+import {
+  descendantsOf, killProcessTree, readProcessTable, spawnInOwnGroup, wantsOwnProcessGroup,
+} from '@commonlyai/seats/process-group';
 /**
  * claude adapter — wraps the local `claude` CLI as a Commonly agent.
  *
@@ -88,7 +93,7 @@ import {
   prepareMcpSpawn, PUBLIC_MCP_UNAVAILABLE_CODE, publicMcpUnavailableError, selectedMcpVersionDir,
 } from '../mcp-home.js';
 import { buildMemoryPreamble, ROW_SLICING_CUE } from '../memory-bridge.js';
-import { claudeConfigDirFor, claudeContextTokens } from '../session-recycle.js';
+import { claudeConfigDirFor, claudeContextTokens, claudeTranscriptPath } from '../session-recycle.js';
 import { skillsAsText } from '../skill-prompt.js';
 import { adapterFailure, spawnCredentials } from '../upstream-refusal.js';
 
@@ -484,7 +489,11 @@ const publicSkillPromptArgs = async (skillSources) => {
 
 // `ownGroup` is set only under the run supervisor, which kills the child's
 // whole group; unsupervised, the spawn options are exactly what they were.
-const runClaude = ({ cmd, args, cwd, env, timeoutMs, credentials = [], spawnImpl = childSpawn, onSpawn, controls = {}, ownGroup = false }) => new Promise((resolve, reject) => {
+const runClaude = ({
+  cmd, args, cwd, env, timeoutMs, credentials = [], spawnImpl = childSpawn, onSpawn, controls = {}, ownGroup = false,
+  transcriptPath = null, firstOutputMs = firstOutputWindowMs(), lockProbe, stallKillGraceMs = STALL_KILL_GRACE_MS,
+  processTable = readProcessTable, survivorIo,
+}) => new Promise((resolve, reject) => {
   const proc = spawnInOwnGroup(spawnImpl, cmd, args, {
     cwd,
     env,
@@ -499,15 +508,62 @@ const runClaude = ({ cmd, args, cwd, env, timeoutMs, credentials = [], spawnImpl
     timedOut = true;
     proc.kill('SIGTERM');
   }, timeoutMs);
+  // A turn that never starts is killed long before the run timeout.
+  let stalled = null;
+  let stallKill = null;
+  let stallSnapshot = [];
+  // SIGKILL snapshotted descendants that are still alive. Safe to repeat. It
+  // runs when the spawn exits, at the grace deadline and on close: a survivor
+  // still holding the spawn's stdout or stderr keeps close from ever firing,
+  // and once the spawn is reaped killProcessTree can no longer reach it.
+  const sweepStallSurvivors = () => {
+    if (stalled) killStallSurvivors(stallSnapshot, survivorIo);
+  };
+  const treeIo = (table) => ({ table, ...(survivorIo?.read ? { read: survivorIo.read } : {}) });
+  const watchdog = watchFirstOutput({
+    proc,
+    windowMs: firstOutputMs,
+    transcriptPath,
+    lockProbe,
+    onStall: (reason) => {
+      stalled = spawnStalledError({ label: 'claude', windowMs: firstOutputMs, reason });
+      // Snapshot the descendants first: once the spawn exits they are
+      // reparented and can no longer be found from it.
+      const table = Number.isInteger(proc.pid) ? processTable() : null;
+      if (table) stallSnapshot = descendantsOf(table, proc.pid);
+      killProcessTree(proc, 'SIGTERM', treeIo(() => table));
+      // A process parked in a system call may not act on SIGTERM. SIGKILL the
+      // child, its group when it leads one, and its descendants.
+      stallKill = setTimeout(() => {
+        killProcessTree(proc, 'SIGKILL', treeIo(processTable));
+        sweepStallSurvivors();
+        // A pipe holder the snapshot missed (ps failed, or it was reparented
+        // before the stall) would still keep close from firing. Dropping our
+        // ends of the pipes bounds every stalled turn at window plus grace.
+        proc.stdout?.destroy?.();
+        proc.stderr?.destroy?.();
+      }, stallKillGraceMs);
+      stallKill.unref?.();
+    },
+  });
+  proc.on('exit', sweepStallSurvivors);
 
   proc.stdout?.on('data', (chunk) => { stdout += chunk.toString(); });
   proc.stderr?.on('data', (chunk) => { stderr += chunk.toString(); });
   proc.on('error', (err) => {
     clearTimeout(timer);
+    clearTimeout(stallKill);
+    watchdog.stop();
     reject(err);
   });
   proc.on('close', (code) => {
     clearTimeout(timer);
+    clearTimeout(stallKill);
+    watchdog.stop();
+    if (stalled) {
+      sweepStallSurvivors();
+      return reject(stalled);
+    }
     if (timedOut) return reject(new Error(`claude timed out after ${timeoutMs}ms`));
     if (code !== 0) {
       // Report stdout too, not just stderr. In `-p` mode claude writes terminal
@@ -1200,11 +1256,18 @@ export default {
         credentials: spawnCredentials(spawnCtx),
         spawnImpl: ctx._spawnImpl, onSpawn: ctx.onSpawn, controls: controlsFromArguments(modelArgs, { modelFlag: '--model', effortFlag: '--effort' }), // test seam only — do not use in production
         ownGroup: wantsOwnProcessGroup(ctx),
+        transcriptPath: claudeTranscriptPath({
+          cwd: physicalPath(ctx.cwd), sessionId, configDir: claudeConfigDirFor({ ...env, HOME: childHome || env?.HOME }),
+        }),
+        lockProbe: ctx._lockProbe, // test seams only
+        stallKillGraceMs: ctx._stallKillGraceMs,
+        processTable: ctx._processTable,
+        survivorIo: ctx._survivorIo,
       });
       // TASK-440: the context this turn left, read from the transcript the
       // child wrote under its own env (an isolated HOME for public seats).
       const contextTokens = claudeContextTokens({
-        cwd: ctx.cwd, sessionId, configDir: claudeConfigDirFor({ ...env, HOME: childHome || env?.HOME }),
+        cwd: physicalPath(ctx.cwd), sessionId, configDir: claudeConfigDirFor({ ...env, HOME: childHome || env?.HOME }),
       });
       return { text: stdout.trim(), newSessionId: sessionId, contextTokens };
     } catch (err) {
@@ -1256,9 +1319,17 @@ export default {
           credentials: spawnCredentials(retryCtx),
           spawnImpl: ctx._spawnImpl, onSpawn: ctx.onSpawn, controls: controlsFromArguments(modelArgs, { modelFlag: '--model', effortFlag: '--effort' }),
           ownGroup: wantsOwnProcessGroup(ctx),
+          transcriptPath: claudeTranscriptPath({
+            cwd: physicalPath(ctx.cwd), sessionId: freshId,
+            configDir: claudeConfigDirFor({ ...retry.env, HOME: retry.childHome || retry.env?.HOME }),
+          }),
+          lockProbe: ctx._lockProbe,
+          stallKillGraceMs: ctx._stallKillGraceMs,
+          processTable: ctx._processTable,
+          survivorIo: ctx._survivorIo,
         });
         const contextTokens = claudeContextTokens({
-          cwd: ctx.cwd, sessionId: freshId,
+          cwd: physicalPath(ctx.cwd), sessionId: freshId,
           configDir: claudeConfigDirFor({ ...retry.env, HOME: retry.childHome || retry.env?.HOME }),
         });
         return { text: stdout.trim(), newSessionId: freshId, contextTokens };

package/src/lib/first-output-watchdog.js

@@ -0,0 +1,141 @@
+/**
+ * First-output watchdog for a seat's model CLI spawn.
+ *
+ * On 2026-10-30 every launchd seat's `claude -p` turn parked while the Mac's
+ * console was locked: no transcript row, no stdout, no stderr, for the full
+ * 45-minute run timeout. Each one held its delivery the whole time, the retry
+ * respawned it into the same hang, and the log said only "claude timed out
+ * after 2700000ms", which names nothing.
+ *
+ * The signal is not stdout. A seat runs `claude -p --output-format text`,
+ * which writes stdout once, at the end, so a healthy 20-minute turn has zero
+ * stdout bytes for 20 minutes. What a healthy turn does write is its session
+ * transcript: measured on the dart seat, the first row landed 0.7 s after the
+ * spawn, and the hung spawns that night wrote no row at all. So a spawn counts
+ * as started when it writes any stdout or stderr byte OR its transcript grows.
+ * A spawn that does neither inside the window is killed with a reason. When
+ * the transcript is missing altogether, the path may simply be wrong, so the
+ * spawn is killed only if the console is also locked.
+ */
+import { realpathSync, statSync } from 'fs';
+import { readProcess } from '@commonlyai/seats/process-group';
+
+export const DEFAULT_FIRST_OUTPUT_WINDOW_MS = 180_000;
+
+// 0 turns the watchdog off; anything unparseable keeps the default.
+export const firstOutputWindowMs = (env = process.env) => {
+  const raw = env?.COMMONLY_FIRST_OUTPUT_WINDOW_MS;
+  if (raw === undefined || raw === '') return DEFAULT_FIRST_OUTPUT_WINDOW_MS;
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : DEFAULT_FIRST_OUTPUT_WINDOW_MS;
+};
+
+export const SPAWN_STALLED_CODE = 'spawn_stalled';
+
+// The message carries the class word spawn-retry.js matches, so the stall gets
+// its own backoff rather than RUNTIME's 5-second retry.
+export const spawnStalledError = ({ label, windowMs, reason }) => {
+  const error = new Error(
+    `${label} produced no output in ${Math.round(windowMs / 1000)}s (${reason}); killed before the run timeout`,
+  );
+  error.code = SPAWN_STALLED_CODE;
+  error.stallReason = reason;
+  return error;
+};
+
+// `ioreg -n Root -d1` prints `"IOConsoleLocked" = Yes` while the screen is
+// locked. True, false, or null when it cannot tell (not darwin, ioreg failed).
+// child_process is loaded on demand: suites that mock it with only `spawn`
+// would otherwise fail to import every adapter that uses this module.
+export const consoleLocked = async ({ platform = process.platform, execFileImpl } = {}) => {
+  if (platform !== 'darwin') return null;
+  const run = execFileImpl || (await import('child_process')).execFile;
+  if (typeof run !== 'function') return null;
+  return new Promise((resolve) => {
+    run('/usr/sbin/ioreg', ['-n', 'Root', '-d1'], { timeout: 5000 }, (error, stdout) => {
+      if (error) return resolve(null);
+      const match = String(stdout).match(/"IOConsoleLocked"\s*=\s*(Yes|No)/);
+      return resolve(match ? match[1] === 'Yes' : null);
+    });
+  });
+};
+
+// Claude names its project folder after process.cwd(), which on macOS is the
+// physical path (/tmp is /private/tmp), so the transcript is looked for there.
+export const physicalPath = (dir) => {
+  if (!dir) return dir;
+  try {
+    return realpathSync(dir);
+  } catch {
+    return dir;
+  }
+};
+
+// How long a stalled spawn gets to exit after SIGTERM before SIGKILL.
+export const STALL_KILL_GRACE_MS = 10_000;
+
+/**
+ * SIGKILL each snapshotted descendant of a stalled spawn that is still alive
+ * once the spawn itself has exited. Its parent may have exited, so it may now
+ * belong to launchd: only the pid and start time are matched, which still
+ * rules out a reused pid. Returns the pids signalled.
+ */
+export const killStallSurvivors = (snapshot, { read = readProcess, kill = process.kill.bind(process) } = {}) => {
+  const killed = [];
+  for (const row of snapshot || []) {
+    const now = read(row.pid);
+    if (!now || now.started !== row.started) continue;
+    try { kill(row.pid, 'SIGKILL'); killed.push(row.pid); } catch { /* gone */ }
+  }
+  return killed;
+};
+
+const sizeOf = (file) => {
+  if (!file) return -1;
+  try {
+    return statSync(file).size;
+  } catch {
+    return -1;
+  }
+};
+
+/**
+ * Watch `proc` for its first sign of life. Returns `{ stop }`; call it when the
+ * process closes. `onStall(reason)` fires at most once, after the caller should
+ * kill the process.
+ */
+export const watchFirstOutput = ({
+  proc,
+  windowMs,
+  transcriptPath = null,
+  onStall,
+  lockProbe = consoleLocked,
+  setTimeoutImpl = setTimeout,
+  clearTimeoutImpl = clearTimeout,
+}) => {
+  if (!windowMs) return { stop: () => {} };
+  let started = false;
+  let stopped = false;
+  const markStarted = () => { started = true; };
+  proc.stdout?.on('data', markStarted);
+  proc.stderr?.on('data', markStarted);
+  // A missing file counts as empty, so creating an empty one is not growth.
+  const baseline = Math.max(0, sizeOf(transcriptPath));
+  const timer = setTimeoutImpl(async () => {
+    const size = sizeOf(transcriptPath);
+    if (started || size > baseline) return;
+    const locked = await lockProbe().catch(() => null);
+    if (stopped) return;
+    // No transcript where one was expected may only mean the path is wrong, so
+    // that alone never kills a turn: it takes a locked console as well.
+    if (size < 0 && locked !== true) return;
+    onStall(locked ? 'console locked' : 'no transcript, stdout or stderr');
+  }, windowMs);
+  timer?.unref?.();
+  return {
+    stop: () => {
+      stopped = true;
+      clearTimeoutImpl(timer);
+    },
+  };
+};

package/src/lib/spawn-retry.js

@@ -16,8 +16,16 @@ export const SPAWN_FAILURE_CLASS = Object.freeze({
   RATE_LIMIT: 'rate_limit',
   CONFIGURATION: 'configuration',
   RUNTIME: 'runtime',
+  // The spawn never started: no transcript row, stdout or stderr inside the
+  // first-output window (lib/first-output-watchdog.js). A host condition such as
+  // a locked console, so the circuit opens at once and the probe waits.
+  STALLED: 'stalled',
 });
 
+// A stalled probe costs one first-output window, so the wait starts short and
+// doubles to the usual ceiling: 2, 4, 8, then 15 minutes.
+export const SPAWN_STALL_RETRY_BASE_MS = 2 * 60 * 1000;
+
 export const SPAWN_CIRCUIT_THRESHOLD = 3;
 // Base ceiling before the stable per-agent 0–20% anti-herd offset.
 export const SPAWN_RETRY_MAX_MS = 15 * 60 * 1000;
@@ -116,6 +124,7 @@ export const classifySpawnFailure = (error) => {
   // First: the refusal text carries none of the words below today, but a
   // refusal must never be read as a transient condition and retried.
   if (CONTENT_REFUSED_RE.test(text)) return SPAWN_FAILURE_CLASS.CONTENT_REFUSED;
+  if (error?.code === 'spawn_stalled') return SPAWN_FAILURE_CLASS.STALLED;
   // Provider APIs commonly report an exhausted account quota as HTTP 429.
   // Prefer the more specific body/message over the generic status code so a
   // hard quota failure gets the long cooldown rather than a one-minute probe.
@@ -194,6 +203,17 @@ export const spawnRetryPolicy = ({
     };
   }
 
+  if (failureClass === SPAWN_FAILURE_CLASS.STALLED) {
+    return {
+      failureClass,
+      circuitOpen: true,
+      delayMs: applyJitter(
+        Math.min(SPAWN_RETRY_MAX_MS, SPAWN_STALL_RETRY_BASE_MS * (2 ** (failureCount - 1))),
+        jitterRatio,
+      ),
+    };
+  }
+
   if (failureClass === SPAWN_FAILURE_CLASS.RATE_LIMIT) {
     return {
       failureClass,