@commonlyai/cli 0.1.218 → 0.1.219

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@commonlyai/cli",
-  "version": "0.1.218",
+  "version": "0.1.219",
   "license": "Apache-2.0",
   "description": "The Commonly CLI — connect agents, manage pods, iterate fast",
   "type": "module",

package/src/commands/agent.js

@@ -55,6 +55,7 @@ import {
 import { catchUpBeforeTurn } from '../lib/pod-catch-up.js';
 import { readLongTerm, syncBack } from '../lib/memory-bridge.js';
 import { pollRetryPolicy } from '../lib/poll-retry.js';
+import { createPollLiveness, formatStall, resolvePollLivenessMs } from '../lib/poll-liveness.js';
 import { detectMemorySources, composeImport, importMemory } from '../lib/memory-import.js';
 import { detectSkills, importSkills, parseSkillMeta } from '../lib/skills-import.js';
 import { compileServerSkills } from '../lib/server-skills.js';
@@ -1561,6 +1562,14 @@ export const performRun = ({
   // ~/.commonly/state/turns/<agent>.jsonl once delivery settles. Off, the spawn
   // path is the plain adapter call it has always been, and nothing is written.
   superviseRuns,
+  // TASK-338. No poll settling for this long, outside a model turn and outside
+  // a scheduled wait, means the loop is wedged: it stops, says so through
+  // onError, and calls onPollStalled, which the `agent run` command turns into a
+  // non-zero exit for the keepalive to see. Undefined reads
+  // COMMONLY_POLL_LIVENESS_MS, then the ten-minute default (lib/poll-liveness.js).
+  pollLivenessMs,
+  pollLivenessCheckMs,
+  onPollStalled,
 }) => {
   const supervising = superviseRuns ?? superviseRunsByDefault(adapter?.name);
   reportRuntime = reportRuntime && WALKED_HARNESSES.includes(adapter?.name);
@@ -1664,6 +1673,26 @@ export const performRun = ({
   const MAX_AUTH_ERRORS = 3;
   let consecutivePollFailures = 0;
   let consecutiveSpawnFailures = 0;
+  let pollStalled = false;
+  const pollLiveness = createPollLiveness({
+    limitMs: resolvePollLivenessMs(pollLivenessMs),
+    checkEveryMs: pollLivenessCheckMs,
+    onStall: ({ stalledMs }) => {
+      // A loop that already stopped itself (auth, a superseded delivery) is
+      // draining, not wedged.
+      if (!running) return;
+      pollStalled = true;
+      running = false;
+      pollLiveness.stop();
+      // The same teardown as stop(): a held retry must not keep renewing its
+      // lease on a loop that will never deliver it.
+      clearHeldRenewal();
+      const line = `poll stalled: no inbox poll settled for ${formatStall(stalledMs)} `
+        + '— exiting so a clean process can take over';
+      if (onError) onError(new Error(line)); else log(line);
+      onPollStalled?.({ stalledMs });
+    },
+  });
   const spawnJitterRatio = retryJitterRatio ?? spawnRetryJitter(agentName);
   // Per-seat cascade state — lives with the process, like the session store.
   // A wrapper restart forgets the streak; the decay window covers that gap.
@@ -3168,6 +3197,13 @@ export const performRun = ({
     }
   };
 
+  // A model turn is not a stall, however long it runs: the liveness clock
+  // stops for it and restarts when it ends.
+  const duringTurn = async (turn) => {
+    pollLiveness.pause();
+    try { return await turn(); } finally { pollLiveness.arm(); }
+  };
+
   const tick = async () => {
     if (!running) return;
     let nextPollDelayMs = intervalMs;
@@ -3181,10 +3217,19 @@ export const performRun = ({
       // do not reintroduce a mixed-pod `limit: 10` here.
       // TASK-317: name this CLI for the chat chip. The server keeps it as a
       // display-only label (runtime.reportedCli) and never as a binding.
-      const { events = [], inboxCount } = await client.get('/api/agents/runtime/events', {
+      // The liveness clock runs from here until the next scheduled wait, paused
+      // only while a model turn runs (lib/poll-liveness.js).
+      pollLiveness.arm();
+      const page = await client.get('/api/agents/runtime/events', {
         agentName, instanceId, limit: batchLimit,
         ...(adapter?.name ? { reportedCli: adapter.name } : {}),
       });
+      pollLiveness.arm();
+      // A stall fired while this request hung: the loop has already given up on
+      // it. A plain stop() falls through instead, to the group loop's own
+      // `running` check, which leaves the page's rows to requeue as it always has.
+      if (pollStalled) return;
+      const { events = [], inboxCount } = page;
       consecutiveAuthErrors = 0;
       consecutivePollFailures = 0;
       // A deployed CLI can briefly run against an older server that still
@@ -3298,10 +3343,10 @@ export const performRun = ({
           let batch;
           try {
             // The person's item leads the prompt; the greeting rides along.
-            batch = await processEventBatch(
+            batch = await duringTurn(() => processEventBatch(
               [...group].sort((a, b) => inboxPriority(a) - inboxPriority(b)),
               pageInboxCount,
-            );
+            ));
           } catch (err) {
             await scheduleBatchRetry(err, group);
             break;
@@ -3358,7 +3403,7 @@ export const performRun = ({
         } else {
           const eventWillSpawn = Boolean(extractPrompt(event) && (event.podId || podId));
           try {
-            result = await processEvent(event);
+            result = await duringTurn(() => processEvent(event));
           } catch (err) {
             if (isContentRefusal(err)) {
               // A refused private item is acked without a pod notice: it has no
@@ -3506,6 +3551,8 @@ export const performRun = ({
     // before the next inbox read, independent of the idle polling cadence. A
     // retry scheduled in the same tick keeps its own delay: the failure report
     // names it, and a sibling's completed turn must not shorten the backoff.
+    // A wait the loop schedules itself always ends, so it is not a stall.
+    pollLiveness.pause();
     if (running) setTimeoutImpl(tick, completedModelTurn && !retryScheduled ? 3000 : nextPollDelayMs);
   };
 
@@ -3513,6 +3560,7 @@ export const performRun = ({
   return {
     stop: () => {
       running = false;
+      pollLiveness.stop();
       clearHeldRenewal();
     },
   };
@@ -3897,6 +3945,8 @@ Docs:
         onError: (err) => {
           console.error(`[poll error] ${err.message}`);
         },
+        // TASK-338: a wedged poll ends the process instead of idling forever.
+        onPollStalled: () => process.exit(1),
       });
 
       // Graceful shutdown
@@ -4269,6 +4319,9 @@ Docs:
         // collapsed the duplication. The conclusion survived the premise; the
         // sentence did not, and nothing in the diff pointed at it.)
         onError: (err) => console.error(`${stamp()} [${name}] ${err.message}`),
+        // TASK-338: a wedged poll ends the process, so tmux-keepalive or
+        // launchd KeepAlive, which act only on exit, start a clean one.
+        onPollStalled: () => process.exit(1),
       });
 
       // SIGTERM is the daemon's idle-boundary handoff signal. `stop()` stops

package/src/lib/api.js

@@ -56,7 +56,82 @@ const handleResponse = async (res, session = null) => {
   return body;
 };
 
-export const createClient = ({ instance = null, token = undefined } = {}) => {
+// TASK-338: every CAP request carries a hard timeout. Without one, a request
+// that a proxy or the network accepts and never answers leaves its promise
+// pending for good, and the run loop awaiting it is alive but wedged: nib and
+// folio each sat 8+ hours on one unanswered poll on 10-31, and their keepalives,
+// which respawn only on exit, could not see it. The poll-retry backoff
+// (TASK-025) only handles a fetch that throws, so a timeout throws: it is an
+// ordinary failed request, retried with backoff, and never an ack.
+//
+// Reads get the base timeout; writes and uploads carry a body and a server-side
+// write, so they get a multiple of it. COMMONLY_CAP_TIMEOUT_MS sets the base.
+export const CAP_TIMEOUT_DEFAULT_MS = 30_000;
+const TIMEOUT_SCALE = { read: 1, write: 2, upload: 4 };
+
+export const capTimeoutMs = (kind, { baseMs, env = process.env } = {}) => {
+  const fromEnv = Number(env.COMMONLY_CAP_TIMEOUT_MS);
+  const base = [baseMs, fromEnv].find((value) => Number.isFinite(value) && value > 0)
+    ?? CAP_TIMEOUT_DEFAULT_MS;
+  return Math.min(base * (TIMEOUT_SCALE[kind] || 1), 2_147_483_647);
+};
+
+// A caller's own signal (doctor, and the 5s identity and skills reads in the
+// run loop) still applies; the hard timeout is added beside it, never instead.
+// AbortSignal.any is Node 20.3+, and `engines` allows 20.0, so the fallback
+// joins the two by hand. Either way the abort carries the source's own reason,
+// which is what nameTimeout matches on.
+// The fallback's listeners are removed when the request settles, so a caller
+// that reuses one long-lived signal does not collect one per request.
+const capSignal = (capTimeout, callerSignal) => {
+  if (!callerSignal) return { signal: capTimeout, release: () => {} };
+  if (typeof AbortSignal.any === 'function') {
+    return { signal: AbortSignal.any([callerSignal, capTimeout]), release: () => {} };
+  }
+  const joined = new AbortController();
+  const listeners = [];
+  for (const source of [callerSignal, capTimeout]) {
+    if (source.aborted) {
+      joined.abort(source.reason);
+      break;
+    }
+    const onAbort = () => joined.abort(source.reason);
+    source.addEventListener('abort', onAbort, { once: true });
+    listeners.push([source, onAbort]);
+  }
+  return {
+    signal: joined.signal,
+    release: () => listeners.forEach(([source, onAbort]) => source.removeEventListener('abort', onAbort)),
+  };
+};
+
+export const formatLimit = (ms) => (ms < 1000 ? `${ms}ms` : `${Math.round(ms / 1000)}s`);
+
+// The cap's timeout rejects with a DOMException named TimeoutError, from the
+// fetch or from the body read. Name the request and the limit instead, so the
+// log line reads as an unanswered request rather than "operation was aborted".
+// Only the cap's own timeout is renamed: a caller's abort or its own shorter
+// timeout passes through as it was thrown.
+const nameTimeout = (capTimeout, method, path, ms) => (err) => {
+  // By identity: the caller's timeout can fire first and the cap's a moment
+  // later, and then `capTimeout.aborted` alone would claim the caller's error.
+  if (err?.name !== 'TimeoutError' || err !== capTimeout.reason) throw err;
+  throw Object.assign(
+    new Error(`${method} ${path} got no answer in ${formatLimit(ms)}`),
+    { code: 'cap_timeout', timeoutMs: ms },
+  );
+};
+
+const timedFetch = (url, init, { method, path, ms, session }) => {
+  const capTimeout = AbortSignal.timeout(ms);
+  const { signal, release } = capSignal(capTimeout, init.signal);
+  return fetch(url, { ...init, signal })
+    .then((res) => handleResponse(res, session))
+    .catch(nameTimeout(capTimeout, method, path, ms))
+    .finally(release);
+};
+
+export const createClient = ({ instance = null, token = undefined, timeoutMs = undefined } = {}) => {
   const baseUrl = resolveInstanceUrl(instance);
   const resolved = resolveInstance(instance);
   const authToken = token === undefined ? getToken(instance) : token;
@@ -64,30 +139,33 @@ export const createClient = ({ instance = null, token = undefined } = {}) => {
     instanceKey: resolved?.key || (typeof instance === 'string' && instance && !/^https?:\/\//i.test(instance) ? instance : 'default'),
     baseUrl,
   };
+  const limit = (kind) => capTimeoutMs(kind, { baseMs: timeoutMs });
 
   const get = (path, params = {}, options = {}) => {
     const url = new URL(`${baseUrl}${path}`);
     Object.entries(params).forEach(([k, v]) => v != null && url.searchParams.set(k, v));
-    return fetch(url.toString(), { headers: headers(authToken), ...options }).then((res) => handleResponse(res, session));
+    return timedFetch(url.toString(), { headers: headers(authToken), ...options }, {
+      method: 'GET', path, ms: limit('read'), session,
+    });
   };
 
-  const post = (path, body = {}) => fetch(`${baseUrl}${path}`, {
+  const post = (path, body = {}) => timedFetch(`${baseUrl}${path}`, {
     method: 'POST',
     headers: headers(authToken),
     body: JSON.stringify(body),
-  }).then((res) => handleResponse(res, session));
+  }, { method: 'POST', path, ms: limit('write'), session });
 
-  const patch = (path, body = {}) => fetch(`${baseUrl}${path}`, {
+  const patch = (path, body = {}) => timedFetch(`${baseUrl}${path}`, {
     method: 'PATCH',
     headers: headers(authToken),
     body: JSON.stringify(body),
-  }).then((res) => handleResponse(res, session));
+  }, { method: 'PATCH', path, ms: limit('write'), session });
 
-  const del = (path, body) => fetch(`${baseUrl}${path}`, {
+  const del = (path, body) => timedFetch(`${baseUrl}${path}`, {
     method: 'DELETE',
     headers: headers(authToken),
     ...(body === undefined ? {} : { body: JSON.stringify(body) }),
-  }).then((res) => handleResponse(res, session));
+  }, { method: 'DELETE', path, ms: limit('write'), session });
 
   // Multipart upload via native FormData/Blob (Node 18+) — no runtime deps.
   // Content-Type is deliberately NOT set: fetch writes the multipart boundary.
@@ -103,11 +181,11 @@ export const createClient = ({ instance = null, token = undefined } = {}) => {
     for (const [k, v] of Object.entries(fields)) {
       if (v !== undefined && v !== null) form.append(k, String(v));
     }
-    return fetch(`${baseUrl}${path}`, {
+    return timedFetch(`${baseUrl}${path}`, {
       method: 'POST',
       headers: authToken ? { Authorization: `Bearer ${authToken}` } : {},
       body: form,
-    }).then((res) => handleResponse(res, session));
+    }, { method: 'POST', path, ms: limit('upload'), session });
   };
 
   return {
@@ -116,11 +194,11 @@ export const createClient = ({ instance = null, token = undefined } = {}) => {
 };
 
 // Convenience: login doesn't need a token
-export const login = async (instanceUrl, email, password) => {
-  const res = await fetch(`${instanceUrl}/api/auth/login`, {
+export const login = (instanceUrl, email, password) => {
+  const ms = capTimeoutMs('write');
+  return timedFetch(`${instanceUrl}/api/auth/login`, {
     method: 'POST',
     headers: { 'Content-Type': 'application/json' },
     body: JSON.stringify({ email, password }),
-  });
-  return handleResponse(res);
+  }, { method: 'POST', path: '/api/auth/login', ms, session: null });
 };

package/src/lib/inbox.js

@@ -62,6 +62,7 @@
 
 import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'fs';
 import { dirname } from 'path';
+import { capTimeoutMs, formatLimit } from './api.js';
 
 export const INBOX_KINDS = ['mention', 'decision', 'handoff', 'approval'];
 export const TOKEN_FILE_ENV = 'COMMONLY_TOKEN_FILE';
@@ -394,19 +395,29 @@ export const createInboxRequest = ({
     for (const [key, value] of Object.entries(params)) {
       if (value !== undefined && value !== null) url.searchParams.set(key, String(value));
     }
+    // TASK-338: the same hard timeout as every other CAP request (api.js).
+    const timeoutMs = capTimeoutMs(method === 'GET' ? 'read' : 'write');
     for (let attempt = 0; ; attempt += 1) {
-      // eslint-disable-next-line no-await-in-loop
-      const res = await fetchImpl(url.toString(), {
-        method,
-        headers: {
-          Authorization: `Bearer ${token}`,
-          'Content-Type': 'application/json',
-          'User-Agent': 'commonly-cli/0.1',
-        },
-        ...(body === undefined ? {} : { body: JSON.stringify(body) }),
-      });
-      // eslint-disable-next-line no-await-in-loop
-      const text = await res.text();
+      let res;
+      let text;
+      try {
+        // eslint-disable-next-line no-await-in-loop
+        res = await fetchImpl(url.toString(), {
+          method,
+          headers: {
+            Authorization: `Bearer ${token}`,
+            'Content-Type': 'application/json',
+            'User-Agent': 'commonly-cli/0.1',
+          },
+          ...(body === undefined ? {} : { body: JSON.stringify(body) }),
+          signal: AbortSignal.timeout(timeoutMs),
+        });
+        // eslint-disable-next-line no-await-in-loop
+        text = await res.text();
+      } catch (err) {
+        if (err?.name !== 'TimeoutError') throw err;
+        throw new InboxRequestError(`${method} ${path} got no answer in ${formatLimit(timeoutMs)}`, null);
+      }
       let parsed;
       try {
         parsed = JSON.parse(text);

package/src/lib/poll-liveness.js

@@ -0,0 +1,60 @@
+/**
+ * Poll liveness for the inbox loops (TASK-338).
+ *
+ * The hard timeout in api.js bounds every CAP request, so no single request can
+ * hang for good any more. This is the second net, for a loop that stops making
+ * progress anyway: if no poll has settled within the limit while the loop was
+ * doing its own work, it is wedged, and the only thing that heals it is a fresh
+ * process. Measured on 10-31: nib's and folio's loops sat 8+ hours on one
+ * unanswered request, alive the whole time, and their keepalives (tmux, launchd
+ * KeepAlive) respawn only on exit. So a stall ends the process, loudly, and the
+ * keepalive does the rest.
+ *
+ * The clock runs while the loop is polling, acking or holding work, and is
+ * paused while a model turn runs and while the loop waits on its own scheduled
+ * retry. A two-hour turn never trips it, and neither does a fifteen-minute
+ * backoff: those always end on their own. A settled poll or a finished turn
+ * restarts the clock.
+ *
+ * Monotonic time, not Date.now: a laptop seat that sleeps, or a clock corrected
+ * by ten minutes, must not read as a stall.
+ */
+import { performance } from 'perf_hooks';
+
+export const POLL_LIVENESS_DEFAULT_MS = 10 * 60 * 1000;
+
+export const resolvePollLivenessMs = (value, env = process.env) => {
+  const candidate = [value, Number(env.COMMONLY_POLL_LIVENESS_MS)]
+    .find((ms) => Number.isFinite(ms) && ms > 0);
+  return candidate ?? POLL_LIVENESS_DEFAULT_MS;
+};
+
+export const formatStall = (ms) => (ms < 60_000
+  ? `${Math.max(1, Math.round(ms / 1000))}s`
+  : `${Math.round(ms / 60_000)} min`);
+
+export const createPollLiveness = ({ limitMs, onStall, checkEveryMs }) => {
+  let armedSince = null;
+  let stalled = false;
+  // A quarter of the limit, so a stall is caught within 1.25x of it, bounded
+  // to a check a second at most and one a minute at least.
+  const every = Number.isFinite(checkEveryMs) && checkEveryMs > 0
+    ? checkEveryMs
+    : Math.min(60_000, Math.max(1000, Math.floor(limitMs / 4)));
+  const timer = setInterval(() => {
+    if (stalled || armedSince === null) return;
+    const stalledMs = performance.now() - armedSince;
+    if (stalledMs < limitMs) return;
+    stalled = true;
+    onStall({ stalledMs, limitMs });
+  }, every);
+  // The check must never be what keeps a stopped loop's process alive.
+  timer.unref?.();
+  return {
+    // (Re)start the clock: the loop is doing its own work, or just made progress.
+    arm: () => { armedSince = performance.now(); },
+    // Stop the clock: a model turn, or a wait the loop scheduled itself.
+    pause: () => { armedSince = null; },
+    stop: () => { clearInterval(timer); },
+  };
+};

package/src/lib/poller.js

@@ -9,6 +9,7 @@
 import { createClient } from './api.js';
 import { formatDigestFrame } from './digest-frame.js';
 import { EMPTY_OUTPUT } from './claim-outcome.js';
+import { createPollLiveness, formatStall, resolvePollLivenessMs } from './poll-liveness.js';
 
 // Delivery failures that must stop a consumer rather than fall through to the
 // normal at-least-once retry path. A stale delivery belongs to a replacement
@@ -39,6 +40,9 @@ export const startPoller = ({
   loginInstance,    // the --instance the connect ran with, so the login hint names it
   onEvent,          // async (event) => { outcome, content? }
   onError,          // (err) => void
+  pollLivenessMs,   // TASK-338: see lib/poll-liveness.js
+  pollLivenessCheckMs,
+  onPollStalled,    // ({ stalledMs }) => void; the connect command exits on it
 }) => {
   const client = createClient({ instance: instanceUrl, token });
   let running = true;
@@ -49,16 +53,35 @@ export const startPoller = ({
   // reprovision-all; 5+ wastes rate-limit budget after the real-revoke case.
   let consecutiveAuthErrors = 0;
   const MAX_AUTH_ERRORS = 3;
+  let pollStalled = false;
+  const pollLiveness = createPollLiveness({
+    limitMs: resolvePollLivenessMs(pollLivenessMs),
+    checkEveryMs: pollLivenessCheckMs,
+    onStall: ({ stalledMs }) => {
+      if (!running) return;
+      pollStalled = true;
+      running = false;
+      pollLiveness.stop();
+      onError?.(new Error(`poll stalled: no inbox poll settled for ${formatStall(stalledMs)} — stopping poller`));
+      onPollStalled?.({ stalledMs });
+    },
+  });
 
   const poll = async () => {
     if (!running) return;
 
     try {
-      const { events = [] } = await client.get('/api/agents/runtime/events', {
+      // Same clock as `agent run`: running from here to the scheduled wait,
+      // paused while the local handler takes the event.
+      pollLiveness.arm();
+      const page = await client.get('/api/agents/runtime/events', {
         agentName,
         instanceId,
         limit: 10,
       });
+      pollLiveness.arm();
+      if (pollStalled) return;
+      const { events = [] } = page;
 
       // A digest is received context, not a webhook invocation. The runtime
       // inbox claims it only alongside a wake from the same pod.
@@ -83,9 +106,12 @@ export const startPoller = ({
         let result = { outcome: 'no_action' };
         try {
           // A handler that returns nothing said nothing: not a read of the digest.
+          pollLiveness.pause();
           result = (await onEvent(forwarded)) || { outcome: 'no_action', reason: EMPTY_OUTPUT };
         } catch (handlerErr) {
           result = { outcome: 'error', reason: handlerErr.message };
+        } finally {
+          pollLiveness.arm();
         }
 
         // Acknowledge the event
@@ -150,11 +176,14 @@ export const startPoller = ({
       }
       consecutiveErrors++;
       onError?.(err);
-      // Back off on repeated errors (max 60s)
+      // Back off on repeated errors (max 60s). A wait this loop scheduled
+      // itself always ends, so the liveness clock stops for it.
       const backoff = Math.min(intervalMs * consecutiveErrors, 60_000);
+      pollLiveness.pause();
       await new Promise((r) => setTimeout(r, backoff));
     }
 
+    pollLiveness.pause();
     if (running) setTimeout(poll, intervalMs);
   };
 
@@ -162,6 +191,9 @@ export const startPoller = ({
   poll();
 
   return {
-    stop: () => { running = false; },
+    stop: () => {
+      running = false;
+      pollLiveness.stop();
+    },
   };
 };