@commonlyai/cli 0.1.218 → 0.1.220

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@commonlyai/cli",
3
- "version": "0.1.218",
3
+ "version": "0.1.220",
4
4
  "license": "Apache-2.0",
5
5
  "description": "The Commonly CLI — connect agents, manage pods, iterate fast",
6
6
  "type": "module",
@@ -55,6 +55,7 @@ import {
55
55
  import { catchUpBeforeTurn } from '../lib/pod-catch-up.js';
56
56
  import { readLongTerm, syncBack } from '../lib/memory-bridge.js';
57
57
  import { pollRetryPolicy } from '../lib/poll-retry.js';
58
+ import { createPollLiveness, formatStall, resolvePollLivenessMs } from '../lib/poll-liveness.js';
58
59
  import { detectMemorySources, composeImport, importMemory } from '../lib/memory-import.js';
59
60
  import { detectSkills, importSkills, parseSkillMeta } from '../lib/skills-import.js';
60
61
  import { compileServerSkills } from '../lib/server-skills.js';
@@ -1561,6 +1562,14 @@ export const performRun = ({
1561
1562
  // ~/.commonly/state/turns/<agent>.jsonl once delivery settles. Off, the spawn
1562
1563
  // path is the plain adapter call it has always been, and nothing is written.
1563
1564
  superviseRuns,
1565
+ // TASK-338. No poll settling for this long, outside a model turn and outside
1566
+ // a scheduled wait, means the loop is wedged: it stops, says so through
1567
+ // onError, and calls onPollStalled, which the `agent run` command turns into a
1568
+ // non-zero exit for the keepalive to see. Undefined reads
1569
+ // COMMONLY_POLL_LIVENESS_MS, then the ten-minute default (lib/poll-liveness.js).
1570
+ pollLivenessMs,
1571
+ pollLivenessCheckMs,
1572
+ onPollStalled,
1564
1573
  }) => {
1565
1574
  const supervising = superviseRuns ?? superviseRunsByDefault(adapter?.name);
1566
1575
  reportRuntime = reportRuntime && WALKED_HARNESSES.includes(adapter?.name);
@@ -1664,6 +1673,26 @@ export const performRun = ({
1664
1673
  const MAX_AUTH_ERRORS = 3;
1665
1674
  let consecutivePollFailures = 0;
1666
1675
  let consecutiveSpawnFailures = 0;
1676
+ let pollStalled = false;
1677
+ const pollLiveness = createPollLiveness({
1678
+ limitMs: resolvePollLivenessMs(pollLivenessMs),
1679
+ checkEveryMs: pollLivenessCheckMs,
1680
+ onStall: ({ stalledMs }) => {
1681
+ // A loop that already stopped itself (auth, a superseded delivery) is
1682
+ // draining, not wedged.
1683
+ if (!running) return;
1684
+ pollStalled = true;
1685
+ running = false;
1686
+ pollLiveness.stop();
1687
+ // The same teardown as stop(): a held retry must not keep renewing its
1688
+ // lease on a loop that will never deliver it.
1689
+ clearHeldRenewal();
1690
+ const line = `poll stalled: no inbox poll settled for ${formatStall(stalledMs)} `
1691
+ + '— exiting so a clean process can take over';
1692
+ if (onError) onError(new Error(line)); else log(line);
1693
+ onPollStalled?.({ stalledMs });
1694
+ },
1695
+ });
1667
1696
  const spawnJitterRatio = retryJitterRatio ?? spawnRetryJitter(agentName);
1668
1697
  // Per-seat cascade state — lives with the process, like the session store.
1669
1698
  // A wrapper restart forgets the streak; the decay window covers that gap.
@@ -3168,6 +3197,13 @@ export const performRun = ({
3168
3197
  }
3169
3198
  };
3170
3199
 
3200
+ // A model turn is not a stall, however long it runs: the liveness clock
3201
+ // stops for it and restarts when it ends.
3202
+ const duringTurn = async (turn) => {
3203
+ pollLiveness.pause();
3204
+ try { return await turn(); } finally { pollLiveness.arm(); }
3205
+ };
3206
+
3171
3207
  const tick = async () => {
3172
3208
  if (!running) return;
3173
3209
  let nextPollDelayMs = intervalMs;
@@ -3181,10 +3217,19 @@ export const performRun = ({
3181
3217
  // do not reintroduce a mixed-pod `limit: 10` here.
3182
3218
  // TASK-317: name this CLI for the chat chip. The server keeps it as a
3183
3219
  // display-only label (runtime.reportedCli) and never as a binding.
3184
- const { events = [], inboxCount } = await client.get('/api/agents/runtime/events', {
3220
+ // The liveness clock runs from here until the next scheduled wait, paused
3221
+ // only while a model turn runs (lib/poll-liveness.js).
3222
+ pollLiveness.arm();
3223
+ const page = await client.get('/api/agents/runtime/events', {
3185
3224
  agentName, instanceId, limit: batchLimit,
3186
3225
  ...(adapter?.name ? { reportedCli: adapter.name } : {}),
3187
3226
  });
3227
+ pollLiveness.arm();
3228
+ // A stall fired while this request hung: the loop has already given up on
3229
+ // it. A plain stop() falls through instead, to the group loop's own
3230
+ // `running` check, which leaves the page's rows to requeue as it always has.
3231
+ if (pollStalled) return;
3232
+ const { events = [], inboxCount } = page;
3188
3233
  consecutiveAuthErrors = 0;
3189
3234
  consecutivePollFailures = 0;
3190
3235
  // A deployed CLI can briefly run against an older server that still
@@ -3298,10 +3343,10 @@ export const performRun = ({
3298
3343
  let batch;
3299
3344
  try {
3300
3345
  // The person's item leads the prompt; the greeting rides along.
3301
- batch = await processEventBatch(
3346
+ batch = await duringTurn(() => processEventBatch(
3302
3347
  [...group].sort((a, b) => inboxPriority(a) - inboxPriority(b)),
3303
3348
  pageInboxCount,
3304
- );
3349
+ ));
3305
3350
  } catch (err) {
3306
3351
  await scheduleBatchRetry(err, group);
3307
3352
  break;
@@ -3358,7 +3403,7 @@ export const performRun = ({
3358
3403
  } else {
3359
3404
  const eventWillSpawn = Boolean(extractPrompt(event) && (event.podId || podId));
3360
3405
  try {
3361
- result = await processEvent(event);
3406
+ result = await duringTurn(() => processEvent(event));
3362
3407
  } catch (err) {
3363
3408
  if (isContentRefusal(err)) {
3364
3409
  // A refused private item is acked without a pod notice: it has no
@@ -3506,6 +3551,8 @@ export const performRun = ({
3506
3551
  // before the next inbox read, independent of the idle polling cadence. A
3507
3552
  // retry scheduled in the same tick keeps its own delay: the failure report
3508
3553
  // names it, and a sibling's completed turn must not shorten the backoff.
3554
+ // A wait the loop schedules itself always ends, so it is not a stall.
3555
+ pollLiveness.pause();
3509
3556
  if (running) setTimeoutImpl(tick, completedModelTurn && !retryScheduled ? 3000 : nextPollDelayMs);
3510
3557
  };
3511
3558
 
@@ -3513,6 +3560,7 @@ export const performRun = ({
3513
3560
  return {
3514
3561
  stop: () => {
3515
3562
  running = false;
3563
+ pollLiveness.stop();
3516
3564
  clearHeldRenewal();
3517
3565
  },
3518
3566
  };
@@ -3897,6 +3945,8 @@ Docs:
3897
3945
  onError: (err) => {
3898
3946
  console.error(`[poll error] ${err.message}`);
3899
3947
  },
3948
+ // TASK-338: a wedged poll ends the process instead of idling forever.
3949
+ onPollStalled: () => process.exit(1),
3900
3950
  });
3901
3951
 
3902
3952
  // Graceful shutdown
@@ -4269,6 +4319,9 @@ Docs:
4269
4319
  // collapsed the duplication. The conclusion survived the premise; the
4270
4320
  // sentence did not, and nothing in the diff pointed at it.)
4271
4321
  onError: (err) => console.error(`${stamp()} [${name}] ${err.message}`),
4322
+ // TASK-338: a wedged poll ends the process, so tmux-keepalive or
4323
+ // launchd KeepAlive, which act only on exit, start a clean one.
4324
+ onPollStalled: () => process.exit(1),
4272
4325
  });
4273
4326
 
4274
4327
  // SIGTERM is the daemon's idle-boundary handoff signal. `stop()` stops
@@ -8,6 +8,7 @@
8
8
 
9
9
  import { createClient } from '../lib/api.js';
10
10
  import { getToken, resolveInstanceUrl } from '../lib/config.js';
11
+ import { createTailPoller } from '../lib/pod-tail.js';
11
12
 
12
13
  export const registerPod = (program) => {
13
14
  const pod = program.command('pod').description('Manage pods');
@@ -96,28 +97,16 @@ Commonly instance (see: commonly login --help).
96
97
 
97
98
  const client = createClient({ instance: resolveInstanceUrl(opts.instance), token });
98
99
 
99
- let lastId = null;
100
100
  console.log(`Watching pod ${podId}... (Ctrl+C to stop)\n`);
101
101
 
102
+ const pollNew = createTailPoller({
103
+ fetchPage: (limit) => client.get(`/api/messages/${podId}`, { limit }),
104
+ filter: opts.filter,
105
+ print: (line) => console.log(line),
106
+ });
102
107
  const poll = async () => {
103
108
  try {
104
- const data = await client.get(`/api/messages/${podId}`, {
105
- limit: 20,
106
- after: lastId,
107
- });
108
- const messages = Array.isArray(data) ? data : data.messages || [];
109
-
110
- messages.forEach((m) => {
111
- const isBot = m.isBot || m.sender?.isBot;
112
- if (opts.filter === 'agents' && !isBot) return;
113
- if (opts.filter === 'humans' && isBot) return;
114
-
115
- const ts = new Date(m.createdAt || m.timestamp).toTimeString().slice(0, 8);
116
- const who = m.username || m.sender?.username || '?';
117
- const marker = isBot ? '🤖' : '👤';
118
- console.log(`[${ts}] ${marker} ${who}: ${m.content}`);
119
- lastId = m._id || m.id;
120
- });
109
+ await pollNew();
121
110
  } catch (err) {
122
111
  console.error(`[poll error] ${err.message}`);
123
112
  }
package/src/lib/api.js CHANGED
@@ -56,7 +56,82 @@ const handleResponse = async (res, session = null) => {
56
56
  return body;
57
57
  };
58
58
 
59
- export const createClient = ({ instance = null, token = undefined } = {}) => {
59
+ // TASK-338: every CAP request carries a hard timeout. Without one, a request
60
+ // that a proxy or the network accepts and never answers leaves its promise
61
+ // pending for good, and the run loop awaiting it is alive but wedged: nib and
62
+ // folio each sat 8+ hours on one unanswered poll on 10-31, and their keepalives,
63
+ // which respawn only on exit, could not see it. The poll-retry backoff
64
+ // (TASK-025) only handles a fetch that throws, so a timeout throws: it is an
65
+ // ordinary failed request, retried with backoff, and never an ack.
66
+ //
67
+ // Reads get the base timeout; writes and uploads carry a body and a server-side
68
+ // write, so they get a multiple of it. COMMONLY_CAP_TIMEOUT_MS sets the base.
69
+ export const CAP_TIMEOUT_DEFAULT_MS = 30_000;
70
+ const TIMEOUT_SCALE = { read: 1, write: 2, upload: 4 };
71
+
72
+ export const capTimeoutMs = (kind, { baseMs, env = process.env } = {}) => {
73
+ const fromEnv = Number(env.COMMONLY_CAP_TIMEOUT_MS);
74
+ const base = [baseMs, fromEnv].find((value) => Number.isFinite(value) && value > 0)
75
+ ?? CAP_TIMEOUT_DEFAULT_MS;
76
+ return Math.min(base * (TIMEOUT_SCALE[kind] || 1), 2_147_483_647);
77
+ };
78
+
79
+ // A caller's own signal (doctor, and the 5s identity and skills reads in the
80
+ // run loop) still applies; the hard timeout is added beside it, never instead.
81
+ // AbortSignal.any is Node 20.3+, and `engines` allows 20.0, so the fallback
82
+ // joins the two by hand. Either way the abort carries the source's own reason,
83
+ // which is what nameTimeout matches on.
84
+ // The fallback's listeners are removed when the request settles, so a caller
85
+ // that reuses one long-lived signal does not collect one per request.
86
+ const capSignal = (capTimeout, callerSignal) => {
87
+ if (!callerSignal) return { signal: capTimeout, release: () => {} };
88
+ if (typeof AbortSignal.any === 'function') {
89
+ return { signal: AbortSignal.any([callerSignal, capTimeout]), release: () => {} };
90
+ }
91
+ const joined = new AbortController();
92
+ const listeners = [];
93
+ for (const source of [callerSignal, capTimeout]) {
94
+ if (source.aborted) {
95
+ joined.abort(source.reason);
96
+ break;
97
+ }
98
+ const onAbort = () => joined.abort(source.reason);
99
+ source.addEventListener('abort', onAbort, { once: true });
100
+ listeners.push([source, onAbort]);
101
+ }
102
+ return {
103
+ signal: joined.signal,
104
+ release: () => listeners.forEach(([source, onAbort]) => source.removeEventListener('abort', onAbort)),
105
+ };
106
+ };
107
+
108
+ export const formatLimit = (ms) => (ms < 1000 ? `${ms}ms` : `${Math.round(ms / 1000)}s`);
109
+
110
+ // The cap's timeout rejects with a DOMException named TimeoutError, from the
111
+ // fetch or from the body read. Name the request and the limit instead, so the
112
+ // log line reads as an unanswered request rather than "operation was aborted".
113
+ // Only the cap's own timeout is renamed: a caller's abort or its own shorter
114
+ // timeout passes through as it was thrown.
115
+ const nameTimeout = (capTimeout, method, path, ms) => (err) => {
116
+ // By identity: the caller's timeout can fire first and the cap's a moment
117
+ // later, and then `capTimeout.aborted` alone would claim the caller's error.
118
+ if (err?.name !== 'TimeoutError' || err !== capTimeout.reason) throw err;
119
+ throw Object.assign(
120
+ new Error(`${method} ${path} got no answer in ${formatLimit(ms)}`),
121
+ { code: 'cap_timeout', timeoutMs: ms },
122
+ );
123
+ };
124
+
125
+ const timedFetch = (url, init, { method, path, ms, session }) => {
126
+ const capTimeout = AbortSignal.timeout(ms);
127
+ const { signal, release } = capSignal(capTimeout, init.signal);
128
+ return fetch(url, { ...init, signal })
129
+ .then((res) => handleResponse(res, session))
130
+ .catch(nameTimeout(capTimeout, method, path, ms))
131
+ .finally(release);
132
+ };
133
+
134
+ export const createClient = ({ instance = null, token = undefined, timeoutMs = undefined } = {}) => {
60
135
  const baseUrl = resolveInstanceUrl(instance);
61
136
  const resolved = resolveInstance(instance);
62
137
  const authToken = token === undefined ? getToken(instance) : token;
@@ -64,30 +139,33 @@ export const createClient = ({ instance = null, token = undefined } = {}) => {
64
139
  instanceKey: resolved?.key || (typeof instance === 'string' && instance && !/^https?:\/\//i.test(instance) ? instance : 'default'),
65
140
  baseUrl,
66
141
  };
142
+ const limit = (kind) => capTimeoutMs(kind, { baseMs: timeoutMs });
67
143
 
68
144
  const get = (path, params = {}, options = {}) => {
69
145
  const url = new URL(`${baseUrl}${path}`);
70
146
  Object.entries(params).forEach(([k, v]) => v != null && url.searchParams.set(k, v));
71
- return fetch(url.toString(), { headers: headers(authToken), ...options }).then((res) => handleResponse(res, session));
147
+ return timedFetch(url.toString(), { headers: headers(authToken), ...options }, {
148
+ method: 'GET', path, ms: limit('read'), session,
149
+ });
72
150
  };
73
151
 
74
- const post = (path, body = {}) => fetch(`${baseUrl}${path}`, {
152
+ const post = (path, body = {}) => timedFetch(`${baseUrl}${path}`, {
75
153
  method: 'POST',
76
154
  headers: headers(authToken),
77
155
  body: JSON.stringify(body),
78
- }).then((res) => handleResponse(res, session));
156
+ }, { method: 'POST', path, ms: limit('write'), session });
79
157
 
80
- const patch = (path, body = {}) => fetch(`${baseUrl}${path}`, {
158
+ const patch = (path, body = {}) => timedFetch(`${baseUrl}${path}`, {
81
159
  method: 'PATCH',
82
160
  headers: headers(authToken),
83
161
  body: JSON.stringify(body),
84
- }).then((res) => handleResponse(res, session));
162
+ }, { method: 'PATCH', path, ms: limit('write'), session });
85
163
 
86
- const del = (path, body) => fetch(`${baseUrl}${path}`, {
164
+ const del = (path, body) => timedFetch(`${baseUrl}${path}`, {
87
165
  method: 'DELETE',
88
166
  headers: headers(authToken),
89
167
  ...(body === undefined ? {} : { body: JSON.stringify(body) }),
90
- }).then((res) => handleResponse(res, session));
168
+ }, { method: 'DELETE', path, ms: limit('write'), session });
91
169
 
92
170
  // Multipart upload via native FormData/Blob (Node 18+) — no runtime deps.
93
171
  // Content-Type is deliberately NOT set: fetch writes the multipart boundary.
@@ -103,11 +181,11 @@ export const createClient = ({ instance = null, token = undefined } = {}) => {
103
181
  for (const [k, v] of Object.entries(fields)) {
104
182
  if (v !== undefined && v !== null) form.append(k, String(v));
105
183
  }
106
- return fetch(`${baseUrl}${path}`, {
184
+ return timedFetch(`${baseUrl}${path}`, {
107
185
  method: 'POST',
108
186
  headers: authToken ? { Authorization: `Bearer ${authToken}` } : {},
109
187
  body: form,
110
- }).then((res) => handleResponse(res, session));
188
+ }, { method: 'POST', path, ms: limit('upload'), session });
111
189
  };
112
190
 
113
191
  return {
@@ -116,11 +194,11 @@ export const createClient = ({ instance = null, token = undefined } = {}) => {
116
194
  };
117
195
 
118
196
  // Convenience: login doesn't need a token
119
- export const login = async (instanceUrl, email, password) => {
120
- const res = await fetch(`${instanceUrl}/api/auth/login`, {
197
+ export const login = (instanceUrl, email, password) => {
198
+ const ms = capTimeoutMs('write');
199
+ return timedFetch(`${instanceUrl}/api/auth/login`, {
121
200
  method: 'POST',
122
201
  headers: { 'Content-Type': 'application/json' },
123
202
  body: JSON.stringify({ email, password }),
124
- });
125
- return handleResponse(res);
203
+ }, { method: 'POST', path: '/api/auth/login', ms, session: null });
126
204
  };
package/src/lib/inbox.js CHANGED
@@ -62,6 +62,7 @@
62
62
 
63
63
  import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'fs';
64
64
  import { dirname } from 'path';
65
+ import { capTimeoutMs, formatLimit } from './api.js';
65
66
 
66
67
  export const INBOX_KINDS = ['mention', 'decision', 'handoff', 'approval'];
67
68
  export const TOKEN_FILE_ENV = 'COMMONLY_TOKEN_FILE';
@@ -394,19 +395,29 @@ export const createInboxRequest = ({
394
395
  for (const [key, value] of Object.entries(params)) {
395
396
  if (value !== undefined && value !== null) url.searchParams.set(key, String(value));
396
397
  }
398
+ // TASK-338: the same hard timeout as every other CAP request (api.js).
399
+ const timeoutMs = capTimeoutMs(method === 'GET' ? 'read' : 'write');
397
400
  for (let attempt = 0; ; attempt += 1) {
398
- // eslint-disable-next-line no-await-in-loop
399
- const res = await fetchImpl(url.toString(), {
400
- method,
401
- headers: {
402
- Authorization: `Bearer ${token}`,
403
- 'Content-Type': 'application/json',
404
- 'User-Agent': 'commonly-cli/0.1',
405
- },
406
- ...(body === undefined ? {} : { body: JSON.stringify(body) }),
407
- });
408
- // eslint-disable-next-line no-await-in-loop
409
- const text = await res.text();
401
+ let res;
402
+ let text;
403
+ try {
404
+ // eslint-disable-next-line no-await-in-loop
405
+ res = await fetchImpl(url.toString(), {
406
+ method,
407
+ headers: {
408
+ Authorization: `Bearer ${token}`,
409
+ 'Content-Type': 'application/json',
410
+ 'User-Agent': 'commonly-cli/0.1',
411
+ },
412
+ ...(body === undefined ? {} : { body: JSON.stringify(body) }),
413
+ signal: AbortSignal.timeout(timeoutMs),
414
+ });
415
+ // eslint-disable-next-line no-await-in-loop
416
+ text = await res.text();
417
+ } catch (err) {
418
+ if (err?.name !== 'TimeoutError') throw err;
419
+ throw new InboxRequestError(`${method} ${path} got no answer in ${formatLimit(timeoutMs)}`, null);
420
+ }
410
421
  let parsed;
411
422
  try {
412
423
  parsed = JSON.parse(text);
@@ -0,0 +1,68 @@
1
+ /**
2
+ * Live pod tail for `commonly pod tail`.
3
+ *
4
+ * GET /api/messages/:podId takes `before` and `limit` only; it ignores `after`
5
+ * and returns the newest `limit` rows. Polling it with a cursor therefore
6
+ * re-serves the same window every tick. The poller keeps the ids it has
7
+ * already printed and prints only the rest, in the order the page returns
8
+ * them (oldest first).
9
+ */
10
+
11
+ // Ids kept for de-duplication. Old entries fall out first, so a long tail
12
+ // stays bounded in memory.
13
+ export const TAIL_SEEN_MAX = 1000;
14
+ export const TAIL_PAGE_LIMIT = 100;
15
+
16
+ // Same bot field the v2 chat reads (frontend/src/v2/utils/threadView.ts
17
+ // participantOf): `user.isBot`, falling back to the mapped `userId.isBot`.
18
+ export const isAgentMessage = (m) => {
19
+ if (typeof m?.user?.isBot === 'boolean') return m.user.isBot;
20
+ return m?.userId?.isBot === true;
21
+ };
22
+
23
+ const authorName = (m) => m?.user?.username || m?.userId?.username || m?.username || '?';
24
+
25
+ const clock = (m) => {
26
+ const raw = m?.createdAt || m?.timestamp;
27
+ const date = raw ? new Date(raw) : null;
28
+ return date && !Number.isNaN(date.getTime()) ? date.toTimeString().slice(0, 8) : '--:--:--';
29
+ };
30
+
31
+ export const formatTailLine = (m) => {
32
+ const marker = isAgentMessage(m) ? '🤖' : '👤';
33
+ return `[${clock(m)}] ${marker} ${authorName(m)}: ${m?.content ?? ''}`;
34
+ };
35
+
36
+ /**
37
+ * @param {object} opts
38
+ * @param {(limit: number) => Promise<object[] | {messages: object[]}>} opts.fetchPage
39
+ * @param {'all'|'agents'|'humans'} [opts.filter='all']
40
+ * @param {(line: string) => void} opts.print
41
+ */
42
+ export const createTailPoller = ({ fetchPage, filter = 'all', print }) => {
43
+ const seen = new Set();
44
+ const remember = (id) => {
45
+ seen.add(id);
46
+ if (seen.size > TAIL_SEEN_MAX) seen.delete(seen.values().next().value);
47
+ };
48
+
49
+ return async () => {
50
+ const data = await fetchPage(TAIL_PAGE_LIMIT);
51
+ const messages = Array.isArray(data) ? data : data?.messages || [];
52
+ let printed = 0;
53
+
54
+ messages.forEach((m) => {
55
+ const id = String(m?._id || m?.id || '');
56
+ if (id) {
57
+ if (seen.has(id)) return;
58
+ remember(id);
59
+ }
60
+ if (filter === 'agents' && !isAgentMessage(m)) return;
61
+ if (filter === 'humans' && isAgentMessage(m)) return;
62
+ print(formatTailLine(m));
63
+ printed += 1;
64
+ });
65
+
66
+ return printed;
67
+ };
68
+ };
@@ -0,0 +1,60 @@
1
+ /**
2
+ * Poll liveness for the inbox loops (TASK-338).
3
+ *
4
+ * The hard timeout in api.js bounds every CAP request, so no single request can
5
+ * hang for good any more. This is the second net, for a loop that stops making
6
+ * progress anyway: if no poll has settled within the limit while the loop was
7
+ * doing its own work, it is wedged, and the only thing that heals it is a fresh
8
+ * process. Measured on 10-31: nib's and folio's loops sat 8+ hours on one
9
+ * unanswered request, alive the whole time, and their keepalives (tmux, launchd
10
+ * KeepAlive) respawn only on exit. So a stall ends the process, loudly, and the
11
+ * keepalive does the rest.
12
+ *
13
+ * The clock runs while the loop is polling, acking or holding work, and is
14
+ * paused while a model turn runs and while the loop waits on its own scheduled
15
+ * retry. A two-hour turn never trips it, and neither does a fifteen-minute
16
+ * backoff: those always end on their own. A settled poll or a finished turn
17
+ * restarts the clock.
18
+ *
19
+ * Monotonic time, not Date.now: a laptop seat that sleeps, or a clock corrected
20
+ * by ten minutes, must not read as a stall.
21
+ */
22
+ import { performance } from 'perf_hooks';
23
+
24
+ export const POLL_LIVENESS_DEFAULT_MS = 10 * 60 * 1000;
25
+
26
+ export const resolvePollLivenessMs = (value, env = process.env) => {
27
+ const candidate = [value, Number(env.COMMONLY_POLL_LIVENESS_MS)]
28
+ .find((ms) => Number.isFinite(ms) && ms > 0);
29
+ return candidate ?? POLL_LIVENESS_DEFAULT_MS;
30
+ };
31
+
32
+ export const formatStall = (ms) => (ms < 60_000
33
+ ? `${Math.max(1, Math.round(ms / 1000))}s`
34
+ : `${Math.round(ms / 60_000)} min`);
35
+
36
+ export const createPollLiveness = ({ limitMs, onStall, checkEveryMs }) => {
37
+ let armedSince = null;
38
+ let stalled = false;
39
+ // A quarter of the limit, so a stall is caught within 1.25x of it, bounded
40
+ // to a check a second at most and one a minute at least.
41
+ const every = Number.isFinite(checkEveryMs) && checkEveryMs > 0
42
+ ? checkEveryMs
43
+ : Math.min(60_000, Math.max(1000, Math.floor(limitMs / 4)));
44
+ const timer = setInterval(() => {
45
+ if (stalled || armedSince === null) return;
46
+ const stalledMs = performance.now() - armedSince;
47
+ if (stalledMs < limitMs) return;
48
+ stalled = true;
49
+ onStall({ stalledMs, limitMs });
50
+ }, every);
51
+ // The check must never be what keeps a stopped loop's process alive.
52
+ timer.unref?.();
53
+ return {
54
+ // (Re)start the clock: the loop is doing its own work, or just made progress.
55
+ arm: () => { armedSince = performance.now(); },
56
+ // Stop the clock: a model turn, or a wait the loop scheduled itself.
57
+ pause: () => { armedSince = null; },
58
+ stop: () => { clearInterval(timer); },
59
+ };
60
+ };
package/src/lib/poller.js CHANGED
@@ -9,6 +9,7 @@
9
9
  import { createClient } from './api.js';
10
10
  import { formatDigestFrame } from './digest-frame.js';
11
11
  import { EMPTY_OUTPUT } from './claim-outcome.js';
12
+ import { createPollLiveness, formatStall, resolvePollLivenessMs } from './poll-liveness.js';
12
13
 
13
14
  // Delivery failures that must stop a consumer rather than fall through to the
14
15
  // normal at-least-once retry path. A stale delivery belongs to a replacement
@@ -39,6 +40,9 @@ export const startPoller = ({
39
40
  loginInstance, // the --instance the connect ran with, so the login hint names it
40
41
  onEvent, // async (event) => { outcome, content? }
41
42
  onError, // (err) => void
43
+ pollLivenessMs, // TASK-338: see lib/poll-liveness.js
44
+ pollLivenessCheckMs,
45
+ onPollStalled, // ({ stalledMs }) => void; the connect command exits on it
42
46
  }) => {
43
47
  const client = createClient({ instance: instanceUrl, token });
44
48
  let running = true;
@@ -49,16 +53,35 @@ export const startPoller = ({
49
53
  // reprovision-all; 5+ wastes rate-limit budget after the real-revoke case.
50
54
  let consecutiveAuthErrors = 0;
51
55
  const MAX_AUTH_ERRORS = 3;
56
+ let pollStalled = false;
57
+ const pollLiveness = createPollLiveness({
58
+ limitMs: resolvePollLivenessMs(pollLivenessMs),
59
+ checkEveryMs: pollLivenessCheckMs,
60
+ onStall: ({ stalledMs }) => {
61
+ if (!running) return;
62
+ pollStalled = true;
63
+ running = false;
64
+ pollLiveness.stop();
65
+ onError?.(new Error(`poll stalled: no inbox poll settled for ${formatStall(stalledMs)} — stopping poller`));
66
+ onPollStalled?.({ stalledMs });
67
+ },
68
+ });
52
69
 
53
70
  const poll = async () => {
54
71
  if (!running) return;
55
72
 
56
73
  try {
57
- const { events = [] } = await client.get('/api/agents/runtime/events', {
74
+ // Same clock as `agent run`: running from here to the scheduled wait,
75
+ // paused while the local handler takes the event.
76
+ pollLiveness.arm();
77
+ const page = await client.get('/api/agents/runtime/events', {
58
78
  agentName,
59
79
  instanceId,
60
80
  limit: 10,
61
81
  });
82
+ pollLiveness.arm();
83
+ if (pollStalled) return;
84
+ const { events = [] } = page;
62
85
 
63
86
  // A digest is received context, not a webhook invocation. The runtime
64
87
  // inbox claims it only alongside a wake from the same pod.
@@ -83,9 +106,12 @@ export const startPoller = ({
83
106
  let result = { outcome: 'no_action' };
84
107
  try {
85
108
  // A handler that returns nothing said nothing: not a read of the digest.
109
+ pollLiveness.pause();
86
110
  result = (await onEvent(forwarded)) || { outcome: 'no_action', reason: EMPTY_OUTPUT };
87
111
  } catch (handlerErr) {
88
112
  result = { outcome: 'error', reason: handlerErr.message };
113
+ } finally {
114
+ pollLiveness.arm();
89
115
  }
90
116
 
91
117
  // Acknowledge the event
@@ -150,11 +176,14 @@ export const startPoller = ({
150
176
  }
151
177
  consecutiveErrors++;
152
178
  onError?.(err);
153
- // Back off on repeated errors (max 60s)
179
+ // Back off on repeated errors (max 60s). A wait this loop scheduled
180
+ // itself always ends, so the liveness clock stops for it.
154
181
  const backoff = Math.min(intervalMs * consecutiveErrors, 60_000);
182
+ pollLiveness.pause();
155
183
  await new Promise((r) => setTimeout(r, backoff));
156
184
  }
157
185
 
186
+ pollLiveness.pause();
158
187
  if (running) setTimeout(poll, intervalMs);
159
188
  };
160
189
 
@@ -162,6 +191,9 @@ export const startPoller = ({
162
191
  poll();
163
192
 
164
193
  return {
165
- stop: () => { running = false; },
194
+ stop: () => {
195
+ running = false;
196
+ pollLiveness.stop();
197
+ },
166
198
  };
167
199
  };