@commonlyai/cli 0.1.217 → 0.1.219
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/commands/agent.js +133 -52
- package/src/lib/api.js +92 -14
- package/src/lib/inbox.js +23 -12
- package/src/lib/poll-liveness.js +60 -0
- package/src/lib/poller.js +35 -3
package/package.json
CHANGED
package/src/commands/agent.js
CHANGED
|
@@ -55,6 +55,7 @@ import {
|
|
|
55
55
|
import { catchUpBeforeTurn } from '../lib/pod-catch-up.js';
|
|
56
56
|
import { readLongTerm, syncBack } from '../lib/memory-bridge.js';
|
|
57
57
|
import { pollRetryPolicy } from '../lib/poll-retry.js';
|
|
58
|
+
import { createPollLiveness, formatStall, resolvePollLivenessMs } from '../lib/poll-liveness.js';
|
|
58
59
|
import { detectMemorySources, composeImport, importMemory } from '../lib/memory-import.js';
|
|
59
60
|
import { detectSkills, importSkills, parseSkillMeta } from '../lib/skills-import.js';
|
|
60
61
|
import { compileServerSkills } from '../lib/server-skills.js';
|
|
@@ -502,6 +503,81 @@ export const bootstrapAgentRecordFromEnv = async ({
|
|
|
502
503
|
};
|
|
503
504
|
};
|
|
504
505
|
|
|
506
|
+
/**
|
|
507
|
+
* `commonly agent list`. Without --pod it reads the admin installs route, which
|
|
508
|
+
* only a global admin can call. With --pod it reads the member-gated pod roster,
|
|
509
|
+
* so a pod member lists its own seats. Returns the exit code, never exits itself.
|
|
510
|
+
* Any throw becomes one Failed line and exit 1.
|
|
511
|
+
*/
|
|
512
|
+
export const runAgentList = async (opts = {}, deps = {}) => {
|
|
513
|
+
const { error = console.error } = deps;
|
|
514
|
+
try {
|
|
515
|
+
return await listAgents(opts, deps);
|
|
516
|
+
} catch (err) {
|
|
517
|
+
error(`Failed: ${err.message}`);
|
|
518
|
+
return 1;
|
|
519
|
+
}
|
|
520
|
+
};
|
|
521
|
+
|
|
522
|
+
const listAgents = async (opts, deps) => {
|
|
523
|
+
const {
|
|
524
|
+
clientFactory = createClient,
|
|
525
|
+
tokenFor = getToken,
|
|
526
|
+
instanceUrlFor = resolveInstanceUrl,
|
|
527
|
+
log = console.log,
|
|
528
|
+
error = console.error,
|
|
529
|
+
} = deps;
|
|
530
|
+
const col = (s, w) => String(s ?? '').padEnd(w).slice(0, w);
|
|
531
|
+
|
|
532
|
+
if (opts.local) {
|
|
533
|
+
const agents = listLocalAgents();
|
|
534
|
+
if (agents.length === 0) {
|
|
535
|
+
log('No agents attached locally. Run: commonly agent attach <adapter> --pod <podId> --name <n>');
|
|
536
|
+
return 0;
|
|
537
|
+
}
|
|
538
|
+
log(`${col('NAME', 20)} ${col('ADAPTER', 10)} ${col('POD', 26)} LAST TURN`);
|
|
539
|
+
log('─'.repeat(80));
|
|
540
|
+
agents.forEach((a) => {
|
|
541
|
+
const lastTurn = a.lastTurn ? new Date(a.lastTurn).toLocaleString() : 'never';
|
|
542
|
+
log(`${col(a.name, 20)} ${col(a.adapter, 10)} ${col(a.podId, 26)} ${lastTurn}`);
|
|
543
|
+
});
|
|
544
|
+
return 0;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
const token = tokenFor(opts.instance);
|
|
548
|
+
if (!token) { error('Not logged in. Run: commonly login'); return 1; }
|
|
549
|
+
|
|
550
|
+
const client = clientFactory({ instance: instanceUrlFor(opts.instance), token });
|
|
551
|
+
|
|
552
|
+
try {
|
|
553
|
+
const rows = opts.pod
|
|
554
|
+
? (await client.get(`/api/registry/pods/${encodeURIComponent(opts.pod)}/agents`)).agents || []
|
|
555
|
+
: (await client.get('/api/registry/admin/installations')).installations || [];
|
|
556
|
+
|
|
557
|
+
if (rows.length === 0) {
|
|
558
|
+
log('No agents installed.');
|
|
559
|
+
return 0;
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
log(`${col('NAME', 16)} ${col('INSTANCE', 10)} ${col('RUNTIME', 14)} ${col('HOST', 6)} ${col('STATUS', 10)} LAST SEEN`);
|
|
563
|
+
log('─'.repeat(80));
|
|
564
|
+
rows.forEach((inst) => {
|
|
565
|
+
// The pod roster names the agent `name`; the admin route uses `agentName`.
|
|
566
|
+
const displayName = opts.pod ? (inst.name || inst.agentName) : inst.agentName;
|
|
567
|
+
const runtimeType = inst.config?.runtime?.runtimeType || inst.runtime?.runtimeType || inst.runtimeType || '?';
|
|
568
|
+
const host = inst.config?.runtime?.host || inst.runtime?.host || '';
|
|
569
|
+
const seen = inst.lastActiveAt || inst.usage?.lastUsedAt || inst.connection?.lastSeenAt;
|
|
570
|
+
const lastSeen = seen ? new Date(seen).toLocaleString() : 'never';
|
|
571
|
+
log(`${col(displayName, 16)} ${col(inst.instanceId, 10)} ${col(runtimeType, 14)} ${col(host, 6)} ${col(inst.status, 10)} ${lastSeen}`);
|
|
572
|
+
});
|
|
573
|
+
return 0;
|
|
574
|
+
} catch (err) {
|
|
575
|
+
error(`Failed: ${err.message}`);
|
|
576
|
+
if (err.status === 403 && !opts.pod) error('Listing every install needs a global admin. Pass --pod <podId> to list one pod you belong to.');
|
|
577
|
+
return 1;
|
|
578
|
+
}
|
|
579
|
+
};
|
|
580
|
+
|
|
505
581
|
/**
|
|
506
582
|
* Enumerate every agent attached on this laptop — i.e. every file in
|
|
507
583
|
* ~/.commonly/tokens/*.json. Cross-references the session store for a
|
|
@@ -1486,6 +1562,14 @@ export const performRun = ({
|
|
|
1486
1562
|
// ~/.commonly/state/turns/<agent>.jsonl once delivery settles. Off, the spawn
|
|
1487
1563
|
// path is the plain adapter call it has always been, and nothing is written.
|
|
1488
1564
|
superviseRuns,
|
|
1565
|
+
// TASK-338. No poll settling for this long, outside a model turn and outside
|
|
1566
|
+
// a scheduled wait, means the loop is wedged: it stops, says so through
|
|
1567
|
+
// onError, and calls onPollStalled, which the `agent run` command turns into a
|
|
1568
|
+
// non-zero exit for the keepalive to see. Undefined reads
|
|
1569
|
+
// COMMONLY_POLL_LIVENESS_MS, then the ten-minute default (lib/poll-liveness.js).
|
|
1570
|
+
pollLivenessMs,
|
|
1571
|
+
pollLivenessCheckMs,
|
|
1572
|
+
onPollStalled,
|
|
1489
1573
|
}) => {
|
|
1490
1574
|
const supervising = superviseRuns ?? superviseRunsByDefault(adapter?.name);
|
|
1491
1575
|
reportRuntime = reportRuntime && WALKED_HARNESSES.includes(adapter?.name);
|
|
@@ -1589,6 +1673,26 @@ export const performRun = ({
|
|
|
1589
1673
|
const MAX_AUTH_ERRORS = 3;
|
|
1590
1674
|
let consecutivePollFailures = 0;
|
|
1591
1675
|
let consecutiveSpawnFailures = 0;
|
|
1676
|
+
let pollStalled = false;
|
|
1677
|
+
const pollLiveness = createPollLiveness({
|
|
1678
|
+
limitMs: resolvePollLivenessMs(pollLivenessMs),
|
|
1679
|
+
checkEveryMs: pollLivenessCheckMs,
|
|
1680
|
+
onStall: ({ stalledMs }) => {
|
|
1681
|
+
// A loop that already stopped itself (auth, a superseded delivery) is
|
|
1682
|
+
// draining, not wedged.
|
|
1683
|
+
if (!running) return;
|
|
1684
|
+
pollStalled = true;
|
|
1685
|
+
running = false;
|
|
1686
|
+
pollLiveness.stop();
|
|
1687
|
+
// The same teardown as stop(): a held retry must not keep renewing its
|
|
1688
|
+
// lease on a loop that will never deliver it.
|
|
1689
|
+
clearHeldRenewal();
|
|
1690
|
+
const line = `poll stalled: no inbox poll settled for ${formatStall(stalledMs)} `
|
|
1691
|
+
+ '— exiting so a clean process can take over';
|
|
1692
|
+
if (onError) onError(new Error(line)); else log(line);
|
|
1693
|
+
onPollStalled?.({ stalledMs });
|
|
1694
|
+
},
|
|
1695
|
+
});
|
|
1592
1696
|
const spawnJitterRatio = retryJitterRatio ?? spawnRetryJitter(agentName);
|
|
1593
1697
|
// Per-seat cascade state — lives with the process, like the session store.
|
|
1594
1698
|
// A wrapper restart forgets the streak; the decay window covers that gap.
|
|
@@ -3093,6 +3197,13 @@ export const performRun = ({
|
|
|
3093
3197
|
}
|
|
3094
3198
|
};
|
|
3095
3199
|
|
|
3200
|
+
// A model turn is not a stall, however long it runs: the liveness clock
|
|
3201
|
+
// stops for it and restarts when it ends.
|
|
3202
|
+
const duringTurn = async (turn) => {
|
|
3203
|
+
pollLiveness.pause();
|
|
3204
|
+
try { return await turn(); } finally { pollLiveness.arm(); }
|
|
3205
|
+
};
|
|
3206
|
+
|
|
3096
3207
|
const tick = async () => {
|
|
3097
3208
|
if (!running) return;
|
|
3098
3209
|
let nextPollDelayMs = intervalMs;
|
|
@@ -3106,10 +3217,19 @@ export const performRun = ({
|
|
|
3106
3217
|
// do not reintroduce a mixed-pod `limit: 10` here.
|
|
3107
3218
|
// TASK-317: name this CLI for the chat chip. The server keeps it as a
|
|
3108
3219
|
// display-only label (runtime.reportedCli) and never as a binding.
|
|
3109
|
-
|
|
3220
|
+
// The liveness clock runs from here until the next scheduled wait, paused
|
|
3221
|
+
// only while a model turn runs (lib/poll-liveness.js).
|
|
3222
|
+
pollLiveness.arm();
|
|
3223
|
+
const page = await client.get('/api/agents/runtime/events', {
|
|
3110
3224
|
agentName, instanceId, limit: batchLimit,
|
|
3111
3225
|
...(adapter?.name ? { reportedCli: adapter.name } : {}),
|
|
3112
3226
|
});
|
|
3227
|
+
pollLiveness.arm();
|
|
3228
|
+
// A stall fired while this request hung: the loop has already given up on
|
|
3229
|
+
// it. A plain stop() falls through instead, to the group loop's own
|
|
3230
|
+
// `running` check, which leaves the page's rows to requeue as it always has.
|
|
3231
|
+
if (pollStalled) return;
|
|
3232
|
+
const { events = [], inboxCount } = page;
|
|
3113
3233
|
consecutiveAuthErrors = 0;
|
|
3114
3234
|
consecutivePollFailures = 0;
|
|
3115
3235
|
// A deployed CLI can briefly run against an older server that still
|
|
@@ -3223,10 +3343,10 @@ export const performRun = ({
|
|
|
3223
3343
|
let batch;
|
|
3224
3344
|
try {
|
|
3225
3345
|
// The person's item leads the prompt; the greeting rides along.
|
|
3226
|
-
batch = await processEventBatch(
|
|
3346
|
+
batch = await duringTurn(() => processEventBatch(
|
|
3227
3347
|
[...group].sort((a, b) => inboxPriority(a) - inboxPriority(b)),
|
|
3228
3348
|
pageInboxCount,
|
|
3229
|
-
);
|
|
3349
|
+
));
|
|
3230
3350
|
} catch (err) {
|
|
3231
3351
|
await scheduleBatchRetry(err, group);
|
|
3232
3352
|
break;
|
|
@@ -3283,7 +3403,7 @@ export const performRun = ({
|
|
|
3283
3403
|
} else {
|
|
3284
3404
|
const eventWillSpawn = Boolean(extractPrompt(event) && (event.podId || podId));
|
|
3285
3405
|
try {
|
|
3286
|
-
result = await processEvent(event);
|
|
3406
|
+
result = await duringTurn(() => processEvent(event));
|
|
3287
3407
|
} catch (err) {
|
|
3288
3408
|
if (isContentRefusal(err)) {
|
|
3289
3409
|
// A refused private item is acked without a pod notice: it has no
|
|
@@ -3431,6 +3551,8 @@ export const performRun = ({
|
|
|
3431
3551
|
// before the next inbox read, independent of the idle polling cadence. A
|
|
3432
3552
|
// retry scheduled in the same tick keeps its own delay: the failure report
|
|
3433
3553
|
// names it, and a sibling's completed turn must not shorten the backoff.
|
|
3554
|
+
// A wait the loop schedules itself always ends, so it is not a stall.
|
|
3555
|
+
pollLiveness.pause();
|
|
3434
3556
|
if (running) setTimeoutImpl(tick, completedModelTurn && !retryScheduled ? 3000 : nextPollDelayMs);
|
|
3435
3557
|
};
|
|
3436
3558
|
|
|
@@ -3438,6 +3560,7 @@ export const performRun = ({
|
|
|
3438
3560
|
return {
|
|
3439
3561
|
stop: () => {
|
|
3440
3562
|
running = false;
|
|
3563
|
+
pollLiveness.stop();
|
|
3441
3564
|
clearHeldRenewal();
|
|
3442
3565
|
},
|
|
3443
3566
|
};
|
|
@@ -3822,6 +3945,8 @@ Docs:
|
|
|
3822
3945
|
onError: (err) => {
|
|
3823
3946
|
console.error(`[poll error] ${err.message}`);
|
|
3824
3947
|
},
|
|
3948
|
+
// TASK-338: a wedged poll ends the process instead of idling forever.
|
|
3949
|
+
onPollStalled: () => process.exit(1),
|
|
3825
3950
|
});
|
|
3826
3951
|
|
|
3827
3952
|
// Graceful shutdown
|
|
@@ -4194,6 +4319,9 @@ Docs:
|
|
|
4194
4319
|
// collapsed the duplication. The conclusion survived the premise; the
|
|
4195
4320
|
// sentence did not, and nothing in the diff pointed at it.)
|
|
4196
4321
|
onError: (err) => console.error(`${stamp()} [${name}] ${err.message}`),
|
|
4322
|
+
// TASK-338: a wedged poll ends the process, so tmux-keepalive or
|
|
4323
|
+
// launchd KeepAlive, which act only on exit, start a clean one.
|
|
4324
|
+
onPollStalled: () => process.exit(1),
|
|
4197
4325
|
});
|
|
4198
4326
|
|
|
4199
4327
|
// SIGTERM is the daemon's idle-boundary handoff signal. `stop()` stops
|
|
@@ -4325,54 +4453,7 @@ The two modes answer different questions:
|
|
|
4325
4453
|
|
|
4326
4454
|
Use --local to find the name you'd pass to 'agent run' or 'agent detach'.
|
|
4327
4455
|
`)
|
|
4328
|
-
.action(async (opts) => {
|
|
4329
|
-
if (opts.local) {
|
|
4330
|
-
const agents = listLocalAgents();
|
|
4331
|
-
if (agents.length === 0) {
|
|
4332
|
-
console.log('No agents attached locally. Run: commonly agent attach <adapter> --pod <podId> --name <n>');
|
|
4333
|
-
return;
|
|
4334
|
-
}
|
|
4335
|
-
const col = (s, w) => String(s ?? '').padEnd(w).slice(0, w);
|
|
4336
|
-
console.log(`${col('NAME', 20)} ${col('ADAPTER', 10)} ${col('POD', 26)} LAST TURN`);
|
|
4337
|
-
console.log('─'.repeat(80));
|
|
4338
|
-
agents.forEach((a) => {
|
|
4339
|
-
const lastTurn = a.lastTurn ? new Date(a.lastTurn).toLocaleString() : 'never';
|
|
4340
|
-
console.log(`${col(a.name, 20)} ${col(a.adapter, 10)} ${col(a.podId, 26)} ${lastTurn}`);
|
|
4341
|
-
});
|
|
4342
|
-
return;
|
|
4343
|
-
}
|
|
4344
|
-
|
|
4345
|
-
const token = getToken(opts.instance);
|
|
4346
|
-
if (!token) { console.error('Not logged in. Run: commonly login'); process.exit(1); }
|
|
4347
|
-
|
|
4348
|
-
const client = createClient({ instance: resolveInstanceUrl(opts.instance), token });
|
|
4349
|
-
|
|
4350
|
-
try {
|
|
4351
|
-
const params = opts.pod ? { podId: opts.pod } : {};
|
|
4352
|
-
const data = await client.get('/api/registry/admin/installations', params);
|
|
4353
|
-
const installations = data.installations || data || [];
|
|
4354
|
-
|
|
4355
|
-
if (installations.length === 0) {
|
|
4356
|
-
console.log('No agents installed.');
|
|
4357
|
-
return;
|
|
4358
|
-
}
|
|
4359
|
-
|
|
4360
|
-
const col = (s, w) => String(s ?? '').padEnd(w).slice(0, w);
|
|
4361
|
-
console.log(`${col('NAME', 16)} ${col('INSTANCE', 10)} ${col('RUNTIME', 14)} ${col('HOST', 6)} ${col('STATUS', 10)} LAST SEEN`);
|
|
4362
|
-
console.log('─'.repeat(80));
|
|
4363
|
-
installations.forEach((inst) => {
|
|
4364
|
-
const runtimeType = inst.config?.runtime?.runtimeType || inst.runtimeType || '?';
|
|
4365
|
-
const host = inst.config?.runtime?.host || '';
|
|
4366
|
-
const lastSeen = inst.usage?.lastUsedAt
|
|
4367
|
-
? new Date(inst.usage.lastUsedAt).toLocaleString()
|
|
4368
|
-
: 'never';
|
|
4369
|
-
console.log(`${col(inst.agentName, 16)} ${col(inst.instanceId, 10)} ${col(runtimeType, 14)} ${col(host, 6)} ${col(inst.status, 10)} ${lastSeen}`);
|
|
4370
|
-
});
|
|
4371
|
-
} catch (err) {
|
|
4372
|
-
console.error(`Failed: ${err.message}`);
|
|
4373
|
-
process.exit(1);
|
|
4374
|
-
}
|
|
4375
|
-
});
|
|
4456
|
+
.action(async (opts) => { process.exit(await runAgentList(opts)); });
|
|
4376
4457
|
|
|
4377
4458
|
// ── logs ──────────────────────────────────────────────────────────────────
|
|
4378
4459
|
// Without --pod, the seat's local daemon log is read. With --pod, the owner
|
package/src/lib/api.js
CHANGED
|
@@ -56,7 +56,82 @@ const handleResponse = async (res, session = null) => {
|
|
|
56
56
|
return body;
|
|
57
57
|
};
|
|
58
58
|
|
|
59
|
-
|
|
59
|
+
// TASK-338: every CAP request carries a hard timeout. Without one, a request
|
|
60
|
+
// that a proxy or the network accepts and never answers leaves its promise
|
|
61
|
+
// pending for good, and the run loop awaiting it is alive but wedged: nib and
|
|
62
|
+
// folio each sat 8+ hours on one unanswered poll on 10-31, and their keepalives,
|
|
63
|
+
// which respawn only on exit, could not see it. The poll-retry backoff
|
|
64
|
+
// (TASK-025) only handles a fetch that throws, so a timeout throws: it is an
|
|
65
|
+
// ordinary failed request, retried with backoff, and never an ack.
|
|
66
|
+
//
|
|
67
|
+
// Reads get the base timeout; writes and uploads carry a body and a server-side
|
|
68
|
+
// write, so they get a multiple of it. COMMONLY_CAP_TIMEOUT_MS sets the base.
|
|
69
|
+
export const CAP_TIMEOUT_DEFAULT_MS = 30_000;
|
|
70
|
+
const TIMEOUT_SCALE = { read: 1, write: 2, upload: 4 };
|
|
71
|
+
|
|
72
|
+
export const capTimeoutMs = (kind, { baseMs, env = process.env } = {}) => {
|
|
73
|
+
const fromEnv = Number(env.COMMONLY_CAP_TIMEOUT_MS);
|
|
74
|
+
const base = [baseMs, fromEnv].find((value) => Number.isFinite(value) && value > 0)
|
|
75
|
+
?? CAP_TIMEOUT_DEFAULT_MS;
|
|
76
|
+
return Math.min(base * (TIMEOUT_SCALE[kind] || 1), 2_147_483_647);
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
// A caller's own signal (doctor, and the 5s identity and skills reads in the
|
|
80
|
+
// run loop) still applies; the hard timeout is added beside it, never instead.
|
|
81
|
+
// AbortSignal.any is Node 20.3+, and `engines` allows 20.0, so the fallback
|
|
82
|
+
// joins the two by hand. Either way the abort carries the source's own reason,
|
|
83
|
+
// which is what nameTimeout matches on.
|
|
84
|
+
// The fallback's listeners are removed when the request settles, so a caller
|
|
85
|
+
// that reuses one long-lived signal does not collect one per request.
|
|
86
|
+
const capSignal = (capTimeout, callerSignal) => {
|
|
87
|
+
if (!callerSignal) return { signal: capTimeout, release: () => {} };
|
|
88
|
+
if (typeof AbortSignal.any === 'function') {
|
|
89
|
+
return { signal: AbortSignal.any([callerSignal, capTimeout]), release: () => {} };
|
|
90
|
+
}
|
|
91
|
+
const joined = new AbortController();
|
|
92
|
+
const listeners = [];
|
|
93
|
+
for (const source of [callerSignal, capTimeout]) {
|
|
94
|
+
if (source.aborted) {
|
|
95
|
+
joined.abort(source.reason);
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
const onAbort = () => joined.abort(source.reason);
|
|
99
|
+
source.addEventListener('abort', onAbort, { once: true });
|
|
100
|
+
listeners.push([source, onAbort]);
|
|
101
|
+
}
|
|
102
|
+
return {
|
|
103
|
+
signal: joined.signal,
|
|
104
|
+
release: () => listeners.forEach(([source, onAbort]) => source.removeEventListener('abort', onAbort)),
|
|
105
|
+
};
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
export const formatLimit = (ms) => (ms < 1000 ? `${ms}ms` : `${Math.round(ms / 1000)}s`);
|
|
109
|
+
|
|
110
|
+
// The cap's timeout rejects with a DOMException named TimeoutError, from the
|
|
111
|
+
// fetch or from the body read. Name the request and the limit instead, so the
|
|
112
|
+
// log line reads as an unanswered request rather than "operation was aborted".
|
|
113
|
+
// Only the cap's own timeout is renamed: a caller's abort or its own shorter
|
|
114
|
+
// timeout passes through as it was thrown.
|
|
115
|
+
const nameTimeout = (capTimeout, method, path, ms) => (err) => {
|
|
116
|
+
// By identity: the caller's timeout can fire first and the cap's a moment
|
|
117
|
+
// later, and then `capTimeout.aborted` alone would claim the caller's error.
|
|
118
|
+
if (err?.name !== 'TimeoutError' || err !== capTimeout.reason) throw err;
|
|
119
|
+
throw Object.assign(
|
|
120
|
+
new Error(`${method} ${path} got no answer in ${formatLimit(ms)}`),
|
|
121
|
+
{ code: 'cap_timeout', timeoutMs: ms },
|
|
122
|
+
);
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
const timedFetch = (url, init, { method, path, ms, session }) => {
|
|
126
|
+
const capTimeout = AbortSignal.timeout(ms);
|
|
127
|
+
const { signal, release } = capSignal(capTimeout, init.signal);
|
|
128
|
+
return fetch(url, { ...init, signal })
|
|
129
|
+
.then((res) => handleResponse(res, session))
|
|
130
|
+
.catch(nameTimeout(capTimeout, method, path, ms))
|
|
131
|
+
.finally(release);
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
export const createClient = ({ instance = null, token = undefined, timeoutMs = undefined } = {}) => {
|
|
60
135
|
const baseUrl = resolveInstanceUrl(instance);
|
|
61
136
|
const resolved = resolveInstance(instance);
|
|
62
137
|
const authToken = token === undefined ? getToken(instance) : token;
|
|
@@ -64,30 +139,33 @@ export const createClient = ({ instance = null, token = undefined } = {}) => {
|
|
|
64
139
|
instanceKey: resolved?.key || (typeof instance === 'string' && instance && !/^https?:\/\//i.test(instance) ? instance : 'default'),
|
|
65
140
|
baseUrl,
|
|
66
141
|
};
|
|
142
|
+
const limit = (kind) => capTimeoutMs(kind, { baseMs: timeoutMs });
|
|
67
143
|
|
|
68
144
|
const get = (path, params = {}, options = {}) => {
|
|
69
145
|
const url = new URL(`${baseUrl}${path}`);
|
|
70
146
|
Object.entries(params).forEach(([k, v]) => v != null && url.searchParams.set(k, v));
|
|
71
|
-
return
|
|
147
|
+
return timedFetch(url.toString(), { headers: headers(authToken), ...options }, {
|
|
148
|
+
method: 'GET', path, ms: limit('read'), session,
|
|
149
|
+
});
|
|
72
150
|
};
|
|
73
151
|
|
|
74
|
-
const post = (path, body = {}) =>
|
|
152
|
+
const post = (path, body = {}) => timedFetch(`${baseUrl}${path}`, {
|
|
75
153
|
method: 'POST',
|
|
76
154
|
headers: headers(authToken),
|
|
77
155
|
body: JSON.stringify(body),
|
|
78
|
-
}
|
|
156
|
+
}, { method: 'POST', path, ms: limit('write'), session });
|
|
79
157
|
|
|
80
|
-
const patch = (path, body = {}) =>
|
|
158
|
+
const patch = (path, body = {}) => timedFetch(`${baseUrl}${path}`, {
|
|
81
159
|
method: 'PATCH',
|
|
82
160
|
headers: headers(authToken),
|
|
83
161
|
body: JSON.stringify(body),
|
|
84
|
-
}
|
|
162
|
+
}, { method: 'PATCH', path, ms: limit('write'), session });
|
|
85
163
|
|
|
86
|
-
const del = (path, body) =>
|
|
164
|
+
const del = (path, body) => timedFetch(`${baseUrl}${path}`, {
|
|
87
165
|
method: 'DELETE',
|
|
88
166
|
headers: headers(authToken),
|
|
89
167
|
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
|
90
|
-
}
|
|
168
|
+
}, { method: 'DELETE', path, ms: limit('write'), session });
|
|
91
169
|
|
|
92
170
|
// Multipart upload via native FormData/Blob (Node 18+) — no runtime deps.
|
|
93
171
|
// Content-Type is deliberately NOT set: fetch writes the multipart boundary.
|
|
@@ -103,11 +181,11 @@ export const createClient = ({ instance = null, token = undefined } = {}) => {
|
|
|
103
181
|
for (const [k, v] of Object.entries(fields)) {
|
|
104
182
|
if (v !== undefined && v !== null) form.append(k, String(v));
|
|
105
183
|
}
|
|
106
|
-
return
|
|
184
|
+
return timedFetch(`${baseUrl}${path}`, {
|
|
107
185
|
method: 'POST',
|
|
108
186
|
headers: authToken ? { Authorization: `Bearer ${authToken}` } : {},
|
|
109
187
|
body: form,
|
|
110
|
-
}
|
|
188
|
+
}, { method: 'POST', path, ms: limit('upload'), session });
|
|
111
189
|
};
|
|
112
190
|
|
|
113
191
|
return {
|
|
@@ -116,11 +194,11 @@ export const createClient = ({ instance = null, token = undefined } = {}) => {
|
|
|
116
194
|
};
|
|
117
195
|
|
|
118
196
|
// Convenience: login doesn't need a token
|
|
119
|
-
export const login =
|
|
120
|
-
const
|
|
197
|
+
export const login = (instanceUrl, email, password) => {
|
|
198
|
+
const ms = capTimeoutMs('write');
|
|
199
|
+
return timedFetch(`${instanceUrl}/api/auth/login`, {
|
|
121
200
|
method: 'POST',
|
|
122
201
|
headers: { 'Content-Type': 'application/json' },
|
|
123
202
|
body: JSON.stringify({ email, password }),
|
|
124
|
-
});
|
|
125
|
-
return handleResponse(res);
|
|
203
|
+
}, { method: 'POST', path: '/api/auth/login', ms, session: null });
|
|
126
204
|
};
|
package/src/lib/inbox.js
CHANGED
|
@@ -62,6 +62,7 @@
|
|
|
62
62
|
|
|
63
63
|
import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'fs';
|
|
64
64
|
import { dirname } from 'path';
|
|
65
|
+
import { capTimeoutMs, formatLimit } from './api.js';
|
|
65
66
|
|
|
66
67
|
export const INBOX_KINDS = ['mention', 'decision', 'handoff', 'approval'];
|
|
67
68
|
export const TOKEN_FILE_ENV = 'COMMONLY_TOKEN_FILE';
|
|
@@ -394,19 +395,29 @@ export const createInboxRequest = ({
|
|
|
394
395
|
for (const [key, value] of Object.entries(params)) {
|
|
395
396
|
if (value !== undefined && value !== null) url.searchParams.set(key, String(value));
|
|
396
397
|
}
|
|
398
|
+
// TASK-338: the same hard timeout as every other CAP request (api.js).
|
|
399
|
+
const timeoutMs = capTimeoutMs(method === 'GET' ? 'read' : 'write');
|
|
397
400
|
for (let attempt = 0; ; attempt += 1) {
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
401
|
+
let res;
|
|
402
|
+
let text;
|
|
403
|
+
try {
|
|
404
|
+
// eslint-disable-next-line no-await-in-loop
|
|
405
|
+
res = await fetchImpl(url.toString(), {
|
|
406
|
+
method,
|
|
407
|
+
headers: {
|
|
408
|
+
Authorization: `Bearer ${token}`,
|
|
409
|
+
'Content-Type': 'application/json',
|
|
410
|
+
'User-Agent': 'commonly-cli/0.1',
|
|
411
|
+
},
|
|
412
|
+
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
|
413
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
414
|
+
});
|
|
415
|
+
// eslint-disable-next-line no-await-in-loop
|
|
416
|
+
text = await res.text();
|
|
417
|
+
} catch (err) {
|
|
418
|
+
if (err?.name !== 'TimeoutError') throw err;
|
|
419
|
+
throw new InboxRequestError(`${method} ${path} got no answer in ${formatLimit(timeoutMs)}`, null);
|
|
420
|
+
}
|
|
410
421
|
let parsed;
|
|
411
422
|
try {
|
|
412
423
|
parsed = JSON.parse(text);
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Poll liveness for the inbox loops (TASK-338).
|
|
3
|
+
*
|
|
4
|
+
* The hard timeout in api.js bounds every CAP request, so no single request can
|
|
5
|
+
* hang for good any more. This is the second net, for a loop that stops making
|
|
6
|
+
* progress anyway: if no poll has settled within the limit while the loop was
|
|
7
|
+
* doing its own work, it is wedged, and the only thing that heals it is a fresh
|
|
8
|
+
* process. Measured on 10-31: nib's and folio's loops sat 8+ hours on one
|
|
9
|
+
* unanswered request, alive the whole time, and their keepalives (tmux, launchd
|
|
10
|
+
* KeepAlive) respawn only on exit. So a stall ends the process, loudly, and the
|
|
11
|
+
* keepalive does the rest.
|
|
12
|
+
*
|
|
13
|
+
* The clock runs while the loop is polling, acking or holding work, and is
|
|
14
|
+
* paused while a model turn runs and while the loop waits on its own scheduled
|
|
15
|
+
* retry. A two-hour turn never trips it, and neither does a fifteen-minute
|
|
16
|
+
* backoff: those always end on their own. A settled poll or a finished turn
|
|
17
|
+
* restarts the clock.
|
|
18
|
+
*
|
|
19
|
+
* Monotonic time, not Date.now: a laptop seat that sleeps, or a clock corrected
|
|
20
|
+
* by ten minutes, must not read as a stall.
|
|
21
|
+
*/
|
|
22
|
+
import { performance } from 'perf_hooks';
|
|
23
|
+
|
|
24
|
+
export const POLL_LIVENESS_DEFAULT_MS = 10 * 60 * 1000;
|
|
25
|
+
|
|
26
|
+
export const resolvePollLivenessMs = (value, env = process.env) => {
|
|
27
|
+
const candidate = [value, Number(env.COMMONLY_POLL_LIVENESS_MS)]
|
|
28
|
+
.find((ms) => Number.isFinite(ms) && ms > 0);
|
|
29
|
+
return candidate ?? POLL_LIVENESS_DEFAULT_MS;
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
export const formatStall = (ms) => (ms < 60_000
|
|
33
|
+
? `${Math.max(1, Math.round(ms / 1000))}s`
|
|
34
|
+
: `${Math.round(ms / 60_000)} min`);
|
|
35
|
+
|
|
36
|
+
export const createPollLiveness = ({ limitMs, onStall, checkEveryMs }) => {
|
|
37
|
+
let armedSince = null;
|
|
38
|
+
let stalled = false;
|
|
39
|
+
// A quarter of the limit, so a stall is caught within 1.25x of it, bounded
|
|
40
|
+
// to a check a second at most and one a minute at least.
|
|
41
|
+
const every = Number.isFinite(checkEveryMs) && checkEveryMs > 0
|
|
42
|
+
? checkEveryMs
|
|
43
|
+
: Math.min(60_000, Math.max(1000, Math.floor(limitMs / 4)));
|
|
44
|
+
const timer = setInterval(() => {
|
|
45
|
+
if (stalled || armedSince === null) return;
|
|
46
|
+
const stalledMs = performance.now() - armedSince;
|
|
47
|
+
if (stalledMs < limitMs) return;
|
|
48
|
+
stalled = true;
|
|
49
|
+
onStall({ stalledMs, limitMs });
|
|
50
|
+
}, every);
|
|
51
|
+
// The check must never be what keeps a stopped loop's process alive.
|
|
52
|
+
timer.unref?.();
|
|
53
|
+
return {
|
|
54
|
+
// (Re)start the clock: the loop is doing its own work, or just made progress.
|
|
55
|
+
arm: () => { armedSince = performance.now(); },
|
|
56
|
+
// Stop the clock: a model turn, or a wait the loop scheduled itself.
|
|
57
|
+
pause: () => { armedSince = null; },
|
|
58
|
+
stop: () => { clearInterval(timer); },
|
|
59
|
+
};
|
|
60
|
+
};
|
package/src/lib/poller.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
import { createClient } from './api.js';
|
|
10
10
|
import { formatDigestFrame } from './digest-frame.js';
|
|
11
11
|
import { EMPTY_OUTPUT } from './claim-outcome.js';
|
|
12
|
+
import { createPollLiveness, formatStall, resolvePollLivenessMs } from './poll-liveness.js';
|
|
12
13
|
|
|
13
14
|
// Delivery failures that must stop a consumer rather than fall through to the
|
|
14
15
|
// normal at-least-once retry path. A stale delivery belongs to a replacement
|
|
@@ -39,6 +40,9 @@ export const startPoller = ({
|
|
|
39
40
|
loginInstance, // the --instance the connect ran with, so the login hint names it
|
|
40
41
|
onEvent, // async (event) => { outcome, content? }
|
|
41
42
|
onError, // (err) => void
|
|
43
|
+
pollLivenessMs, // TASK-338: see lib/poll-liveness.js
|
|
44
|
+
pollLivenessCheckMs,
|
|
45
|
+
onPollStalled, // ({ stalledMs }) => void; the connect command exits on it
|
|
42
46
|
}) => {
|
|
43
47
|
const client = createClient({ instance: instanceUrl, token });
|
|
44
48
|
let running = true;
|
|
@@ -49,16 +53,35 @@ export const startPoller = ({
|
|
|
49
53
|
// reprovision-all; 5+ wastes rate-limit budget after the real-revoke case.
|
|
50
54
|
let consecutiveAuthErrors = 0;
|
|
51
55
|
const MAX_AUTH_ERRORS = 3;
|
|
56
|
+
let pollStalled = false;
|
|
57
|
+
const pollLiveness = createPollLiveness({
|
|
58
|
+
limitMs: resolvePollLivenessMs(pollLivenessMs),
|
|
59
|
+
checkEveryMs: pollLivenessCheckMs,
|
|
60
|
+
onStall: ({ stalledMs }) => {
|
|
61
|
+
if (!running) return;
|
|
62
|
+
pollStalled = true;
|
|
63
|
+
running = false;
|
|
64
|
+
pollLiveness.stop();
|
|
65
|
+
onError?.(new Error(`poll stalled: no inbox poll settled for ${formatStall(stalledMs)} — stopping poller`));
|
|
66
|
+
onPollStalled?.({ stalledMs });
|
|
67
|
+
},
|
|
68
|
+
});
|
|
52
69
|
|
|
53
70
|
const poll = async () => {
|
|
54
71
|
if (!running) return;
|
|
55
72
|
|
|
56
73
|
try {
|
|
57
|
-
|
|
74
|
+
// Same clock as `agent run`: running from here to the scheduled wait,
|
|
75
|
+
// paused while the local handler takes the event.
|
|
76
|
+
pollLiveness.arm();
|
|
77
|
+
const page = await client.get('/api/agents/runtime/events', {
|
|
58
78
|
agentName,
|
|
59
79
|
instanceId,
|
|
60
80
|
limit: 10,
|
|
61
81
|
});
|
|
82
|
+
pollLiveness.arm();
|
|
83
|
+
if (pollStalled) return;
|
|
84
|
+
const { events = [] } = page;
|
|
62
85
|
|
|
63
86
|
// A digest is received context, not a webhook invocation. The runtime
|
|
64
87
|
// inbox claims it only alongside a wake from the same pod.
|
|
@@ -83,9 +106,12 @@ export const startPoller = ({
|
|
|
83
106
|
let result = { outcome: 'no_action' };
|
|
84
107
|
try {
|
|
85
108
|
// A handler that returns nothing said nothing: not a read of the digest.
|
|
109
|
+
pollLiveness.pause();
|
|
86
110
|
result = (await onEvent(forwarded)) || { outcome: 'no_action', reason: EMPTY_OUTPUT };
|
|
87
111
|
} catch (handlerErr) {
|
|
88
112
|
result = { outcome: 'error', reason: handlerErr.message };
|
|
113
|
+
} finally {
|
|
114
|
+
pollLiveness.arm();
|
|
89
115
|
}
|
|
90
116
|
|
|
91
117
|
// Acknowledge the event
|
|
@@ -150,11 +176,14 @@ export const startPoller = ({
|
|
|
150
176
|
}
|
|
151
177
|
consecutiveErrors++;
|
|
152
178
|
onError?.(err);
|
|
153
|
-
// Back off on repeated errors (max 60s)
|
|
179
|
+
// Back off on repeated errors (max 60s). A wait this loop scheduled
|
|
180
|
+
// itself always ends, so the liveness clock stops for it.
|
|
154
181
|
const backoff = Math.min(intervalMs * consecutiveErrors, 60_000);
|
|
182
|
+
pollLiveness.pause();
|
|
155
183
|
await new Promise((r) => setTimeout(r, backoff));
|
|
156
184
|
}
|
|
157
185
|
|
|
186
|
+
pollLiveness.pause();
|
|
158
187
|
if (running) setTimeout(poll, intervalMs);
|
|
159
188
|
};
|
|
160
189
|
|
|
@@ -162,6 +191,9 @@ export const startPoller = ({
|
|
|
162
191
|
poll();
|
|
163
192
|
|
|
164
193
|
return {
|
|
165
|
-
stop: () => {
|
|
194
|
+
stop: () => {
|
|
195
|
+
running = false;
|
|
196
|
+
pollLiveness.stop();
|
|
197
|
+
},
|
|
166
198
|
};
|
|
167
199
|
};
|