@commonlyai/cli 0.1.13 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/commands/agent.js +65 -4
- package/src/lib/enforcement.js +42 -7
- package/src/lib/poll-retry.js +81 -0
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@commonlyai/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.18",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
|
-
"description": "The Commonly CLI
|
|
5
|
+
"description": "The Commonly CLI \u2014 connect agents, manage pods, iterate fast",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"main": "./src/index.js",
|
|
8
8
|
"bin": {
|
package/src/commands/agent.js
CHANGED
|
@@ -29,6 +29,7 @@ import {
|
|
|
29
29
|
recordHandledEvent,
|
|
30
30
|
} from '../lib/session-store.js';
|
|
31
31
|
import { readLongTerm, syncBack } from '../lib/memory-bridge.js';
|
|
32
|
+
import { pollRetryPolicy } from '../lib/poll-retry.js';
|
|
32
33
|
import { detectMemorySources, composeImport, importMemory } from '../lib/memory-import.js';
|
|
33
34
|
import { detectSkills, importSkills } from '../lib/skills-import.js';
|
|
34
35
|
import { parseEnvironmentFile, resolveWorkspace } from '../lib/environment.js';
|
|
@@ -732,6 +733,7 @@ export const performRun = ({
|
|
|
732
733
|
// reprovision-all; 5+ wastes rate-limit budget after the real-revoke case.
|
|
733
734
|
let consecutiveAuthErrors = 0;
|
|
734
735
|
const MAX_AUTH_ERRORS = 3;
|
|
736
|
+
let consecutivePollFailures = 0;
|
|
735
737
|
let consecutiveSpawnFailures = 0;
|
|
736
738
|
const spawnJitterRatio = retryJitterRatio ?? spawnRetryJitter(agentName);
|
|
737
739
|
// Per-seat cascade state — lives with the process, like the session store.
|
|
@@ -859,7 +861,7 @@ export const performRun = ({
|
|
|
859
861
|
// Recording a human trigger sets the streak to 0, so this call and the
|
|
860
862
|
// completion-time one are idempotent with each other.
|
|
861
863
|
if (trigger === 'human') cascadeGovernor.record(eventPodId, trigger);
|
|
862
|
-
const admission = cascadeGovernor.admit(eventPodId, trigger, event.type);
|
|
864
|
+
const admission = cascadeGovernor.admit(eventPodId, trigger, event.type, event.payload);
|
|
863
865
|
if (!admission.allowed) {
|
|
864
866
|
// Name the MESSAGE, not just the pod. Without this the refusal is silent at
|
|
865
867
|
// three ends, not two: the mentioning agent gets no signal its mention died,
|
|
@@ -945,10 +947,20 @@ export const performRun = ({
|
|
|
945
947
|
});
|
|
946
948
|
const claim = await claimKeeper.acquire();
|
|
947
949
|
if (!claim.claimed && !claim.failOpen) {
|
|
948
|
-
|
|
950
|
+
// Per-event EVIDENCE, deliberately not a widening of
|
|
951
|
+
// ADDRESSED_EVENT_TYPES (that set is a pricing table): the backend
|
|
952
|
+
// stamps repliesToYourMessage when the woken message replies to or
|
|
953
|
+
// threads on a message THIS seat authored. Without it, a peer's
|
|
954
|
+
// claim on its own reply ordered the replied-to author out of its
|
|
955
|
+
// own conversation (Sage stood down twice on Anvil's thread
|
|
956
|
+
// replies, 2026-08-24). Interim for TASK-058.
|
|
957
|
+
const repliesToThisSeat = event.payload?.repliesToYourMessage === true;
|
|
958
|
+
if (ADDRESSED_EVENT_TYPES.has(event.type) || repliesToThisSeat) {
|
|
949
959
|
log(
|
|
950
960
|
`[${event.type}] message ${claimMessageId} held by ${claim.holder} — `
|
|
951
|
-
+
|
|
961
|
+
+ (repliesToThisSeat
|
|
962
|
+
? 'proceeding peer-aware (the message replies to this seat\'s own message)'
|
|
963
|
+
: 'proceeding peer-aware (this seat was directly addressed)'),
|
|
952
964
|
);
|
|
953
965
|
peerFrame = peerHoldsFrame(claim.holder, claimMessageId);
|
|
954
966
|
claimKeeper = null; // nothing held: no renewal, no release, no isLost gate
|
|
@@ -1171,10 +1183,32 @@ export const performRun = ({
|
|
|
1171
1183
|
if (!running) return;
|
|
1172
1184
|
let nextPollDelayMs = intervalMs;
|
|
1173
1185
|
try {
|
|
1186
|
+
// ONE event per fetch, because the fetch IS the claim.
|
|
1187
|
+
//
|
|
1188
|
+
// `AgentEventService.list()` does not hand back a preview — it marks
|
|
1189
|
+
// every candidate `delivered` with `$inc: { attempts: 1 }` before
|
|
1190
|
+
// returning. This loop then processes them SERIALLY, one full model
|
|
1191
|
+
// turn each. So asking for 10 claims 10 and starts 1.
|
|
1192
|
+
//
|
|
1193
|
+
// The nine it cannot start are then reclaimed out from under it: the
|
|
1194
|
+
// backend requeues `delivered` rows older than
|
|
1195
|
+
// `requeueDeliveredMinutes` (default 10, swept on `*/10`), and turns
|
|
1196
|
+
// routinely outlast that — measured on the pod-architect seat over
|
|
1197
|
+
// 11.5h: median 128s, p90 669s, 13 turns over 600s, max 1153s. Each
|
|
1198
|
+
// sweep returns the untouched siblings to `pending` at `attempts + 1`,
|
|
1199
|
+
// and `attempts >= 3` retires an event to `failed`, which is terminal
|
|
1200
|
+
// and invisible to `list()`. That cap exists to bound POISON events;
|
|
1201
|
+
// over-claiming feeds it work no model ever saw, so a mention can be
|
|
1202
|
+
// dropped without once being read.
|
|
1203
|
+
//
|
|
1204
|
+
// `limit: 1` costs nothing: capacity here is one turn at a time
|
|
1205
|
+
// regardless, and the poll interval is 5s. It only stops the loop
|
|
1206
|
+
// claiming work it has no way to begin.
|
|
1174
1207
|
const { events = [] } = await client.get('/api/agents/runtime/events', {
|
|
1175
|
-
agentName, instanceId, limit:
|
|
1208
|
+
agentName, instanceId, limit: 1,
|
|
1176
1209
|
});
|
|
1177
1210
|
consecutiveAuthErrors = 0;
|
|
1211
|
+
consecutivePollFailures = 0;
|
|
1178
1212
|
for (const event of events) {
|
|
1179
1213
|
if (!running) break;
|
|
1180
1214
|
let result;
|
|
@@ -1265,6 +1299,33 @@ export const performRun = ({
|
|
|
1265
1299
|
running = false;
|
|
1266
1300
|
return;
|
|
1267
1301
|
}
|
|
1302
|
+
} else {
|
|
1303
|
+
// TASK-025: everything that is not an auth rejection used to fall
|
|
1304
|
+
// through to `onError` with `nextPollDelayMs` still at `intervalMs`,
|
|
1305
|
+
// because that variable is only reassigned in the spawn-retry branch
|
|
1306
|
+
// above and a failed fetch never reaches it. So a network outage
|
|
1307
|
+
// retried flat at the poll interval, forever, with one indistinct
|
|
1308
|
+
// line per attempt — 797 consecutive `fetch failed` across three
|
|
1309
|
+
// seats, ~66 minutes, and nothing that read as an outage.
|
|
1310
|
+
//
|
|
1311
|
+
// Deliberately NOT a stop. `MAX_AUTH_ERRORS` halts because a rejected
|
|
1312
|
+
// token cannot heal itself; a network failure usually can, and a seat
|
|
1313
|
+
// that stops on one is dead until a human notices.
|
|
1314
|
+
consecutivePollFailures += 1;
|
|
1315
|
+
const retry = pollRetryPolicy({
|
|
1316
|
+
consecutiveFailures: consecutivePollFailures,
|
|
1317
|
+
intervalMs,
|
|
1318
|
+
jitterRatio: spawnJitterRatio,
|
|
1319
|
+
});
|
|
1320
|
+
nextPollDelayMs = retry.delayMs;
|
|
1321
|
+
if (retry.escalate) {
|
|
1322
|
+
log(
|
|
1323
|
+
`poll failed ${consecutivePollFailures}x in a row `
|
|
1324
|
+
+ `(${err?.message || 'unknown error'}) — backing off to `
|
|
1325
|
+
+ `${formatRetryDelay(retry.delayMs)}`
|
|
1326
|
+
+ `${retry.atCeiling ? ', at the ceiling' : ''}`,
|
|
1327
|
+
);
|
|
1328
|
+
}
|
|
1268
1329
|
}
|
|
1269
1330
|
onError?.(err);
|
|
1270
1331
|
}
|
package/src/lib/enforcement.js
CHANGED
|
@@ -45,6 +45,14 @@ export const CLAIMABLE_EVENT_TYPES = new Set([
|
|
|
45
45
|
*/
|
|
46
46
|
export const classifyTrigger = (event, recentMessages) => {
|
|
47
47
|
const p = event?.payload || {};
|
|
48
|
+
// The third pricing branch (#1044, fable 55845). Kernel-found work is not
|
|
49
|
+
// agent churn and must not price as it: counted as 'agent' it eats cascade
|
|
50
|
+
// budget exactly when the board is busiest; counted as 'human' it CLEARS the
|
|
51
|
+
// brake on unrelated cascades. 'kernel' is neutral like 'unknown' — no
|
|
52
|
+
// count, no reset — but chosen on purpose rather than fallen into, and the
|
|
53
|
+
// refusal/boot logs can say so. Checked before dmKind: triggerAuthor is the
|
|
54
|
+
// honest field (#1018) and wins where both appear.
|
|
55
|
+
if (p.triggerAuthor === 'kernel') return 'kernel';
|
|
48
56
|
if (p.dmKind === 'agent-agent') return 'agent';
|
|
49
57
|
if (p.dmKind === 'user-agent') return 'human';
|
|
50
58
|
if (!p.messageId || !Array.isArray(recentMessages)) return 'unknown';
|
|
@@ -62,6 +70,19 @@ export const classifyTrigger = (event, recentMessages) => {
|
|
|
62
70
|
// race is a free stand-down.
|
|
63
71
|
export const ADDRESSED_EVENT_TYPES = new Set(['chat.mention', 'thread.mention', 'dm.message']);
|
|
64
72
|
|
|
73
|
+
// These are the two event types the kernel's bot-to-bot mention dampener
|
|
74
|
+
// bounds as one shared budget. Keep the wrapper out of that same loop's
|
|
75
|
+
// admission path: named seats need to respond even when broadcasts have
|
|
76
|
+
// exhausted their cascade budget. The backend cannot be a runtime import of
|
|
77
|
+
// the published CLI, so cli/__tests__/mention-event-types.contract.test.mjs
|
|
78
|
+
// imports both modules and pins the two lists together.
|
|
79
|
+
//
|
|
80
|
+
// Agent-DM wakes also arrive as chat.mention, but carry payload.dmKind. They
|
|
81
|
+
// stay on this governor's bounded path: event type alone does not identify the
|
|
82
|
+
// producer that owns the kernel mention budget. dm.message is legacy consumer
|
|
83
|
+
// vocabulary and remains on that same ordinary addressed-grace path.
|
|
84
|
+
export const MENTION_EVENT_TYPES = new Set(['chat.mention', 'thread.mention']);
|
|
85
|
+
|
|
65
86
|
// ── cascade governor ────────────────────────────────────────────────────────
|
|
66
87
|
|
|
67
88
|
export const CASCADE_DEFAULTS = Object.freeze({
|
|
@@ -213,9 +234,24 @@ export const createCascadeGovernor = ({
|
|
|
213
234
|
};
|
|
214
235
|
|
|
215
236
|
return {
|
|
216
|
-
admit(podId, trigger, eventType) {
|
|
237
|
+
admit(podId, trigger, eventType, payload = {}) {
|
|
217
238
|
if (trigger !== 'agent') return { allowed: true, streak: 0, addressed: false };
|
|
218
239
|
const s = stateFor(podId);
|
|
240
|
+
const addressed = ADDRESSED_EVENT_TYPES.has(eventType);
|
|
241
|
+
if (MENTION_EVENT_TYPES.has(eventType) && !payload?.dmKind) {
|
|
242
|
+
// Named mentions are bounded upstream by the kernel's shared
|
|
243
|
+
// bot-to-bot mention dampener. A DM emits chat.mention too, but its
|
|
244
|
+
// dmKind identifies a different producer, so it remains locally
|
|
245
|
+
// bounded. Every admitted turn still reaches record() after it
|
|
246
|
+
// completes, consuming broadcast liveness rather than creating an
|
|
247
|
+
// unmetered second loop.
|
|
248
|
+
return {
|
|
249
|
+
allowed: true,
|
|
250
|
+
streak: s.streak,
|
|
251
|
+
addressed,
|
|
252
|
+
graceApplied: false,
|
|
253
|
+
};
|
|
254
|
+
}
|
|
219
255
|
// Being NAMED outranks a mechanical brake — the same judgement the claim
|
|
220
256
|
// path already makes forty lines down in agent.js. Without this, a peer
|
|
221
257
|
// can @mention a capped seat and get silence, with no signal to either
|
|
@@ -223,11 +259,9 @@ export const createCascadeGovernor = ({
|
|
|
223
259
|
// 51 wakes and 28 consecutive cap refusals, five of them chat.mention,
|
|
224
260
|
// and answered none of them.
|
|
225
261
|
//
|
|
226
|
-
//
|
|
227
|
-
//
|
|
228
|
-
//
|
|
229
|
-
// cap + addressedGrace instead of never.
|
|
230
|
-
const addressed = ADDRESSED_EVENT_TYPES.has(eventType);
|
|
262
|
+
// Legacy direct-address events retain a GRACE, not an exemption. They
|
|
263
|
+
// still count toward the streak, so they terminate at cap +
|
|
264
|
+
// addressedGrace instead of never.
|
|
231
265
|
const limit = addressed ? cap + addressedGrace : cap;
|
|
232
266
|
// `addressed` describes the EVENT; `graceApplied` describes what this
|
|
233
267
|
// governor actually did with it. They diverge whenever addressedGrace is
|
|
@@ -249,7 +283,8 @@ export const createCascadeGovernor = ({
|
|
|
249
283
|
const s = stateFor(podId);
|
|
250
284
|
pods.set(podId, { streak: s.streak + 1, lastAgentTurnAt: now() });
|
|
251
285
|
}
|
|
252
|
-
// 'unknown' is neutral
|
|
286
|
+
// 'unknown' is neutral by FALLBACK; 'kernel' is neutral by DESIGN —
|
|
287
|
+
// both take this path, but only one of them is an accident.
|
|
253
288
|
},
|
|
254
289
|
};
|
|
255
290
|
};
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retry policy for the run loop's own FETCH failures (TASK-025).
|
|
3
|
+
*
|
|
4
|
+
* `spawn-retry.js` bounds failures of the subprocess a poll produces. This
|
|
5
|
+
* bounds failures of the poll itself — the request to
|
|
6
|
+
* `/api/agents/runtime/events` that has to succeed before there is anything to
|
|
7
|
+
* spawn. They are different failures with different remedies and, until now,
|
|
8
|
+
* only one of them had a policy.
|
|
9
|
+
*
|
|
10
|
+
* The gap, measured: `agent run`'s tick set `nextPollDelayMs = intervalMs` and
|
|
11
|
+
* only ever reassigned it inside the spawn-retry branch. A fetch that threw
|
|
12
|
+
* never reached that branch, so a network failure retried at a flat 5s
|
|
13
|
+
* forever — no backoff, no ceiling, and no escalation. 797 consecutive
|
|
14
|
+
* `fetch failed` ran across three seats that way: ~66 minutes at 5s, and the
|
|
15
|
+
* only trace was one `onError` line per attempt in a log nobody was tailing.
|
|
16
|
+
*
|
|
17
|
+
* Two deliberate differences from the auth path directly above it:
|
|
18
|
+
*
|
|
19
|
+
* 1. NO STOP. `MAX_AUTH_ERRORS` halts the loop because a rejected token does
|
|
20
|
+
* not heal on its own — retrying is pure cost and the operator must act.
|
|
21
|
+
* A network failure is usually transient, and a seat that stops on one is
|
|
22
|
+
* dead until someone notices. So this backs off and stays alive.
|
|
23
|
+
*
|
|
24
|
+
* 2. ESCALATION IS THE POINT. The harm in the measured outage was not the
|
|
25
|
+
* retry rate, it was that 797 failures produced no signal distinguishable
|
|
26
|
+
* from one failure. `escalate` fires on a small set of thresholds so a
|
|
27
|
+
* sustained outage announces itself at widening intervals instead of
|
|
28
|
+
* disappearing into per-attempt noise.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
// Same ceiling as the spawn path: past this, more waiting buys nothing and a
|
|
32
|
+
// human is the only thing that resolves it.
|
|
33
|
+
export const POLL_RETRY_MAX_MS = 15 * 60 * 1000;
|
|
34
|
+
|
|
35
|
+
// Backoff starts only after the first failure has been retried once at the
|
|
36
|
+
// normal interval, so a single blip costs nothing.
|
|
37
|
+
export const POLL_BACKOFF_AFTER = 1;
|
|
38
|
+
|
|
39
|
+
// Failure counts that emit a loud line. Chosen against the measured outage:
|
|
40
|
+
// at intervalMs=5000 these land at roughly 15s, 1min, 5min, 20min and 1h of
|
|
41
|
+
// sustained failure, so an operator reading the log sees the shape of the
|
|
42
|
+
// outage rather than 797 identical lines.
|
|
43
|
+
export const POLL_ESCALATE_AT = Object.freeze([3, 10, 30, 60, 120]);
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Next poll delay after a failed fetch, and whether this attempt should be
|
|
47
|
+
* announced loudly.
|
|
48
|
+
*
|
|
49
|
+
* Exponential from `intervalMs`, bounded by POLL_RETRY_MAX_MS. Jitter is the
|
|
50
|
+
* same anti-herd offset the spawn path uses — without it, every seat that lost
|
|
51
|
+
* the same upstream retries in lockstep and re-creates the thundering herd on
|
|
52
|
+
* recovery.
|
|
53
|
+
*/
|
|
54
|
+
export const pollRetryPolicy = ({
|
|
55
|
+
consecutiveFailures,
|
|
56
|
+
intervalMs,
|
|
57
|
+
jitterRatio = 0,
|
|
58
|
+
}) => {
|
|
59
|
+
const safeIntervalMs = Number.isFinite(intervalMs) && intervalMs > 0 ? intervalMs : 5000;
|
|
60
|
+
const failureCount = Number.isInteger(consecutiveFailures) && consecutiveFailures > 0
|
|
61
|
+
? consecutiveFailures
|
|
62
|
+
: 1;
|
|
63
|
+
|
|
64
|
+
const steps = Math.max(0, failureCount - POLL_BACKOFF_AFTER);
|
|
65
|
+
const raw = safeIntervalMs * (2 ** steps);
|
|
66
|
+
const bounded = Math.min(POLL_RETRY_MAX_MS, raw);
|
|
67
|
+
|
|
68
|
+
// Clamp rather than trust the caller: a jitterRatio above the cap would
|
|
69
|
+
// widen the herd window instead of narrowing it.
|
|
70
|
+
const safeJitter = Number.isFinite(jitterRatio)
|
|
71
|
+
? Math.min(0.2, Math.max(0, jitterRatio))
|
|
72
|
+
: 0;
|
|
73
|
+
|
|
74
|
+
return {
|
|
75
|
+
delayMs: Math.round(bounded * (1 + safeJitter)),
|
|
76
|
+
escalate: POLL_ESCALATE_AT.includes(failureCount),
|
|
77
|
+
atCeiling: bounded >= POLL_RETRY_MAX_MS,
|
|
78
|
+
};
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
export default pollRetryPolicy;
|