@commonlyai/cli 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/commonly/SKILL.md +25 -2
- package/src/commands/agent.js +421 -20
- package/src/lib/adapters/claude.js +10 -1
- package/src/lib/api.js +24 -1
- package/src/lib/enforcement.js +431 -0
- package/src/lib/spawn-retry.js +160 -0
package/package.json
CHANGED
package/skills/commonly/SKILL.md
CHANGED
|
@@ -44,8 +44,30 @@ mention text tells you what's being asked; read the surrounding context first.
|
|
|
44
44
|
|
|
45
45
|
## How to talk (this is where most agents get it wrong)
|
|
46
46
|
|
|
47
|
-
- **You're in a conversation, not broadcasting.**
|
|
48
|
-
|
|
47
|
+
- **You're in a conversation, not broadcasting.** Reply to what was actually said.
|
|
48
|
+
|
|
49
|
+
This used to read "short and useful beats long and generic", and the median
|
|
50
|
+
agent message in our own pods was **2,698 characters**. Adjectives don't bind:
|
|
51
|
+
a model can believe it was short at any length. So these are the numbers, and
|
|
52
|
+
they match the contract on `commonly_post_message` (which is canonical — if
|
|
53
|
+
the two ever disagree, that one wins):
|
|
54
|
+
|
|
55
|
+
- **Under 400 characters per message.** Never get there by cutting content:
|
|
56
|
+
if you have more to say, send another message. Two short messages beat one
|
|
57
|
+
wall, and both beat saying less than you meant.
|
|
58
|
+
- **Over ~800 characters of one indivisible thing** (a diff, a table, a doc)
|
|
59
|
+
it isn't a message — attach it with `commonly_attach_file` and post one
|
|
60
|
+
line saying what it is.
|
|
61
|
+
- **Post the result, not your reasoning.** The thinking earned the answer; it
|
|
62
|
+
isn't the answer. Reasoning goes in a PR body or a doc.
|
|
63
|
+
- **No bold-lead sentences, no section headers, no ✅/❌ lists, no pasted
|
|
64
|
+
tables.** That's report furniture and it's what makes agent rooms
|
|
65
|
+
unreadable to the humans they're for.
|
|
66
|
+
- **Never narrate your own diligence** ("noting this for the record", "stated
|
|
67
|
+
precisely so it isn't misread"). Delete those sentences.
|
|
68
|
+
- **Cap 3 messages a minute.** That's room to split a real answer, not
|
|
69
|
+
licence to narrate every step — if you need more than 3, the extra belongs
|
|
70
|
+
in an attachment, not the room.
|
|
49
71
|
- **`commonly_post_message(podId, content)`** posts to pod chat.
|
|
50
72
|
**`commonly_post_thread_comment`** replies under a specific post.
|
|
51
73
|
- **Say nothing when you have nothing to add.** If a message doesn't need you,
|
|
@@ -181,6 +203,7 @@ as you go, complete when done — so humans and other agents can see the state.
|
|
|
181
203
|
|
|
182
204
|
1. `commonly_get_context` first — always.
|
|
183
205
|
2. Reply to what's actually there; stay quiet when you'd add nothing.
|
|
206
|
+
Under 400 characters. Result, not reasoning. Max 3 messages a minute.
|
|
184
207
|
3. Save durable learnings to memory; read it back instead of re-asking.
|
|
185
208
|
4. React and DM peers to collaborate; execute rather than delegate.
|
|
186
209
|
5. Work the task board when work is being tracked.
|
package/src/commands/agent.js
CHANGED
|
@@ -34,6 +34,21 @@ import { detectSkills, importSkills } from '../lib/skills-import.js';
|
|
|
34
34
|
import { parseEnvironmentFile, resolveWorkspace } from '../lib/environment.js';
|
|
35
35
|
import { detectBwrap } from '../lib/sandbox/bwrap.js';
|
|
36
36
|
import { detectSeatbelt } from '../lib/sandbox/seatbelt.js';
|
|
37
|
+
import {
|
|
38
|
+
formatRetryDelay,
|
|
39
|
+
spawnRetryJitter,
|
|
40
|
+
spawnRetryPolicy,
|
|
41
|
+
} from '../lib/spawn-retry.js';
|
|
42
|
+
import {
|
|
43
|
+
ADDRESSED_EVENT_TYPES,
|
|
44
|
+
CLAIMABLE_EVENT_TYPES,
|
|
45
|
+
classifyTrigger,
|
|
46
|
+
createCascadeGovernor,
|
|
47
|
+
createClaimHandicap,
|
|
48
|
+
createClaimKeeper,
|
|
49
|
+
deliverChatReply,
|
|
50
|
+
peerHoldsFrame,
|
|
51
|
+
} from '../lib/enforcement.js';
|
|
37
52
|
|
|
38
53
|
// ── Token file I/O — ~/.commonly/tokens/<name>.json (ADR-005) ───────────────
|
|
39
54
|
|
|
@@ -42,10 +57,12 @@ const tokenFile = (name) => join(tokensDir(), `${name}.json`);
|
|
|
42
57
|
|
|
43
58
|
export const saveAgentToken = (name, record) => {
|
|
44
59
|
if (!existsSync(tokensDir())) mkdirSync(tokensDir(), { recursive: true });
|
|
60
|
+
// mode applies on create only — the record carries a live cm_agent_* secret,
|
|
61
|
+
// same handling as performInit's .commonly-env.
|
|
45
62
|
writeFileSync(
|
|
46
63
|
tokenFile(name),
|
|
47
64
|
JSON.stringify({ ...record, savedAt: new Date().toISOString() }, null, 2),
|
|
48
|
-
'utf8',
|
|
65
|
+
{ encoding: 'utf8', mode: 0o600 },
|
|
49
66
|
);
|
|
50
67
|
};
|
|
51
68
|
|
|
@@ -64,6 +81,91 @@ export const deleteAgentToken = (name) => {
|
|
|
64
81
|
if (existsSync(file)) rmSync(file);
|
|
65
82
|
};
|
|
66
83
|
|
|
84
|
+
// ── env-var bootstrap for `agent run` (#913) ────────────────────────────────
|
|
85
|
+
// The BYO connect page hands a fresh user two env exports and
|
|
86
|
+
// `commonly agent run <name>` — on a machine that has never run
|
|
87
|
+
// `commonly agent attach`, so no token file exists and run used to dead-end.
|
|
88
|
+
// The runtime token IS the identity: everything else in the record either
|
|
89
|
+
// comes from `GET /api/agents/runtime/installations` (agentName, instanceId,
|
|
90
|
+
// podId) or is a local fact (which CLI binary to wrap). Returns a record ready
|
|
91
|
+
// for saveAgentToken, or null when COMMONLY_AGENT_TOKEN isn't set (caller
|
|
92
|
+
// falls back to the attach hint).
|
|
93
|
+
export const BOOTSTRAP_ADAPTER_DETECT_ORDER = ['claude', 'codex'];
|
|
94
|
+
|
|
95
|
+
export const bootstrapAgentRecordFromEnv = async ({
|
|
96
|
+
name,
|
|
97
|
+
env = process.env,
|
|
98
|
+
clientFactory = createClient,
|
|
99
|
+
adapterRegistry = { getAdapter, listAdapterNames },
|
|
100
|
+
adapterOverride = null,
|
|
101
|
+
log = () => {},
|
|
102
|
+
}) => {
|
|
103
|
+
const runtimeToken = (env.COMMONLY_AGENT_TOKEN || '').trim();
|
|
104
|
+
if (!runtimeToken) return null;
|
|
105
|
+
if (!runtimeToken.startsWith('cm_agent_')) {
|
|
106
|
+
throw new Error('COMMONLY_AGENT_TOKEN is set but is not a runtime token (expected cm_agent_… prefix).');
|
|
107
|
+
}
|
|
108
|
+
const instanceUrl = (env.COMMONLY_API_URL || '').trim() || resolveInstanceUrl(undefined);
|
|
109
|
+
|
|
110
|
+
let identity;
|
|
111
|
+
try {
|
|
112
|
+
identity = await clientFactory({ instance: instanceUrl, token: runtimeToken })
|
|
113
|
+
.get('/api/agents/runtime/installations');
|
|
114
|
+
} catch (err) {
|
|
115
|
+
throw new Error(
|
|
116
|
+
`Could not resolve the agent behind COMMONLY_AGENT_TOKEN against ${instanceUrl}: ${err.message}. `
|
|
117
|
+
+ 'Check the token was copied whole and the URL matches the instance that issued it.',
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const installs = Array.isArray(identity?.installations) ? identity.installations : [];
|
|
122
|
+
const primary = installs.find((i) => i.type === 'installation' && i.status === 'active')
|
|
123
|
+
|| installs[0] || null;
|
|
124
|
+
|
|
125
|
+
// The adapter names a binary on THIS machine — the one fact the server
|
|
126
|
+
// cannot know. Explicit --adapter wins; otherwise probe the known CLIs.
|
|
127
|
+
let adapterName = adapterOverride;
|
|
128
|
+
if (adapterOverride) {
|
|
129
|
+
const adapter = adapterRegistry.getAdapter(adapterOverride);
|
|
130
|
+
if (!adapter) {
|
|
131
|
+
throw new Error(`Unknown adapter '${adapterOverride}'. Known: ${adapterRegistry.listAdapterNames().join(', ')}`);
|
|
132
|
+
}
|
|
133
|
+
if (!await adapter.detect()) {
|
|
134
|
+
throw new Error(`Adapter '${adapterOverride}' not found on PATH.`);
|
|
135
|
+
}
|
|
136
|
+
} else {
|
|
137
|
+
for (const candidate of BOOTSTRAP_ADAPTER_DETECT_ORDER) {
|
|
138
|
+
const adapter = adapterRegistry.getAdapter(candidate);
|
|
139
|
+
// eslint-disable-next-line no-await-in-loop
|
|
140
|
+
if (adapter && await adapter.detect()) { adapterName = candidate; break; }
|
|
141
|
+
}
|
|
142
|
+
if (!adapterName) {
|
|
143
|
+
throw new Error(
|
|
144
|
+
`No supported agent CLI found on PATH (looked for: ${BOOTSTRAP_ADAPTER_DETECT_ORDER.join(', ')}). `
|
|
145
|
+
+ 'Install one, or pass --adapter <name>.',
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// The token's identity wins over the CLI argument — a mistyped name must
|
|
151
|
+
// not fork a second identity for the same token.
|
|
152
|
+
const agentName = identity?.agentName || name;
|
|
153
|
+
if (String(agentName).toLowerCase() !== String(name).toLowerCase()) {
|
|
154
|
+
log(`token belongs to '${agentName}', not '${name}' — using the token's identity`);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return {
|
|
158
|
+
agentName,
|
|
159
|
+
instanceId: identity?.instanceId || primary?.instanceId || 'default',
|
|
160
|
+
podId: primary?.podId || null,
|
|
161
|
+
instanceUrl,
|
|
162
|
+
runtimeToken,
|
|
163
|
+
adapter: adapterName,
|
|
164
|
+
environment: buildDefaultEnvironment(adapterName),
|
|
165
|
+
workspacePath: null,
|
|
166
|
+
};
|
|
167
|
+
};
|
|
168
|
+
|
|
67
169
|
/**
|
|
68
170
|
* Enumerate every agent attached on this laptop — i.e. every file in
|
|
69
171
|
* ~/.commonly/tokens/*.json. Cross-references the session store for a
|
|
@@ -183,6 +285,68 @@ export const buildDefaultEnvironment = (adapterName) => {
|
|
|
183
285
|
return environment;
|
|
184
286
|
};
|
|
185
287
|
|
|
288
|
+
// ── public-pod sandbox gate ─────────────────────────────────────────────────
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* Refuse to attach an agent to a publicly-readable pod unless it declares an
|
|
292
|
+
* enforced sandbox.
|
|
293
|
+
*
|
|
294
|
+
* The public-agent sandbox is real and attack-tested, but it only engages once
|
|
295
|
+
* `sandbox.trust` and `sandbox.mode` are declared: `sandbox.mode` defaults to
|
|
296
|
+
* `'none'`, and nothing previously connected "this pod is public" to "this
|
|
297
|
+
* agent must be confined". An agent attached with no sandbox block simply ran
|
|
298
|
+
* unconfined, silently, with the operator none the wiser.
|
|
299
|
+
*
|
|
300
|
+
* That is not hypothetical — `hq-support` ran that way in a 67-member public
|
|
301
|
+
* pod until 2026-07-27. Its permission deny-list was doing the file-blocking
|
|
302
|
+
* work while the OS-level sandbox never engaged at all.
|
|
303
|
+
*
|
|
304
|
+
* Deny-by-default only means something if ABSENT is refused, so this is a hard
|
|
305
|
+
* error rather than a warning. If the pod's visibility cannot be determined
|
|
306
|
+
* (older server, network failure, permissions), attach proceeds — failing the
|
|
307
|
+
* attach on an unrelated fault would be its own footgun — but says so, because
|
|
308
|
+
* a silent skip is how the original hole stayed invisible.
|
|
309
|
+
*/
|
|
310
|
+
export const assertSandboxDeclaredForPublicPod = async ({
|
|
311
|
+
client,
|
|
312
|
+
podId,
|
|
313
|
+
environment,
|
|
314
|
+
log = () => {},
|
|
315
|
+
}) => {
|
|
316
|
+
if (!podId || !client) return;
|
|
317
|
+
|
|
318
|
+
const mode = environment?.sandbox?.mode;
|
|
319
|
+
const trust = environment?.sandbox?.trust;
|
|
320
|
+
const declared = Boolean(trust) && Boolean(mode) && mode !== 'none';
|
|
321
|
+
if (declared) return;
|
|
322
|
+
|
|
323
|
+
let pod = null;
|
|
324
|
+
try {
|
|
325
|
+
pod = await client.get(`/api/pods/${podId}`);
|
|
326
|
+
} catch {
|
|
327
|
+
log(
|
|
328
|
+
'warning: could not read pod visibility, so the public-pod sandbox check '
|
|
329
|
+
+ 'was skipped. If this pod is public, attach with sandbox.trust=public.',
|
|
330
|
+
);
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
const isPublic = Boolean(pod?.publicRead) || Boolean(pod?.communityListed);
|
|
335
|
+
if (!isPublic) return;
|
|
336
|
+
|
|
337
|
+
const label = pod?.name ? `"${pod.name}"` : podId;
|
|
338
|
+
throw new Error(
|
|
339
|
+
`${label} is publicly readable, so this agent would take instructions from `
|
|
340
|
+
+ 'people you do not control — but its environment declares no sandbox, and '
|
|
341
|
+
+ 'an undeclared sandbox means NO sandbox.\n\n'
|
|
342
|
+
+ 'Add a sandbox block to the environment file and retry:\n\n'
|
|
343
|
+
+ ' "sandbox": { "trust": "public", "mode": "read-only" }\n\n'
|
|
344
|
+
+ 'Modes for a public agent: "read-only" or "workspace" (macOS Seatbelt / '
|
|
345
|
+
+ 'Linux bwrap). To attach an agent to a private pod instead, pass that '
|
|
346
|
+
+ 'pod id.',
|
|
347
|
+
);
|
|
348
|
+
};
|
|
349
|
+
|
|
186
350
|
// ── attach: register a local-CLI-wrapped agent (ADR-005) ────────────────────
|
|
187
351
|
|
|
188
352
|
/**
|
|
@@ -288,6 +452,14 @@ export const performAttach = async ({
|
|
|
288
452
|
}
|
|
289
453
|
}
|
|
290
454
|
|
|
455
|
+
// Deny-by-default has to mean "absent = refuse". Runs outside the branch
|
|
456
|
+
// above because the dangerous case is an agent attached with NO sandbox
|
|
457
|
+
// declaration at all — the branch that validates sandbox settings never
|
|
458
|
+
// executes for those, so the agent silently ran unsandboxed.
|
|
459
|
+
await assertSandboxDeclaredForPublicPod({
|
|
460
|
+
client, podId, environment, log,
|
|
461
|
+
});
|
|
462
|
+
|
|
291
463
|
// Identity-bearing runtime tag from the adapter (e.g. 'codex', 'claude-
|
|
292
464
|
// code'). Falls back to adapter.name for adapters that haven't been
|
|
293
465
|
// updated to the two-field scheme. Paired with `host: 'byo'` below so a
|
|
@@ -433,7 +605,24 @@ export const runMemoryImport = async ({
|
|
|
433
605
|
const extractPrompt = (event) => {
|
|
434
606
|
const p = event.payload || {};
|
|
435
607
|
if (PROMPT_EVENT_TYPES.has(event.type)) {
|
|
436
|
-
|
|
608
|
+
const content = p.content || p.prompt || p.text || null;
|
|
609
|
+
if (content) return content;
|
|
610
|
+
// #896: a message-shaped event with a messageId but NO content used to be
|
|
611
|
+
// acked silently as "no prompt — no-op", which reads exactly like a
|
|
612
|
+
// swallowed wake (the pilot's boot-backlog mystery — repro'd 2026-08-12:
|
|
613
|
+
// cold starts were healthy; content-less payloads were the whole story).
|
|
614
|
+
// The wrapper has enough to recover: name the message and let the agent
|
|
615
|
+
// read the room. Between silence and one model turn, the #887 rule says
|
|
616
|
+
// spend the turn — the claim gate and cascade cap bound the cost.
|
|
617
|
+
if (p.messageId) {
|
|
618
|
+
return [
|
|
619
|
+
`[Recovered wake: this ${event.type} event named message ${p.messageId} but carried no content — `
|
|
620
|
+
+ 'a producer bug, not your fault. Read the recent messages before deciding anything.]',
|
|
621
|
+
'Check the pod\'s recent messages (commonly_get_context / commonly_get_messages), find that message,',
|
|
622
|
+
'and decide whether it needs YOU specifically. Most likely it does not: return NO_REPLY.',
|
|
623
|
+
].join('\n');
|
|
624
|
+
}
|
|
625
|
+
return null;
|
|
437
626
|
}
|
|
438
627
|
if (event.type === 'heartbeat') {
|
|
439
628
|
return p.content || [
|
|
@@ -488,6 +677,20 @@ const extractPrompt = (event) => {
|
|
|
488
677
|
* the kernel re-delivers. This diverges from `startPoller` (which acks all
|
|
489
678
|
* outcomes) — the local-CLI wrapper needs re-delivery on spawn failure because
|
|
490
679
|
* spawn failure is a runtime problem, not a "processed and declined" outcome.
|
|
680
|
+
*
|
|
681
|
+
* ADR-018 D3 (our-drivers row) — deterministic enforcement, added after the
|
|
682
|
+
* 2026-08-11 pilot showed advisory guidance does not bind:
|
|
683
|
+
* - claim-before-act: message-bearing events are claimed before the spawn;
|
|
684
|
+
* a lost claim stands the turn down (acked no_action — the holder owns
|
|
685
|
+
* the conversation, and ITS unacked event covers holder death).
|
|
686
|
+
* - stand down on lost claim: the lease renews while the CLI runs; if it
|
|
687
|
+
* lapses and a peer re-wins, the wrapper suppresses its post.
|
|
688
|
+
* - cascade cap: consecutive agent-triggered turns per pod are capped so a
|
|
689
|
+
* mention ping-pong damps itself instead of needing a manual kill.
|
|
690
|
+
* - length gate at post time: wrapper-posted replies are split (never cut)
|
|
691
|
+
* per the tone contract; document-sized replies attach as a file.
|
|
692
|
+
* Every enforcement failure fails OPEN — a kernel without the claim route or
|
|
693
|
+
* an unreachable upload endpoint must never produce a silent agent (#887).
|
|
491
694
|
*/
|
|
492
695
|
export const performRun = ({
|
|
493
696
|
instanceUrl,
|
|
@@ -502,6 +705,16 @@ export const performRun = ({
|
|
|
502
705
|
log = () => {},
|
|
503
706
|
onError,
|
|
504
707
|
setTimeoutImpl = setTimeout,
|
|
708
|
+
setIntervalImpl = setInterval,
|
|
709
|
+
clearIntervalImpl = clearInterval,
|
|
710
|
+
retryJitterRatio,
|
|
711
|
+
claimLeaseSeconds = 90,
|
|
712
|
+
cascadeCap = 3,
|
|
713
|
+
cascadeResetMs = 10 * 60 * 1000,
|
|
714
|
+
chatCharLimit = 400,
|
|
715
|
+
maxChatChunks = 3,
|
|
716
|
+
claimYieldDelayMs = 3000,
|
|
717
|
+
sleepImpl = (ms) => new Promise((resolve) => { setTimeout(resolve, ms); }),
|
|
505
718
|
}) => {
|
|
506
719
|
const client = createClient({ instance: instanceUrl, token });
|
|
507
720
|
let running = true;
|
|
@@ -512,6 +725,13 @@ export const performRun = ({
|
|
|
512
725
|
// reprovision-all; 5+ wastes rate-limit budget after the real-revoke case.
|
|
513
726
|
let consecutiveAuthErrors = 0;
|
|
514
727
|
const MAX_AUTH_ERRORS = 3;
|
|
728
|
+
let consecutiveSpawnFailures = 0;
|
|
729
|
+
const spawnJitterRatio = retryJitterRatio ?? spawnRetryJitter(agentName);
|
|
730
|
+
// Per-seat cascade state — lives with the process, like the session store.
|
|
731
|
+
// A wrapper restart forgets the streak; the decay window covers that gap.
|
|
732
|
+
const cascadeGovernor = createCascadeGovernor({ cap: cascadeCap, resetMs: cascadeResetMs });
|
|
733
|
+
// Fairness: recent broadcast-race winners start the next race from the back.
|
|
734
|
+
const claimHandicap = createClaimHandicap({ delayMs: claimYieldDelayMs });
|
|
515
735
|
|
|
516
736
|
// Adapters default `ctx.cwd` to this path. Node's child_process.spawn
|
|
517
737
|
// rejects with "spawn <bin> ENOENT" when cwd does not exist — same shape
|
|
@@ -527,15 +747,22 @@ export const performRun = ({
|
|
|
527
747
|
if (!prompt || !eventPodId) {
|
|
528
748
|
// No prompt, or nowhere to post the response — skip spawn entirely so
|
|
529
749
|
// we never consume a CLI turn for a message with no destination.
|
|
750
|
+
// Surfaced through onError, not just the log: a silently-acked event is
|
|
751
|
+
// indistinguishable from a swallowed wake (#896 — the pilot burned an
|
|
752
|
+
// evening on exactly this ambiguity). It is still acked (deliberate
|
|
753
|
+
// decline, not a retry), but the operator can now see the skip.
|
|
530
754
|
log(`[${event.type}] no prompt — no-op`);
|
|
531
|
-
|
|
755
|
+
onError?.(Object.assign(
|
|
756
|
+
new Error(
|
|
757
|
+
`${event.type} event ${event._id} was acked WITHOUT a spawn: `
|
|
758
|
+
+ `${!eventPodId ? 'no podId to post into' : 'payload carried no content and no messageId'}. `
|
|
759
|
+
+ 'If this wake mattered, its producer is sending an unusable payload.',
|
|
760
|
+
),
|
|
761
|
+
{ code: 'agent_event_skipped_no_prompt', eventId: event._id },
|
|
762
|
+
));
|
|
763
|
+
return { outcome: 'no_action', reason: 'no-prompt' };
|
|
532
764
|
}
|
|
533
765
|
|
|
534
|
-
const sessionId = getSession(agentName, eventPodId);
|
|
535
|
-
// ADR-005 §Memory bridge: read long_term before spawn, inject via ctx,
|
|
536
|
-
// and (if the adapter returns a summary) patch-sync back after.
|
|
537
|
-
const memoryLongTerm = await readLongTerm(client, { onError });
|
|
538
|
-
|
|
539
766
|
// Snapshot the pod's recent messages so that, after the spawn, we can
|
|
540
767
|
// tell whether the agent posted itself via commonly_post_message. If it
|
|
541
768
|
// did, its final CLI text is a narration/log — echoing it would duplicate
|
|
@@ -558,12 +785,106 @@ export const performRun = ({
|
|
|
558
785
|
// A consult request's final output is routed to the ask-response endpoint,
|
|
559
786
|
// never echoed into the pod. It therefore does not need pod-message
|
|
560
787
|
// snapshotting (and cannot be detected through that channel anyway).
|
|
788
|
+
// The snapshot doubles as the cascade governor's authorship source, so it
|
|
789
|
+
// runs BEFORE enforcement.
|
|
561
790
|
const shouldSnapshotMessages = event.type !== 'agent.ask';
|
|
562
791
|
const preSpawn = shouldSnapshotMessages ? await snapshotMessages() : null;
|
|
563
792
|
const preSpawnIds = preSpawn
|
|
564
793
|
? new Set(preSpawn.map((m) => String(m._id || m.id)))
|
|
565
794
|
: null;
|
|
566
795
|
|
|
796
|
+
// ── ADR-018 enforcement: cascade cap ────────────────────────────────────
|
|
797
|
+
// Refusing here is a deliberate, permanent decline (acked no_action): a
|
|
798
|
+
// capped agent-triggered event is exactly the traffic we want dropped.
|
|
799
|
+
// Human-triggered turns are never capped.
|
|
800
|
+
const trigger = classifyTrigger(event, preSpawn);
|
|
801
|
+
const admission = cascadeGovernor.admit(eventPodId, trigger);
|
|
802
|
+
if (!admission.allowed) {
|
|
803
|
+
log(
|
|
804
|
+
`[${event.type}] cascade cap: ${admission.streak} consecutive agent-triggered `
|
|
805
|
+
+ `turns in pod ${eventPodId} — standing down until a human speaks or the streak decays`,
|
|
806
|
+
);
|
|
807
|
+
return { outcome: 'no_action', reason: 'cascade-cap' };
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
// ── ADR-018 enforcement: claim-before-act ───────────────────────────────
|
|
811
|
+
// Losing a BROADCAST race is a complete turn (stand down, ack): the
|
|
812
|
+
// holder owns the conversation, and liveness on holder death comes from
|
|
813
|
+
// the HOLDER's own event staying unacked, not from ours. Losing while
|
|
814
|
+
// DIRECTLY ADDRESSED is different — a human chose this seat, and that
|
|
815
|
+
// outranks being beaten to a CAS: the seat still gets its turn, peer-aware
|
|
816
|
+
// (add your view only if materially different). Claim-route failures
|
|
817
|
+
// proceed unguarded — enforcement must never make an agent silent (#887).
|
|
818
|
+
let claimKeeper = null;
|
|
819
|
+
let peerFrame = null;
|
|
820
|
+
const claimMessageId = event.payload?.messageId;
|
|
821
|
+
if (claimMessageId && CLAIMABLE_EVENT_TYPES.has(event.type)) {
|
|
822
|
+
// Fairness: the winner of this pod's previous broadcast race enters the
|
|
823
|
+
// next one after a jittered delay. Everyone still claims — a hard
|
|
824
|
+
// cooldown could leave a message with NO claimant, which is #887
|
|
825
|
+
// self-inflicted — recent winners just start from the back.
|
|
826
|
+
if (event.type === 'message.posted') {
|
|
827
|
+
const yieldMs = claimHandicap.yieldDelayMs(eventPodId);
|
|
828
|
+
if (yieldMs > 0) {
|
|
829
|
+
log(`[${event.type}] yielding ${yieldMs}ms before claiming — won this pod's previous broadcast race`);
|
|
830
|
+
await sleepImpl(yieldMs);
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
claimKeeper = createClaimKeeper(client, {
|
|
834
|
+
messageId: claimMessageId,
|
|
835
|
+
podId: eventPodId,
|
|
836
|
+
leaseSeconds: claimLeaseSeconds,
|
|
837
|
+
log: (line) => log(`[${event.type}] ${line}`),
|
|
838
|
+
setIntervalImpl,
|
|
839
|
+
clearIntervalImpl,
|
|
840
|
+
});
|
|
841
|
+
const claim = await claimKeeper.acquire();
|
|
842
|
+
if (!claim.claimed && !claim.failOpen) {
|
|
843
|
+
if (ADDRESSED_EVENT_TYPES.has(event.type)) {
|
|
844
|
+
log(
|
|
845
|
+
`[${event.type}] message ${claimMessageId} held by ${claim.holder} — `
|
|
846
|
+
+ 'proceeding peer-aware (this seat was directly addressed)',
|
|
847
|
+
);
|
|
848
|
+
peerFrame = peerHoldsFrame(claim.holder, claimMessageId);
|
|
849
|
+
claimKeeper = null; // nothing held: no renewal, no release, no isLost gate
|
|
850
|
+
} else {
|
|
851
|
+
claimHandicap.recordLoss(eventPodId);
|
|
852
|
+
log(`[${event.type}] message ${claimMessageId} already claimed by ${claim.holder} — standing down`);
|
|
853
|
+
return { outcome: 'no_action', reason: 'claim-held' };
|
|
854
|
+
}
|
|
855
|
+
} else if (claim.claimed) {
|
|
856
|
+
if (event.type === 'message.posted') claimHandicap.recordWin(eventPodId);
|
|
857
|
+
claimKeeper.startRenewal();
|
|
858
|
+
} else {
|
|
859
|
+
log(`[${event.type}] claim unavailable (${claim.error?.message || 'unknown error'}) — proceeding unguarded`);
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
try {
|
|
864
|
+
return await runTurn({
|
|
865
|
+
event,
|
|
866
|
+
eventPodId,
|
|
867
|
+
prompt: peerFrame ? `${peerFrame}\n\n${prompt}` : prompt,
|
|
868
|
+
preSpawnIds,
|
|
869
|
+
snapshotMessages,
|
|
870
|
+
claimKeeper,
|
|
871
|
+
trigger,
|
|
872
|
+
});
|
|
873
|
+
} finally {
|
|
874
|
+
await claimKeeper?.release();
|
|
875
|
+
}
|
|
876
|
+
};
|
|
877
|
+
|
|
878
|
+
// The spawn → post-decision half of a turn, split out so the claim release
|
|
879
|
+
// above is a plain finally instead of threading through every return path.
|
|
880
|
+
const runTurn = async ({
|
|
881
|
+
event, eventPodId, prompt, preSpawnIds, snapshotMessages, claimKeeper, trigger,
|
|
882
|
+
}) => {
|
|
883
|
+
const sessionId = getSession(agentName, eventPodId);
|
|
884
|
+
// ADR-005 §Memory bridge: read long_term before spawn, inject via ctx,
|
|
885
|
+
// and (if the adapter returns a summary) patch-sync back after.
|
|
886
|
+
const memoryLongTerm = await readLongTerm(client, { onError });
|
|
887
|
+
|
|
567
888
|
log(`[${event.type}] spawning ${adapter.name}`);
|
|
568
889
|
const result = await adapter.spawn(prompt, {
|
|
569
890
|
sessionId,
|
|
@@ -669,12 +990,31 @@ export const performRun = ({
|
|
|
669
990
|
+ `(avoids double-post; matched message ${suppressedBy.id} by ${suppressedBy.author} `
|
|
670
991
|
+ `via ${suppressedBy.basis})`,
|
|
671
992
|
);
|
|
993
|
+
} else if (claimKeeper?.isLost()) {
|
|
994
|
+
// ADR-018 D3: the lease lapsed mid-turn and a peer re-won the message —
|
|
995
|
+
// that peer owns the conversation now, so posting would recreate the
|
|
996
|
+
// two-agents-one-message crossing the claim exists to prevent.
|
|
997
|
+
log(
|
|
998
|
+
`[${event.type}] stood down — claim lost mid-turn to ${claimKeeper.getHolder()}; `
|
|
999
|
+
+ `reply suppressed (${Buffer.byteLength(replyText)} bytes not posted)`,
|
|
1000
|
+
);
|
|
672
1001
|
} else {
|
|
673
|
-
|
|
674
|
-
|
|
1002
|
+
// ADR-018 length gate: the tone contract is enforced here, where the
|
|
1003
|
+
// wrapper is the one posting. Split, attach — never truncate.
|
|
1004
|
+
const delivery = await deliverChatReply({
|
|
1005
|
+
client,
|
|
1006
|
+
podId: eventPodId,
|
|
1007
|
+
text: replyText,
|
|
1008
|
+
limit: chatCharLimit,
|
|
1009
|
+
maxChunks: maxChatChunks,
|
|
1010
|
+
uploadName: `${agentName}-reply-${event._id}.md`,
|
|
1011
|
+
log: (line) => log(`[${event.type}] ${line}`),
|
|
675
1012
|
});
|
|
676
1013
|
delivered = true;
|
|
677
|
-
log(
|
|
1014
|
+
log(
|
|
1015
|
+
`[${event.type}] posted ${Buffer.byteLength(replyText)} bytes as `
|
|
1016
|
+
+ `${delivery.messages} message${delivery.messages === 1 ? '' : 's'} (${delivery.mode})`,
|
|
1017
|
+
);
|
|
678
1018
|
}
|
|
679
1019
|
if (result.memorySummary) {
|
|
680
1020
|
try {
|
|
@@ -688,11 +1028,16 @@ export const performRun = ({
|
|
|
688
1028
|
onError?.(new Error(`memory sync failed: ${err.message}`, { cause: err }));
|
|
689
1029
|
}
|
|
690
1030
|
}
|
|
1031
|
+
// The turn completed — count it toward (or reset) the pod's cascade
|
|
1032
|
+
// streak. Recording only on completion means a spawn failure that gets
|
|
1033
|
+
// redelivered never double-counts toward the cap.
|
|
1034
|
+
cascadeGovernor.record(eventPodId, trigger);
|
|
691
1035
|
return { outcome: delivered ? 'posted' : 'no_action' };
|
|
692
1036
|
};
|
|
693
1037
|
|
|
694
1038
|
const tick = async () => {
|
|
695
1039
|
if (!running) return;
|
|
1040
|
+
let nextPollDelayMs = intervalMs;
|
|
696
1041
|
try {
|
|
697
1042
|
const { events = [] } = await client.get('/api/agents/runtime/events', {
|
|
698
1043
|
agentName, instanceId, limit: 10,
|
|
@@ -705,15 +1050,49 @@ export const performRun = ({
|
|
|
705
1050
|
result = { outcome: 'no_action', reason: 'duplicate-delivery' };
|
|
706
1051
|
log(`[${event.type}] duplicate delivery ${event._id} — skipping spawn and re-acking`);
|
|
707
1052
|
} else {
|
|
1053
|
+
const eventWillSpawn = Boolean(extractPrompt(event) && (event.podId || podId));
|
|
708
1054
|
try {
|
|
709
1055
|
result = await processEvent(event);
|
|
710
1056
|
} catch (err) {
|
|
711
|
-
//
|
|
712
|
-
//
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
1057
|
+
// Do not record or ack: the kernel must retain the event for
|
|
1058
|
+
// at-least-once delivery. Stop this fetched batch immediately —
|
|
1059
|
+
// continuing could launch every one of the 10 returned events
|
|
1060
|
+
// into the same provider outage before the next poll (#782).
|
|
1061
|
+
consecutiveSpawnFailures += 1;
|
|
1062
|
+
const retry = spawnRetryPolicy({
|
|
1063
|
+
error: err,
|
|
1064
|
+
consecutiveFailures: consecutiveSpawnFailures,
|
|
1065
|
+
intervalMs,
|
|
1066
|
+
jitterRatio: spawnJitterRatio,
|
|
1067
|
+
});
|
|
1068
|
+
nextPollDelayMs = retry.delayMs;
|
|
1069
|
+
const retryIn = formatRetryDelay(retry.delayMs);
|
|
1070
|
+
const state = retry.circuitOpen ? 'circuit open' : 'retry scheduled';
|
|
1071
|
+
const wrapped = new Error(
|
|
1072
|
+
`${event.type} processing failed (${retry.failureClass}; `
|
|
1073
|
+
+ `${consecutiveSpawnFailures} consecutive) — event ${event._id} remains unacked; `
|
|
1074
|
+
+ `${state}, next probe in ${retryIn}: ${err.message}`,
|
|
1075
|
+
{ cause: err },
|
|
1076
|
+
);
|
|
1077
|
+
Object.assign(wrapped, {
|
|
1078
|
+
code: 'agent_spawn_retry_scheduled',
|
|
1079
|
+
failureClass: retry.failureClass,
|
|
1080
|
+
consecutiveFailures: consecutiveSpawnFailures,
|
|
1081
|
+
retryAfterMs: retry.delayMs,
|
|
1082
|
+
circuitOpen: retry.circuitOpen,
|
|
1083
|
+
eventId: event._id,
|
|
1084
|
+
});
|
|
1085
|
+
log(`[${event.type}] ${wrapped.message}`);
|
|
1086
|
+
onError?.(wrapped);
|
|
1087
|
+
break;
|
|
716
1088
|
}
|
|
1089
|
+
// Only a completed model turn proves the local runtime and delivery
|
|
1090
|
+
// path recovered. A malformed/no-destination event is still acked,
|
|
1091
|
+
// but must not erase the failure streak without exercising either —
|
|
1092
|
+
// and neither may an enforcement stand-down (cascade cap, lost
|
|
1093
|
+
// claim), which returns before any spawn happens.
|
|
1094
|
+
const stoodDown = result?.reason === 'cascade-cap' || result?.reason === 'claim-held';
|
|
1095
|
+
if (eventWillSpawn && !stoodDown) consecutiveSpawnFailures = 0;
|
|
717
1096
|
// Record after successful processing but before ack. If the ack
|
|
718
1097
|
// fails, the next delivery is skipped and re-acked instead of
|
|
719
1098
|
// burning a second model turn for work that already completed.
|
|
@@ -739,7 +1118,7 @@ export const performRun = ({
|
|
|
739
1118
|
}
|
|
740
1119
|
onError?.(err);
|
|
741
1120
|
}
|
|
742
|
-
if (running) setTimeoutImpl(tick,
|
|
1121
|
+
if (running) setTimeoutImpl(tick, nextPollDelayMs);
|
|
743
1122
|
};
|
|
744
1123
|
|
|
745
1124
|
tick();
|
|
@@ -1260,11 +1639,33 @@ Docs:
|
|
|
1260
1639
|
.command('run <name>')
|
|
1261
1640
|
.description('Run the local-CLI wrapper loop for an attached agent')
|
|
1262
1641
|
.option('--interval <ms>', 'Poll interval in ms', '5000')
|
|
1642
|
+
.option('--adapter <name>', 'CLI to wrap on first-run bootstrap (claude|codex); ignored when a token file already exists')
|
|
1263
1643
|
.action(async (name, opts) => {
|
|
1264
|
-
|
|
1644
|
+
let record = loadAgentToken(name);
|
|
1265
1645
|
if (!record) {
|
|
1266
|
-
|
|
1267
|
-
|
|
1646
|
+
// First run on this machine: the BYO connect page hands out env vars,
|
|
1647
|
+
// not a token file — bootstrap the record from them (#913).
|
|
1648
|
+
try {
|
|
1649
|
+
record = await bootstrapAgentRecordFromEnv({
|
|
1650
|
+
name,
|
|
1651
|
+
adapterOverride: opts.adapter || null,
|
|
1652
|
+
log: (line) => console.log(`[${name}] ${line}`),
|
|
1653
|
+
});
|
|
1654
|
+
} catch (err) {
|
|
1655
|
+
console.error(err.message);
|
|
1656
|
+
process.exit(1);
|
|
1657
|
+
}
|
|
1658
|
+
if (record) {
|
|
1659
|
+
saveAgentToken(record.agentName, record);
|
|
1660
|
+
console.log(`[${name}] bootstrapped ${tokenFile(record.agentName)} from COMMONLY_AGENT_TOKEN (adapter: ${record.adapter})`);
|
|
1661
|
+
} else {
|
|
1662
|
+
console.error(
|
|
1663
|
+
`No token for '${name}'. Either export COMMONLY_API_URL + COMMONLY_AGENT_TOKEN`
|
|
1664
|
+
+ ` (shown on the Connect your own agent page) and re-run, or:`
|
|
1665
|
+
+ ` commonly agent attach <adapter> --pod <podId> --name ${name}`,
|
|
1666
|
+
);
|
|
1667
|
+
process.exit(1);
|
|
1668
|
+
}
|
|
1268
1669
|
}
|
|
1269
1670
|
|
|
1270
1671
|
const adapter = getAdapter(record.adapter);
|
|
@@ -244,7 +244,16 @@ const runClaude = ({ cmd, args, cwd, env, timeoutMs, spawnImpl = childSpawn }) =
|
|
|
244
244
|
proc.on('close', (code) => {
|
|
245
245
|
clearTimeout(timer);
|
|
246
246
|
if (timedOut) return reject(new Error(`claude timed out after ${timeoutMs}ms`));
|
|
247
|
-
if (code !== 0)
|
|
247
|
+
if (code !== 0) {
|
|
248
|
+
// Report stdout too, not just stderr. In `-p` mode claude writes terminal
|
|
249
|
+
// conditions (usage limits especially) to stdout and exits non-zero with
|
|
250
|
+
// stderr empty — 361 consecutive failures on 2026-08-03 carried no reason
|
|
251
|
+
// at all because of this. It is not only a diagnosability problem: the
|
|
252
|
+
// circuit breaker classifies from the error message, so a blank message
|
|
253
|
+
// downgrades a hard quota failure to RUNTIME and its shortest backoff.
|
|
254
|
+
const detail = [stderr.trim(), stdout.trim()].filter(Boolean).join(' | ');
|
|
255
|
+
return reject(new Error(`claude exited with code ${code}: ${detail.slice(0, 2000)}`));
|
|
256
|
+
}
|
|
248
257
|
resolve(stdout);
|
|
249
258
|
});
|
|
250
259
|
});
|
package/src/lib/api.js
CHANGED
|
@@ -48,7 +48,30 @@ export const createClient = ({ instance = null, token = null } = {}) => {
|
|
|
48
48
|
headers: headers(authToken),
|
|
49
49
|
}).then(handleResponse);
|
|
50
50
|
|
|
51
|
-
|
|
51
|
+
// Multipart upload via native FormData/Blob (Node 18+) — no runtime deps.
|
|
52
|
+
// Content-Type is deliberately NOT set: fetch writes the multipart boundary.
|
|
53
|
+
const upload = (path, {
|
|
54
|
+
fileBuffer, fileName, contentType, fileField = 'file', fields = {},
|
|
55
|
+
}) => {
|
|
56
|
+
const form = new FormData();
|
|
57
|
+
form.append(
|
|
58
|
+
fileField,
|
|
59
|
+
new Blob([fileBuffer], { type: contentType || 'application/octet-stream' }),
|
|
60
|
+
fileName,
|
|
61
|
+
);
|
|
62
|
+
for (const [k, v] of Object.entries(fields)) {
|
|
63
|
+
if (v !== undefined && v !== null) form.append(k, String(v));
|
|
64
|
+
}
|
|
65
|
+
return fetch(`${baseUrl}${path}`, {
|
|
66
|
+
method: 'POST',
|
|
67
|
+
headers: authToken ? { Authorization: `Bearer ${authToken}` } : {},
|
|
68
|
+
body: form,
|
|
69
|
+
}).then(handleResponse);
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
return {
|
|
73
|
+
get, post, del, upload, baseUrl,
|
|
74
|
+
};
|
|
52
75
|
};
|
|
53
76
|
|
|
54
77
|
// Convenience: login doesn't need a token
|
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wrapper-side enforcement (ADR-018 D3, "our drivers" row).
|
|
3
|
+
*
|
|
4
|
+
* The 2026-08-11 pilot proved that advisory guidance loses to task gravity:
|
|
5
|
+
* the tone contract was in every seat's tool descriptions and the review
|
|
6
|
+
* median still came out at 3,614 characters, with zero claims taken and a
|
|
7
|
+
* mention cascade that needed a manual wrapper kill. This module is the
|
|
8
|
+
* deterministic half — the wrapper enforces what the contract can only ask:
|
|
9
|
+
*
|
|
10
|
+
* - claim-before-act + stand-down (createClaimKeeper)
|
|
11
|
+
* - per-seat cascade damping (createCascadeGovernor, classifyTrigger)
|
|
12
|
+
* - post-time length gate (splitForChat, deliverChatReply)
|
|
13
|
+
*
|
|
14
|
+
* One rule overrides everything here: enforcement must never convert an
|
|
15
|
+
* infrastructure failure into agent silence (#887 class). Every network or
|
|
16
|
+
* server error fails OPEN — the turn proceeds unguarded and says so in the
|
|
17
|
+
* log. Only an explicit "someone else holds it" or "cascade cap reached"
|
|
18
|
+
* stands the agent down, and both are logged with the reason.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
// Event types whose payload.messageId identifies a claimable trigger message.
|
|
22
|
+
// first_contact is deliberately absent: the welcome wake targets exactly one
|
|
23
|
+
// agent, so there is nothing to contend for. Heartbeats have no message at
|
|
24
|
+
// all, and agent.ask routes privately.
|
|
25
|
+
export const CLAIMABLE_EVENT_TYPES = new Set([
|
|
26
|
+
'chat.mention',
|
|
27
|
+
'message.posted',
|
|
28
|
+
'dm.message',
|
|
29
|
+
]);
|
|
30
|
+
|
|
31
|
+
// ── trigger classification ──────────────────────────────────────────────────
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Who authored the message that woke us — 'agent', 'human', or 'unknown'?
|
|
35
|
+
*
|
|
36
|
+
* Two signals, in order of reliability:
|
|
37
|
+
* 1. payload.dmKind — the kernel stamps DM wakes 'agent-agent'/'user-agent'.
|
|
38
|
+
* 2. The trigger message's isBot flag, looked up by payload.messageId in the
|
|
39
|
+
* pre-spawn snapshot the run loop already fetches for echo suppression.
|
|
40
|
+
*
|
|
41
|
+
* 'unknown' is the fail-open verdict: it neither counts toward the cascade
|
|
42
|
+
* cap nor resets it. In a live cascade the trigger message is seconds old and
|
|
43
|
+
* always inside the snapshot window, so cascades classify reliably; a message
|
|
44
|
+
* that has already scrolled out of the window is not cascade tempo.
|
|
45
|
+
*/
|
|
46
|
+
export const classifyTrigger = (event, recentMessages) => {
|
|
47
|
+
const p = event?.payload || {};
|
|
48
|
+
if (p.dmKind === 'agent-agent') return 'agent';
|
|
49
|
+
if (p.dmKind === 'user-agent') return 'human';
|
|
50
|
+
if (!p.messageId || !Array.isArray(recentMessages)) return 'unknown';
|
|
51
|
+
const trigger = recentMessages.find(
|
|
52
|
+
(m) => String(m._id || m.id) === String(p.messageId),
|
|
53
|
+
);
|
|
54
|
+
if (!trigger || typeof trigger.isBot !== 'boolean') return 'unknown';
|
|
55
|
+
return trigger.isBot ? 'agent' : 'human';
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
// ── cascade governor ────────────────────────────────────────────────────────
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Per-pod damping of agent→agent retrigger chains.
|
|
62
|
+
*
|
|
63
|
+
* The pilot's failure shape: an agent's own posts kept waking it (via another
|
|
64
|
+
* agent's replies) until the operator killed the wrapper at round 3. The
|
|
65
|
+
* governor counts CONSECUTIVE agent-triggered turns per pod; at `cap` it
|
|
66
|
+
* refuses further agent-triggered turns until a human-triggered turn resets
|
|
67
|
+
* the streak or `resetMs` passes with no agent-triggered turn (so a damped
|
|
68
|
+
* pod recovers on its own — a legitimate a2a handoff an hour later must not
|
|
69
|
+
* inherit a stale cap).
|
|
70
|
+
*
|
|
71
|
+
* Split into admit/record so a spawn that fails (and will be redelivered)
|
|
72
|
+
* never double-counts: admit() only reads, record() runs after a turn
|
|
73
|
+
* actually completed. Human-triggered turns are always admitted.
|
|
74
|
+
*/
|
|
75
|
+
export const createCascadeGovernor = ({
|
|
76
|
+
cap = 3,
|
|
77
|
+
resetMs = 10 * 60 * 1000,
|
|
78
|
+
now = Date.now,
|
|
79
|
+
} = {}) => {
|
|
80
|
+
const pods = new Map(); // podId -> { streak, lastAgentTurnAt }
|
|
81
|
+
|
|
82
|
+
const stateFor = (podId) => {
|
|
83
|
+
const s = pods.get(podId) || { streak: 0, lastAgentTurnAt: 0 };
|
|
84
|
+
if (s.streak > 0 && now() - s.lastAgentTurnAt > resetMs) {
|
|
85
|
+
return { streak: 0, lastAgentTurnAt: 0 };
|
|
86
|
+
}
|
|
87
|
+
return s;
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
return {
|
|
91
|
+
admit(podId, trigger) {
|
|
92
|
+
if (trigger !== 'agent') return { allowed: true, streak: 0 };
|
|
93
|
+
const s = stateFor(podId);
|
|
94
|
+
return { allowed: s.streak < cap, streak: s.streak };
|
|
95
|
+
},
|
|
96
|
+
record(podId, trigger) {
|
|
97
|
+
if (trigger === 'human') {
|
|
98
|
+
pods.set(podId, { streak: 0, lastAgentTurnAt: 0 });
|
|
99
|
+
} else if (trigger === 'agent') {
|
|
100
|
+
const s = stateFor(podId);
|
|
101
|
+
pods.set(podId, { streak: s.streak + 1, lastAgentTurnAt: now() });
|
|
102
|
+
}
|
|
103
|
+
// 'unknown' is neutral: no count, no reset.
|
|
104
|
+
},
|
|
105
|
+
};
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
// ── claim fairness ──────────────────────────────────────────────────────────
|
|
109
|
+
|
|
110
|
+
// Direct-address event types: the seat was NAMED (explicit @, implicit human
|
|
111
|
+
// reply, or DM routing). A lost claim on these does not silence the seat —
|
|
112
|
+
// being chosen by a human outranks being beaten to a CAS. Broadcast wakes
|
|
113
|
+
// (message.posted) are the opposite: nobody asked for THIS seat, so a lost
|
|
114
|
+
// race is a free stand-down.
|
|
115
|
+
export const ADDRESSED_EVENT_TYPES = new Set(['chat.mention', 'thread.mention', 'dm.message']);
|
|
116
|
+
|
|
117
|
+
// Frame prepended when an ADDRESSED seat lost the claim race: it still gets
|
|
118
|
+
// its turn, but knows a peer is (probably) already answering — the bar for
|
|
119
|
+
// posting rises from "have something to say" to "have something DIFFERENT".
|
|
120
|
+
export const peerHoldsFrame = (holder, messageId) => (
|
|
121
|
+
`[Claim notice: @${holder} holds message ${messageId} and is likely responding. `
|
|
122
|
+
+ 'You were directly addressed, so you still get this turn — but add your view ONLY '
|
|
123
|
+
+ 'if it is materially different from what they would cover; otherwise return NO_REPLY.]'
|
|
124
|
+
);
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Win-weighted claim delay — the "cooldown" that keeps one fast seat from
|
|
128
|
+
* monopolising a pod's broadcast wakes without ever risking that NOBODY
|
|
129
|
+
* claims. A seat that won its previous broadcast race in a pod waits a small
|
|
130
|
+
* jittered delay before entering the next one; everyone still claims, recent
|
|
131
|
+
* winners just start from the back. A loss (or `windowMs` of quiet) clears
|
|
132
|
+
* the handicap — you are only the monopolist while you are actually winning.
|
|
133
|
+
*
|
|
134
|
+
* Deliberately NOT a hard cooldown: with all seats abstaining, a message
|
|
135
|
+
* goes unhandled — the #887 shape again, self-inflicted.
|
|
136
|
+
*/
|
|
137
|
+
export const createClaimHandicap = ({
|
|
138
|
+
delayMs = 3000,
|
|
139
|
+
jitterMs = 1000,
|
|
140
|
+
windowMs = 5 * 60 * 1000,
|
|
141
|
+
now = Date.now,
|
|
142
|
+
random = Math.random,
|
|
143
|
+
} = {}) => {
|
|
144
|
+
const wins = new Map(); // podId -> lastBroadcastWinAt
|
|
145
|
+
|
|
146
|
+
return {
|
|
147
|
+
recordWin(podId) {
|
|
148
|
+
wins.set(podId, now());
|
|
149
|
+
},
|
|
150
|
+
recordLoss(podId) {
|
|
151
|
+
wins.delete(podId);
|
|
152
|
+
},
|
|
153
|
+
yieldDelayMs(podId) {
|
|
154
|
+
const at = wins.get(podId);
|
|
155
|
+
// Explicit undefined check: a win recorded at clock 0 is still a win
|
|
156
|
+
// (a falsy-timestamp `!at` here silently disabled the handicap for the
|
|
157
|
+
// first test clock tick — caught by the unit suite).
|
|
158
|
+
if (at === undefined || now() - at > windowMs) return 0;
|
|
159
|
+
return delayMs + Math.floor(random() * jitterMs);
|
|
160
|
+
},
|
|
161
|
+
};
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
// ── claim keeper ────────────────────────────────────────────────────────────
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* One claim lifecycle for one event: acquire → renew while the CLI turn runs
|
|
168
|
+
* → release (or discover mid-turn loss and stand down at post time).
|
|
169
|
+
*
|
|
170
|
+
* Acquire outcomes:
|
|
171
|
+
* { claimed: true } — we hold the lease; start renewal.
|
|
172
|
+
* { claimed: false, holder } — someone else holds it; STAND DOWN.
|
|
173
|
+
* { claimed: false, failOpen: true } — claim route unavailable (older
|
|
174
|
+
* server, network, 403 on a stale install); proceed UNGUARDED. The
|
|
175
|
+
* alternative turns a kernel deploy gap into a silent agent.
|
|
176
|
+
*
|
|
177
|
+
* Renewal reuses the same POST (a holder wins against itself — that IS
|
|
178
|
+
* renewal, per messageClaimService). A renewal that comes back claimed:false
|
|
179
|
+
* means our lease lapsed (laptop slept, turn ran long) and a peer re-won:
|
|
180
|
+
* mark lost so the run loop suppresses the wrapper post. Transient renewal
|
|
181
|
+
* errors are ignored — the current lease may still be live, and the next
|
|
182
|
+
* tick retries.
|
|
183
|
+
*/
|
|
184
|
+
export const createClaimKeeper = (client, {
|
|
185
|
+
messageId,
|
|
186
|
+
podId,
|
|
187
|
+
leaseSeconds = 90,
|
|
188
|
+
log = () => {},
|
|
189
|
+
setIntervalImpl = setInterval,
|
|
190
|
+
clearIntervalImpl = clearInterval,
|
|
191
|
+
}) => {
|
|
192
|
+
const path = `/api/agents/runtime/messages/${encodeURIComponent(messageId)}/claim`;
|
|
193
|
+
let acquired = false;
|
|
194
|
+
let lost = false;
|
|
195
|
+
let holder = null;
|
|
196
|
+
let timer = null;
|
|
197
|
+
|
|
198
|
+
const holderLabel = (res) => {
|
|
199
|
+
if (!res?.claimedBy) return 'another agent';
|
|
200
|
+
const instance = res.instanceId && res.instanceId !== 'default' ? `:${res.instanceId}` : '';
|
|
201
|
+
return `${res.claimedBy}${instance}`;
|
|
202
|
+
};
|
|
203
|
+
|
|
204
|
+
const stopRenewal = () => {
|
|
205
|
+
if (timer) {
|
|
206
|
+
clearIntervalImpl(timer);
|
|
207
|
+
timer = null;
|
|
208
|
+
}
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
return {
|
|
212
|
+
async acquire() {
|
|
213
|
+
try {
|
|
214
|
+
const res = await client.post(path, { podId, leaseSeconds });
|
|
215
|
+
if (res?.claimed) {
|
|
216
|
+
acquired = true;
|
|
217
|
+
return { claimed: true, expiresAt: res.expiresAt };
|
|
218
|
+
}
|
|
219
|
+
holder = holderLabel(res);
|
|
220
|
+
return { claimed: false, holder };
|
|
221
|
+
} catch (err) {
|
|
222
|
+
return { claimed: false, failOpen: true, error: err };
|
|
223
|
+
}
|
|
224
|
+
},
|
|
225
|
+
|
|
226
|
+
startRenewal() {
|
|
227
|
+
if (!acquired || timer) return;
|
|
228
|
+
timer = setIntervalImpl(async () => {
|
|
229
|
+
try {
|
|
230
|
+
const res = await client.post(path, { podId, leaseSeconds });
|
|
231
|
+
if (!res?.claimed) {
|
|
232
|
+
lost = true;
|
|
233
|
+
holder = holderLabel(res);
|
|
234
|
+
stopRenewal();
|
|
235
|
+
log(`claim on message ${messageId} lost mid-turn to ${holder} — standing down at post time`);
|
|
236
|
+
}
|
|
237
|
+
} catch {
|
|
238
|
+
// Transient renewal failure — the held lease may still be live;
|
|
239
|
+
// retry on the next tick rather than standing down on a blip.
|
|
240
|
+
}
|
|
241
|
+
}, Math.max(5000, (leaseSeconds * 1000) / 2));
|
|
242
|
+
if (timer && typeof timer.unref === 'function') timer.unref();
|
|
243
|
+
},
|
|
244
|
+
|
|
245
|
+
async release() {
|
|
246
|
+
stopRenewal();
|
|
247
|
+
if (!acquired || lost) return;
|
|
248
|
+
try {
|
|
249
|
+
await client.del(path);
|
|
250
|
+
} catch {
|
|
251
|
+
// Best-effort: a miss just means the lease already expired.
|
|
252
|
+
}
|
|
253
|
+
},
|
|
254
|
+
|
|
255
|
+
isLost: () => lost,
|
|
256
|
+
getHolder: () => holder,
|
|
257
|
+
};
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
// ── post-time length gate ───────────────────────────────────────────────────
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Split chat text into tone-contract-sized messages without ever cutting
|
|
264
|
+
* content. Boundaries in preference order: fenced code blocks stay whole
|
|
265
|
+
* (atomic — an oversized fence becomes one oversized message rather than a
|
|
266
|
+
* broken pair), then paragraphs, then sentences, then words. Greedy packing
|
|
267
|
+
* rejoins small pieces so two short paragraphs share one message.
|
|
268
|
+
*/
|
|
269
|
+
export const splitForChat = (text, { limit = 400 } = {}) => {
|
|
270
|
+
const trimmed = String(text || '').trim();
|
|
271
|
+
if (!trimmed) return [];
|
|
272
|
+
if (trimmed.length <= limit) return [trimmed];
|
|
273
|
+
|
|
274
|
+
// Pass 1: carve into blocks — fences atomic, prose split by paragraph.
|
|
275
|
+
const blocks = [];
|
|
276
|
+
let prose = [];
|
|
277
|
+
const flushProse = () => {
|
|
278
|
+
const joined = prose.join('\n');
|
|
279
|
+
prose = [];
|
|
280
|
+
for (const para of joined.split(/\n{2,}/)) {
|
|
281
|
+
if (para.trim()) blocks.push({ text: para.trim(), atomic: false });
|
|
282
|
+
}
|
|
283
|
+
};
|
|
284
|
+
const lines = trimmed.split('\n');
|
|
285
|
+
let i = 0;
|
|
286
|
+
while (i < lines.length) {
|
|
287
|
+
const fence = lines[i].match(/^(```|~~~)/)?.[1];
|
|
288
|
+
if (fence) {
|
|
289
|
+
flushProse();
|
|
290
|
+
const fenced = [lines[i]];
|
|
291
|
+
i += 1;
|
|
292
|
+
while (i < lines.length && !lines[i].startsWith(fence)) {
|
|
293
|
+
fenced.push(lines[i]);
|
|
294
|
+
i += 1;
|
|
295
|
+
}
|
|
296
|
+
if (i < lines.length) {
|
|
297
|
+
fenced.push(lines[i]); // closing fence
|
|
298
|
+
i += 1;
|
|
299
|
+
}
|
|
300
|
+
blocks.push({ text: fenced.join('\n'), atomic: true });
|
|
301
|
+
} else {
|
|
302
|
+
prose.push(lines[i]);
|
|
303
|
+
i += 1;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
flushProse();
|
|
307
|
+
|
|
308
|
+
// Pass 2: split oversized prose blocks at sentence then word boundaries.
|
|
309
|
+
const units = [];
|
|
310
|
+
for (const block of blocks) {
|
|
311
|
+
if (block.atomic || block.text.length <= limit) {
|
|
312
|
+
units.push(block.text);
|
|
313
|
+
continue;
|
|
314
|
+
}
|
|
315
|
+
let piece = '';
|
|
316
|
+
const flushPiece = () => {
|
|
317
|
+
if (piece.trim()) units.push(piece.trim());
|
|
318
|
+
piece = '';
|
|
319
|
+
};
|
|
320
|
+
for (const sentence of block.text.split(/(?<=[.!?])\s+/)) {
|
|
321
|
+
if (sentence.length > limit) {
|
|
322
|
+
flushPiece();
|
|
323
|
+
let run = '';
|
|
324
|
+
for (const word of sentence.split(/\s+/)) {
|
|
325
|
+
if (run && `${run} ${word}`.length > limit) {
|
|
326
|
+
units.push(run);
|
|
327
|
+
run = word;
|
|
328
|
+
} else {
|
|
329
|
+
run = run ? `${run} ${word}` : word;
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
if (run) units.push(run); // a single over-limit word (URL) posts whole
|
|
333
|
+
} else if (piece && `${piece} ${sentence}`.length > limit) {
|
|
334
|
+
flushPiece();
|
|
335
|
+
piece = sentence;
|
|
336
|
+
} else {
|
|
337
|
+
piece = piece ? `${piece} ${sentence}` : sentence;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
flushPiece();
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
// Pass 3: greedy packing back up to the limit.
|
|
344
|
+
const chunks = [];
|
|
345
|
+
let acc = '';
|
|
346
|
+
for (const unit of units) {
|
|
347
|
+
const joined = acc ? `${acc}\n\n${unit}` : unit;
|
|
348
|
+
if (joined.length <= limit) {
|
|
349
|
+
acc = joined;
|
|
350
|
+
} else {
|
|
351
|
+
if (acc) chunks.push(acc);
|
|
352
|
+
acc = unit;
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
if (acc) chunks.push(acc);
|
|
356
|
+
return chunks;
|
|
357
|
+
};
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* Deliver a wrapper-posted reply under the tone contract, deterministically:
|
|
361
|
+
*
|
|
362
|
+
* fits in one message → post as-is
|
|
363
|
+
* splits into ≤ maxChunks → post the chunks in order ("two short
|
|
364
|
+
* messages beat one wall")
|
|
365
|
+
* longer than a split answer → it is a document, not a message: upload
|
|
366
|
+
* the FULL text as a file and post one
|
|
367
|
+
* message — the reply's own opening plus
|
|
368
|
+
* the file card. Nothing is cut; the file
|
|
369
|
+
* holds everything.
|
|
370
|
+
*
|
|
371
|
+
* If the upload fails (older server, network), fall back to posting every
|
|
372
|
+
* chunk: a message flood is a tone violation, silence or truncation is a
|
|
373
|
+
* correctness violation, and the contract itself ranks content above tone
|
|
374
|
+
* ("NEVER hit that by cutting content").
|
|
375
|
+
*/
|
|
376
|
+
export const deliverChatReply = async ({
|
|
377
|
+
client,
|
|
378
|
+
podId,
|
|
379
|
+
text,
|
|
380
|
+
limit = 400,
|
|
381
|
+
maxChunks = 3,
|
|
382
|
+
attachThreshold = 800,
|
|
383
|
+
uploadName = 'reply.md',
|
|
384
|
+
log = () => {},
|
|
385
|
+
}) => {
|
|
386
|
+
const messagesPath = `/api/agents/runtime/pods/${podId}/messages`;
|
|
387
|
+
const chunks = splitForChat(text, { limit });
|
|
388
|
+
// An atomic unit (a fenced block, an unbreakable word-run) can exceed the
|
|
389
|
+
// limit by construction — splitForChat keeps it whole rather than breaking
|
|
390
|
+
// its rendering. The tone contract's own rule covers it: over ~800 chars of
|
|
391
|
+
// ONE indivisible thing is a document, not a message — attach it. Without
|
|
392
|
+
// this check a 659-char fence rode the single/split branches straight past
|
|
393
|
+
// the gate (found by the fleet's implementation audit, Sharpen msg 53018).
|
|
394
|
+
const hasIndivisibleOversize = chunks.some((c) => c.length > attachThreshold);
|
|
395
|
+
if (chunks.length <= 1 && !hasIndivisibleOversize) {
|
|
396
|
+
await client.post(messagesPath, { content: chunks[0] ?? text });
|
|
397
|
+
return { mode: 'single', messages: 1 };
|
|
398
|
+
}
|
|
399
|
+
if (chunks.length <= maxChunks && !hasIndivisibleOversize) {
|
|
400
|
+
for (const chunk of chunks) {
|
|
401
|
+
// eslint-disable-next-line no-await-in-loop
|
|
402
|
+
await client.post(messagesPath, { content: chunk }); // in order, so the reply reads top-down
|
|
403
|
+
}
|
|
404
|
+
return { mode: 'split', messages: chunks.length };
|
|
405
|
+
}
|
|
406
|
+
try {
|
|
407
|
+
const uploaded = await client.upload(`/api/agents/runtime/pods/${podId}/uploads`, {
|
|
408
|
+
fileBuffer: Buffer.from(String(text), 'utf8'),
|
|
409
|
+
fileName: uploadName,
|
|
410
|
+
contentType: 'text/markdown',
|
|
411
|
+
fields: { podId },
|
|
412
|
+
});
|
|
413
|
+
const u = uploaded || {};
|
|
414
|
+
const directive = `[[upload:${u.fileName || uploadName}|${u.originalName || uploadName}|${u.size ?? Buffer.byteLength(String(text))}|${u.kind || 'document'}]]`;
|
|
415
|
+
// Lead with the reply's own opening — unless that opening is itself the
|
|
416
|
+
// oversized atomic unit (a fence-only reply), in which case a generic
|
|
417
|
+
// line keeps the message under the gate and the card carries the content.
|
|
418
|
+
const lead = chunks[0] && chunks[0].length <= limit
|
|
419
|
+
? chunks[0]
|
|
420
|
+
: '(reply too large for chat — attached in full)';
|
|
421
|
+
await client.post(messagesPath, { content: `${lead}\n\n${directive}` });
|
|
422
|
+
return { mode: 'attach', messages: 1 };
|
|
423
|
+
} catch (err) {
|
|
424
|
+
log(`attach fallback failed (${err.message}) — posting ${chunks.length} split messages instead`);
|
|
425
|
+
for (const chunk of chunks) {
|
|
426
|
+
// eslint-disable-next-line no-await-in-loop
|
|
427
|
+
await client.post(messagesPath, { content: chunk });
|
|
428
|
+
}
|
|
429
|
+
return { mode: 'split-fallback', messages: chunks.length };
|
|
430
|
+
}
|
|
431
|
+
};
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retry policy for local wrapper event-processing failures (#782).
|
|
3
|
+
*
|
|
4
|
+
* The kernel deliberately re-delivers an event that the wrapper does not
|
|
5
|
+
* acknowledge. That preserves at-least-once handling, but a flat poll loop
|
|
6
|
+
* turns a model-provider outage into repeated subprocess launches. Keep the
|
|
7
|
+
* retry policy here so every adapter gets the same bounded behavior.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
export const SPAWN_FAILURE_CLASS = Object.freeze({
|
|
11
|
+
QUOTA: 'quota',
|
|
12
|
+
RATE_LIMIT: 'rate_limit',
|
|
13
|
+
CONFIGURATION: 'configuration',
|
|
14
|
+
RUNTIME: 'runtime',
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
export const SPAWN_CIRCUIT_THRESHOLD = 3;
|
|
18
|
+
// Base ceiling before the stable per-agent 0–20% anti-herd offset.
|
|
19
|
+
export const SPAWN_RETRY_MAX_MS = 15 * 60 * 1000;
|
|
20
|
+
export const SPAWN_RETRY_JITTER_MAX_RATIO = 0.2;
|
|
21
|
+
|
|
22
|
+
// `out of credits` is codex's exact wording for an exhausted workspace balance
|
|
23
|
+
// ("Your workspace is out of credits. Ask your workspace owner to refill…").
|
|
24
|
+
// Without it that outage classified as RUNTIME and drew the shortest backoff —
|
|
25
|
+
// observed live on 2026-08-03 before this pattern was added.
|
|
26
|
+
const QUOTA_RE = /(?:quota|usage limit|credit balance|out of credits|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i;
|
|
27
|
+
const RATE_LIMIT_RE = /(?:rate[ -]?limit|too many requests|\b429\b|overloaded|capacity)/i;
|
|
28
|
+
const CONFIGURATION_RE = /(?:ENOENT|command not found|not on PATH|login required|not logged in|invalid api key|authentication failed|unauthori[sz]ed|forbidden|\b40[13]\b)/i;
|
|
29
|
+
|
|
30
|
+
const errorText = (error) => [
|
|
31
|
+
error?.message,
|
|
32
|
+
error?.stderr,
|
|
33
|
+
error?.body?.error,
|
|
34
|
+
error?.body?.message,
|
|
35
|
+
]
|
|
36
|
+
.filter(Boolean)
|
|
37
|
+
.map(String)
|
|
38
|
+
.join('\n');
|
|
39
|
+
|
|
40
|
+
export const classifySpawnFailure = (error) => {
|
|
41
|
+
const text = errorText(error);
|
|
42
|
+
// Provider APIs commonly report an exhausted account quota as HTTP 429.
|
|
43
|
+
// Prefer the more specific body/message over the generic status code so a
|
|
44
|
+
// hard quota failure gets the long cooldown rather than a one-minute probe.
|
|
45
|
+
if (QUOTA_RE.test(text)) return SPAWN_FAILURE_CLASS.QUOTA;
|
|
46
|
+
if (error?.status === 429 || RATE_LIMIT_RE.test(text)) {
|
|
47
|
+
return SPAWN_FAILURE_CLASS.RATE_LIMIT;
|
|
48
|
+
}
|
|
49
|
+
if (
|
|
50
|
+
error?.code === 'ENOENT'
|
|
51
|
+
|| error?.status === 401
|
|
52
|
+
|| error?.status === 403
|
|
53
|
+
|| CONFIGURATION_RE.test(text)
|
|
54
|
+
) {
|
|
55
|
+
return SPAWN_FAILURE_CLASS.CONFIGURATION;
|
|
56
|
+
}
|
|
57
|
+
return SPAWN_FAILURE_CLASS.RUNTIME;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
// A stable per-agent offset keeps a fleet from probing a recovering provider
|
|
61
|
+
// in lockstep. Stability matters: random jitter makes operator logs and tests
|
|
62
|
+
// harder to reason about, while distinct agent names already provide entropy.
|
|
63
|
+
export const spawnRetryJitter = (agentName) => {
|
|
64
|
+
let hash = 2166136261;
|
|
65
|
+
for (const char of String(agentName || 'agent')) {
|
|
66
|
+
hash ^= char.charCodeAt(0);
|
|
67
|
+
hash = Math.imul(hash, 16777619);
|
|
68
|
+
}
|
|
69
|
+
return ((hash >>> 0) % 2001) / 10000;
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
const applyJitter = (delayMs, jitterRatio) => {
|
|
73
|
+
const safeJitter = Number.isFinite(jitterRatio)
|
|
74
|
+
? Math.min(SPAWN_RETRY_JITTER_MAX_RATIO, Math.max(0, jitterRatio))
|
|
75
|
+
: 0;
|
|
76
|
+
return Math.round(delayMs * (1 + safeJitter));
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Return the next probe delay and whether the circuit is open.
|
|
81
|
+
*
|
|
82
|
+
* Known non-transient failures open immediately. Unknown runtime failures get
|
|
83
|
+
* two quick retries, then open the circuit on the third consecutive failure.
|
|
84
|
+
* Later probes back off exponentially to the same 15-minute base ceiling.
|
|
85
|
+
*/
|
|
86
|
+
export const spawnRetryPolicy = ({
|
|
87
|
+
error,
|
|
88
|
+
consecutiveFailures,
|
|
89
|
+
intervalMs,
|
|
90
|
+
jitterRatio = 0,
|
|
91
|
+
}) => {
|
|
92
|
+
const failureClass = classifySpawnFailure(error);
|
|
93
|
+
const safeIntervalMs = Number.isFinite(intervalMs) && intervalMs > 0 ? intervalMs : 5000;
|
|
94
|
+
const failureCount = Number.isInteger(consecutiveFailures) && consecutiveFailures > 0
|
|
95
|
+
? consecutiveFailures
|
|
96
|
+
: 1;
|
|
97
|
+
|
|
98
|
+
if (
|
|
99
|
+
failureClass === SPAWN_FAILURE_CLASS.QUOTA
|
|
100
|
+
|| failureClass === SPAWN_FAILURE_CLASS.CONFIGURATION
|
|
101
|
+
) {
|
|
102
|
+
return {
|
|
103
|
+
failureClass,
|
|
104
|
+
circuitOpen: true,
|
|
105
|
+
delayMs: applyJitter(SPAWN_RETRY_MAX_MS, jitterRatio),
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if (failureClass === SPAWN_FAILURE_CLASS.RATE_LIMIT) {
|
|
110
|
+
return {
|
|
111
|
+
failureClass,
|
|
112
|
+
circuitOpen: true,
|
|
113
|
+
delayMs: applyJitter(
|
|
114
|
+
Math.min(
|
|
115
|
+
SPAWN_RETRY_MAX_MS,
|
|
116
|
+
60 * 1000 * (2 ** (failureCount - 1)),
|
|
117
|
+
),
|
|
118
|
+
jitterRatio,
|
|
119
|
+
),
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
if (failureCount < SPAWN_CIRCUIT_THRESHOLD) {
|
|
124
|
+
return {
|
|
125
|
+
failureClass,
|
|
126
|
+
circuitOpen: false,
|
|
127
|
+
delayMs: applyJitter(
|
|
128
|
+
Math.min(
|
|
129
|
+
SPAWN_RETRY_MAX_MS,
|
|
130
|
+
safeIntervalMs * (2 ** (failureCount - 1)),
|
|
131
|
+
),
|
|
132
|
+
jitterRatio,
|
|
133
|
+
),
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
return {
|
|
138
|
+
failureClass,
|
|
139
|
+
circuitOpen: true,
|
|
140
|
+
delayMs: applyJitter(
|
|
141
|
+
Math.min(
|
|
142
|
+
SPAWN_RETRY_MAX_MS,
|
|
143
|
+
60 * 1000 * (2 ** (failureCount - SPAWN_CIRCUIT_THRESHOLD)),
|
|
144
|
+
),
|
|
145
|
+
jitterRatio,
|
|
146
|
+
),
|
|
147
|
+
};
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
export const formatRetryDelay = (delayMs) => {
|
|
151
|
+
if (delayMs >= 60000) {
|
|
152
|
+
const minutes = delayMs / 60000;
|
|
153
|
+
return `${Number.isInteger(minutes) ? minutes : minutes.toFixed(1)}m`;
|
|
154
|
+
}
|
|
155
|
+
if (delayMs >= 1000) {
|
|
156
|
+
const seconds = delayMs / 1000;
|
|
157
|
+
return `${Number.isInteger(seconds) ? seconds : seconds.toFixed(1)}s`;
|
|
158
|
+
}
|
|
159
|
+
return `${delayMs}ms`;
|
|
160
|
+
};
|