@commonlyai/cli 0.1.30 → 0.1.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/commands/agent.js +318 -24
- package/src/commands/daemon.js +77 -1
- package/src/lib/daemon-supervisor.js +211 -0
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@commonlyai/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.32",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
|
-
"description": "The Commonly CLI
|
|
5
|
+
"description": "The Commonly CLI — connect agents, manage pods, iterate fast",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"main": "./src/index.js",
|
|
8
8
|
"bin": {
|
package/src/commands/agent.js
CHANGED
|
@@ -223,6 +223,11 @@ const PROMPT_EVENT_TYPES = new Set([
|
|
|
223
223
|
'first_contact',
|
|
224
224
|
]);
|
|
225
225
|
|
|
226
|
+
// Consultations have a per-request private response channel, rather than a
|
|
227
|
+
// pod-chat delivery. They stay on their existing single-event path; every
|
|
228
|
+
// event that represents the shared pod inbox is batched below.
|
|
229
|
+
const PRIVATE_RESPONSE_EVENT_TYPES = new Set(['agent.ask', 'agent.ask.response']);
|
|
230
|
+
|
|
226
231
|
// ── default environment for adapters that benefit from auto-MCP wiring ─────
|
|
227
232
|
|
|
228
233
|
// Adapters that can consume `mcp[]` from the resolved environment spec.
|
|
@@ -746,6 +751,10 @@ export const performRun = ({
|
|
|
746
751
|
chatCharLimit = 400,
|
|
747
752
|
maxChatChunks = 3,
|
|
748
753
|
claimYieldDelayMs = 3000,
|
|
754
|
+
// ADR-024 D3: a poll is an inbox batch, not ten independent interrupts.
|
|
755
|
+
// Kept injectable for small-page regression cases; the runtime ships with
|
|
756
|
+
// the server's ordinary ten-event page.
|
|
757
|
+
inboxBatchLimit = 10,
|
|
749
758
|
sleepImpl = (ms) => new Promise((resolve) => { setTimeout(resolve, ms); }),
|
|
750
759
|
}) => {
|
|
751
760
|
const client = createClient({ instance: instanceUrl, token });
|
|
@@ -793,6 +802,7 @@ export const performRun = ({
|
|
|
793
802
|
// workspace path replaces the /tmp default.
|
|
794
803
|
const agentCwd = workspacePath || join(tmpdir(), 'commonly-agents', agentName);
|
|
795
804
|
if (!existsSync(agentCwd)) mkdirSync(agentCwd, { recursive: true });
|
|
805
|
+
const batchLimit = Math.min(50, Math.max(1, Number(inboxBatchLimit) || 10));
|
|
796
806
|
|
|
797
807
|
const processEvent = async (event) => {
|
|
798
808
|
const eventPodId = event.podId || podId;
|
|
@@ -1102,7 +1112,7 @@ export const performRun = ({
|
|
|
1102
1112
|
}
|
|
1103
1113
|
}
|
|
1104
1114
|
}
|
|
1105
|
-
const heartbeatControlReply = event.type === 'heartbeat'
|
|
1115
|
+
const heartbeatControlReply = (event.type === 'heartbeat' || event.payload?.hasHeartbeat === true)
|
|
1106
1116
|
&& /^(HEARTBEAT_OK|HEARTBEAT_NOOP)$/i.test(replyText);
|
|
1107
1117
|
const silentReply = !replyText || replyText === 'NO_REPLY' || heartbeatControlReply;
|
|
1108
1118
|
let delivered = agentPostedItself;
|
|
@@ -1253,38 +1263,322 @@ export const performRun = ({
|
|
|
1253
1263
|
return { outcome: delivered ? 'posted' : 'no_action' };
|
|
1254
1264
|
};
|
|
1255
1265
|
|
|
1266
|
+
// ADR-024 D3: the batch header states the real inbox count even when we cap
|
|
1267
|
+
// copied bodies. The cap is the fetched page itself: never acknowledge a
|
|
1268
|
+
// claimed event whose body the one turn did not receive. Additional pending
|
|
1269
|
+
// events stay in the kernel queue and are cued in the header for a later
|
|
1270
|
+
// pull, rather than being silently consumed here.
|
|
1271
|
+
const buildInboxPrompt = (entries, inboxCount = entries.length) => {
|
|
1272
|
+
const BODY_LIMIT = batchLimit;
|
|
1273
|
+
const bodies = entries.slice(0, BODY_LIMIT).map((entry, index) => {
|
|
1274
|
+
const messageId = entry.event.payload?.messageId;
|
|
1275
|
+
const prefix = [
|
|
1276
|
+
`Event ${index + 1}/${entries.length}: ${entry.event.type}`,
|
|
1277
|
+
messageId ? `message ${messageId}` : null,
|
|
1278
|
+
].filter(Boolean).join(' — ');
|
|
1279
|
+
const claimContext = entry.readOnly
|
|
1280
|
+
? `${peerHoldsFrame(entry.holder, messageId)}\n`
|
|
1281
|
+
+ '[This item is read-only context: a peer won its binding claim. Do not act on it.]\n'
|
|
1282
|
+
: (entry.peerFrame ? `${entry.peerFrame}\n` : '');
|
|
1283
|
+
const body = entry.prompt || '[No usable prompt was delivered for this item.]';
|
|
1284
|
+
return `${prefix}\n${claimContext}${body}`;
|
|
1285
|
+
});
|
|
1286
|
+
const omitted = Math.max(0, inboxCount - bodies.length);
|
|
1287
|
+
return [
|
|
1288
|
+
`[Inbox batch: ${inboxCount} new event${inboxCount === 1 ? '' : 's'} in this pod. `
|
|
1289
|
+
+ 'This is one turn: read the included items together, then decide what needs YOU.]',
|
|
1290
|
+
...bodies,
|
|
1291
|
+
...(omitted > 0
|
|
1292
|
+
? [`[${omitted} more event${omitted === 1 ? '' : 's'} arrived. Their count is real; pull pod context if needed.]`]
|
|
1293
|
+
: []),
|
|
1294
|
+
'Do not narrate the inbox. Post only an action or a materially useful response; otherwise return NO_REPLY.',
|
|
1295
|
+
].join('\n\n');
|
|
1296
|
+
};
|
|
1297
|
+
|
|
1298
|
+
const batchAdmissionResult = (event, admission) => ({
|
|
1299
|
+
outcome: 'no_action',
|
|
1300
|
+
reason: 'cascade-cap',
|
|
1301
|
+
details: {
|
|
1302
|
+
messageId: event?.payload?.messageId || null,
|
|
1303
|
+
streak: admission.streak,
|
|
1304
|
+
cap: cascadeSettings.cap,
|
|
1305
|
+
addressedGrace: cascadeSettings.addressedGrace,
|
|
1306
|
+
resetMs: cascadeSettings.resetMs,
|
|
1307
|
+
addressed: admission.addressed,
|
|
1308
|
+
graceApplied: admission.graceApplied,
|
|
1309
|
+
},
|
|
1310
|
+
});
|
|
1311
|
+
|
|
1312
|
+
// Claim the binding portion before the ONE composite spawn. A lost binding
|
|
1313
|
+
// claim is not discarded: it remains in the model's context with the same
|
|
1314
|
+
// peer frame the single-event path already uses, but is explicitly read-only.
|
|
1315
|
+
// Advisory losses retain their existing peer-aware behaviour.
|
|
1316
|
+
const processEventBatch = async (events, inboxCount = events.length) => {
|
|
1317
|
+
const eventPodId = events[0]?.podId || podId;
|
|
1318
|
+
if (!eventPodId || events.some((event) => (event.podId || podId) !== eventPodId)) {
|
|
1319
|
+
throw new Error('Inbox batch crossed pod boundaries; refusing to mix private pod context');
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
const entries = events.map((event) => ({
|
|
1323
|
+
event,
|
|
1324
|
+
prompt: extractPrompt(event),
|
|
1325
|
+
result: null,
|
|
1326
|
+
claimKeeper: null,
|
|
1327
|
+
peerFrame: null,
|
|
1328
|
+
readOnly: false,
|
|
1329
|
+
holder: null,
|
|
1330
|
+
}));
|
|
1331
|
+
|
|
1332
|
+
// Duplicate deliveries have already completed locally. Re-ack them but do
|
|
1333
|
+
// not spend the new batch turn displaying old work as if it were new.
|
|
1334
|
+
const seenEventIds = new Set();
|
|
1335
|
+
for (const entry of entries) {
|
|
1336
|
+
const eventId = String(entry.event._id);
|
|
1337
|
+
if (seenEventIds.has(eventId) || wasEventHandled(agentName, entry.event._id)) {
|
|
1338
|
+
entry.result = { outcome: 'no_action', reason: 'duplicate-delivery' };
|
|
1339
|
+
log(`[${entry.event.type}] duplicate delivery ${entry.event._id} — re-acking without batch spawn`);
|
|
1340
|
+
} else {
|
|
1341
|
+
seenEventIds.add(eventId);
|
|
1342
|
+
}
|
|
1343
|
+
if (entry.result?.reason === 'duplicate-delivery') {
|
|
1344
|
+
continue;
|
|
1345
|
+
}
|
|
1346
|
+
if (!entry.prompt) {
|
|
1347
|
+
entry.result = { outcome: 'no_action', reason: 'no-prompt' };
|
|
1348
|
+
log(`[${entry.event.type}] no prompt — no-op`);
|
|
1349
|
+
onError?.(Object.assign(
|
|
1350
|
+
new Error(
|
|
1351
|
+
`${entry.event.type} event ${entry.event._id} was acked WITHOUT a spawn: `
|
|
1352
|
+
+ 'payload carried no content and no messageId. If this wake mattered, its producer is sending an unusable payload.',
|
|
1353
|
+
),
|
|
1354
|
+
{ code: 'agent_event_skipped_no_prompt', eventId: entry.event._id },
|
|
1355
|
+
));
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
const snapshotMessages = async () => {
|
|
1360
|
+
try {
|
|
1361
|
+
const { messages = [] } = await client.get(
|
|
1362
|
+
`/api/agents/runtime/pods/${eventPodId}/messages`, { limit: 10 },
|
|
1363
|
+
);
|
|
1364
|
+
return messages;
|
|
1365
|
+
} catch {
|
|
1366
|
+
return null;
|
|
1367
|
+
}
|
|
1368
|
+
};
|
|
1369
|
+
const preSpawn = await snapshotMessages();
|
|
1370
|
+
const preSpawnIds = preSpawn
|
|
1371
|
+
? new Set(preSpawn.map((message) => String(message._id || message.id)))
|
|
1372
|
+
: null;
|
|
1373
|
+
const activeEntries = entries.filter((entry) => !entry.result);
|
|
1374
|
+
const triggers = activeEntries.map((entry) => classifyTrigger(entry.event, preSpawn));
|
|
1375
|
+
const trigger = triggers.includes('human') ? 'human'
|
|
1376
|
+
: (triggers.includes('agent') ? 'agent' : 'unknown');
|
|
1377
|
+
if (trigger === 'human') cascadeGovernor.record(eventPodId, trigger);
|
|
1378
|
+
const admissionEvent = activeEntries.find((entry) => ADDRESSED_EVENT_TYPES.has(entry.event.type))
|
|
1379
|
+
|| activeEntries[0];
|
|
1380
|
+
if (admissionEvent) {
|
|
1381
|
+
const admission = cascadeGovernor.admit(
|
|
1382
|
+
eventPodId,
|
|
1383
|
+
trigger,
|
|
1384
|
+
admissionEvent.event.type,
|
|
1385
|
+
admissionEvent.event.payload,
|
|
1386
|
+
);
|
|
1387
|
+
if (!admission.allowed) {
|
|
1388
|
+
for (const entry of activeEntries) {
|
|
1389
|
+
entry.result = batchAdmissionResult(entry.event, admission);
|
|
1390
|
+
}
|
|
1391
|
+
return { entries };
|
|
1392
|
+
}
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
const bindingEntries = activeEntries.filter((entry) => (
|
|
1396
|
+
entry.event.type === 'message.posted' && entry.event.payload?.messageId
|
|
1397
|
+
));
|
|
1398
|
+
if (bindingEntries.length > 0) {
|
|
1399
|
+
const yieldMs = claimHandicap.yieldDelayMs(eventPodId);
|
|
1400
|
+
if (yieldMs > 0) {
|
|
1401
|
+
log(`[inbox.batch] yielding ${yieldMs}ms before claiming ${bindingEntries.length} binding event${bindingEntries.length === 1 ? '' : 's'}`);
|
|
1402
|
+
await sleepImpl(yieldMs);
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1406
|
+
for (const entry of activeEntries) {
|
|
1407
|
+
const messageId = entry.event.payload?.messageId;
|
|
1408
|
+
if (!messageId || !CLAIMABLE_EVENT_TYPES.has(entry.event.type)) continue;
|
|
1409
|
+
const claimKeeper = createClaimKeeper(client, {
|
|
1410
|
+
messageId,
|
|
1411
|
+
podId: eventPodId,
|
|
1412
|
+
leaseSeconds: claimLeaseSeconds,
|
|
1413
|
+
log: (line) => log(`[${entry.event.type}] ${line}`),
|
|
1414
|
+
setIntervalImpl,
|
|
1415
|
+
clearIntervalImpl,
|
|
1416
|
+
});
|
|
1417
|
+
const claim = await claimKeeper.acquire();
|
|
1418
|
+
if (claim.claimed) {
|
|
1419
|
+
entry.claimKeeper = claimKeeper;
|
|
1420
|
+
claimKeeper.startRenewal();
|
|
1421
|
+
if (entry.event.type === 'message.posted') claimHandicap.recordWin(eventPodId);
|
|
1422
|
+
} else if (!claim.failOpen) {
|
|
1423
|
+
entry.holder = claim.holder;
|
|
1424
|
+
if (entry.event.type === 'message.posted') {
|
|
1425
|
+
// A reply to this seat's own message is direct-address evidence,
|
|
1426
|
+
// even though its transport type is the broadcast-shaped
|
|
1427
|
+
// `message.posted`. Preserve the existing ADR-018/TASK-058
|
|
1428
|
+
// peer-aware exception; D3a otherwise partitions lost bindings as
|
|
1429
|
+
// read-only so a generic broadcast cannot widen itself after CAS.
|
|
1430
|
+
if (entry.event.payload?.repliesToYourMessage === true) {
|
|
1431
|
+
entry.peerFrame = peerHoldsFrame(claim.holder, messageId);
|
|
1432
|
+
continue;
|
|
1433
|
+
}
|
|
1434
|
+
// D3a's partition: no later model judgement can widen this item's
|
|
1435
|
+
// authority after a peer won the binding CAS.
|
|
1436
|
+
entry.readOnly = true;
|
|
1437
|
+
entry.result = { outcome: 'no_action', reason: 'claim-held' };
|
|
1438
|
+
claimHandicap.recordLoss(eventPodId);
|
|
1439
|
+
} else {
|
|
1440
|
+
entry.peerFrame = peerHoldsFrame(claim.holder, messageId);
|
|
1441
|
+
}
|
|
1442
|
+
} else {
|
|
1443
|
+
log(`[${entry.event.type}] claim unavailable (${claim.error?.message || 'unknown error'}) — proceeding unguarded`);
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
|
|
1447
|
+
const actionEntries = activeEntries.filter((entry) => !entry.result);
|
|
1448
|
+
if (actionEntries.length === 0) return { entries };
|
|
1449
|
+
|
|
1450
|
+
const claimKeepers = actionEntries
|
|
1451
|
+
.map((entry) => entry.claimKeeper)
|
|
1452
|
+
.filter(Boolean);
|
|
1453
|
+
const compositeClaimKeeper = claimKeepers.length > 0 ? {
|
|
1454
|
+
isLost: () => claimKeepers.some((keeper) => keeper.isLost()),
|
|
1455
|
+
getHolder: () => claimKeepers.find((keeper) => keeper.isLost())?.getHolder(),
|
|
1456
|
+
} : null;
|
|
1457
|
+
const batchEvent = {
|
|
1458
|
+
...actionEntries[0].event,
|
|
1459
|
+
_id: `batch-${actionEntries[0].event._id}`,
|
|
1460
|
+
type: 'inbox.batch',
|
|
1461
|
+
payload: {
|
|
1462
|
+
...actionEntries[0].event.payload,
|
|
1463
|
+
batchEventIds: entries.map((entry) => String(entry.event._id)),
|
|
1464
|
+
hasHeartbeat: entries.some((entry) => entry.event.type === 'heartbeat'),
|
|
1465
|
+
},
|
|
1466
|
+
};
|
|
1467
|
+
let turnResult;
|
|
1468
|
+
try {
|
|
1469
|
+
turnResult = await runTurn({
|
|
1470
|
+
event: batchEvent,
|
|
1471
|
+
eventPodId,
|
|
1472
|
+
prompt: buildInboxPrompt(entries, inboxCount),
|
|
1473
|
+
preSpawnIds,
|
|
1474
|
+
snapshotMessages,
|
|
1475
|
+
claimKeeper: compositeClaimKeeper,
|
|
1476
|
+
trigger,
|
|
1477
|
+
});
|
|
1478
|
+
for (const entry of actionEntries) entry.result = turnResult;
|
|
1479
|
+
return { entries };
|
|
1480
|
+
} finally {
|
|
1481
|
+
await Promise.all(actionEntries
|
|
1482
|
+
.filter((entry) => entry.claimKeeper)
|
|
1483
|
+
.map(async (entry) => {
|
|
1484
|
+
// Preserve ADR-018 D6.1 per binding message: a silent human
|
|
1485
|
+
// broadcast may be handed to one remaining listener, while every
|
|
1486
|
+
// other completed batch item closes normally.
|
|
1487
|
+
const claimOutcome = entry.event.type === 'message.posted'
|
|
1488
|
+
&& entry.event.payload?.senderIsHuman === true
|
|
1489
|
+
&& turnResult?.outcome === 'no_action' && !turnResult?.reason
|
|
1490
|
+
? 'declined'
|
|
1491
|
+
: (turnResult ? 'completed' : undefined);
|
|
1492
|
+
await entry.claimKeeper.release(claimOutcome);
|
|
1493
|
+
}));
|
|
1494
|
+
}
|
|
1495
|
+
};
|
|
1496
|
+
|
|
1256
1497
|
const tick = async () => {
|
|
1257
1498
|
if (!running) return;
|
|
1258
1499
|
let nextPollDelayMs = intervalMs;
|
|
1259
1500
|
try {
|
|
1260
|
-
//
|
|
1261
|
-
//
|
|
1262
|
-
//
|
|
1263
|
-
//
|
|
1264
|
-
//
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
// The nine it cannot start are then reclaimed out from under it: the
|
|
1268
|
-
// backend requeues `delivered` rows older than
|
|
1269
|
-
// `requeueDeliveredMinutes` (default 10, swept on `*/10`), and turns
|
|
1270
|
-
// routinely outlast that — measured on the pod-architect seat over
|
|
1271
|
-
// 11.5h: median 128s, p90 669s, 13 turns over 600s, max 1153s. Each
|
|
1272
|
-
// sweep returns the untouched siblings to `pending` at `attempts + 1`,
|
|
1273
|
-
// and `attempts >= 3` retires an event to `failed`, which is terminal
|
|
1274
|
-
// and invisible to `list()`. That cap exists to bound POISON events;
|
|
1275
|
-
// over-claiming feeds it work no model ever saw, so a mention can be
|
|
1276
|
-
// dropped without once being read.
|
|
1277
|
-
//
|
|
1278
|
-
// `limit: 1` costs nothing: capacity here is one turn at a time
|
|
1279
|
-
// regardless, and the poll interval is 5s. It only stops the loop
|
|
1280
|
-
// claiming work it has no way to begin.
|
|
1281
|
-
const { events = [] } = await client.get('/api/agents/runtime/events', {
|
|
1282
|
-
agentName, instanceId, limit: 1,
|
|
1501
|
+
// ADR-024 D3: this is an inbox fetch, not a stream of interrupts. The
|
|
1502
|
+
// backend returns one pod's oldest pending batch, so every delivered row
|
|
1503
|
+
// below reaches the SAME composite turn before it can age into the
|
|
1504
|
+
// requeue sweep. The service selects the pod before claiming any row;
|
|
1505
|
+
// do not reintroduce a mixed-pod `limit: 10` here.
|
|
1506
|
+
const { events = [], inboxCount } = await client.get('/api/agents/runtime/events', {
|
|
1507
|
+
agentName, instanceId, limit: batchLimit,
|
|
1283
1508
|
});
|
|
1284
1509
|
consecutiveAuthErrors = 0;
|
|
1285
1510
|
consecutivePollFailures = 0;
|
|
1511
|
+
// A deployed CLI can briefly run against an older server that still
|
|
1512
|
+
// returns multiple pods in one page. Preserve pod privacy in that
|
|
1513
|
+
// migration window by partitioning locally. Current servers produce one
|
|
1514
|
+
// group, so the normal path remains one tick → one turn.
|
|
1515
|
+
const eventGroups = new Map();
|
|
1286
1516
|
for (const event of events) {
|
|
1517
|
+
const eventPodId = event.podId || podId;
|
|
1518
|
+
const key = PRIVATE_RESPONSE_EVENT_TYPES.has(event.type)
|
|
1519
|
+
? `private:${event._id}`
|
|
1520
|
+
: (eventPodId || `private:${event._id}`);
|
|
1521
|
+
const group = eventGroups.get(key) || [];
|
|
1522
|
+
group.push(event);
|
|
1523
|
+
eventGroups.set(key, group);
|
|
1524
|
+
}
|
|
1525
|
+
for (const group of eventGroups.values()) {
|
|
1287
1526
|
if (!running) break;
|
|
1527
|
+
if ((group[0].podId || podId) && !PRIVATE_RESPONSE_EVENT_TYPES.has(group[0].type)) {
|
|
1528
|
+
let batch;
|
|
1529
|
+
try {
|
|
1530
|
+
batch = await processEventBatch(group, inboxCount);
|
|
1531
|
+
} catch (err) {
|
|
1532
|
+
consecutiveSpawnFailures += 1;
|
|
1533
|
+
const retry = spawnRetryPolicy({
|
|
1534
|
+
error: err,
|
|
1535
|
+
consecutiveFailures: consecutiveSpawnFailures,
|
|
1536
|
+
intervalMs,
|
|
1537
|
+
jitterRatio: spawnJitterRatio,
|
|
1538
|
+
});
|
|
1539
|
+
nextPollDelayMs = retry.delayMs;
|
|
1540
|
+
const event = group[0];
|
|
1541
|
+
const wrapped = new Error(
|
|
1542
|
+
`inbox batch processing failed (${retry.failureClass}; ${consecutiveSpawnFailures} consecutive) `
|
|
1543
|
+
+ `— ${group.length} events remain unacked; ${retry.circuitOpen ? 'circuit open' : 'retry scheduled'}, `
|
|
1544
|
+
+ `next probe in ${formatRetryDelay(retry.delayMs)}: ${err.message}`,
|
|
1545
|
+
{ cause: err },
|
|
1546
|
+
);
|
|
1547
|
+
Object.assign(wrapped, {
|
|
1548
|
+
code: 'agent_spawn_retry_scheduled',
|
|
1549
|
+
failureClass: retry.failureClass,
|
|
1550
|
+
consecutiveFailures: consecutiveSpawnFailures,
|
|
1551
|
+
retryAfterMs: retry.delayMs,
|
|
1552
|
+
circuitOpen: retry.circuitOpen,
|
|
1553
|
+
eventId: event._id,
|
|
1554
|
+
});
|
|
1555
|
+
if (onError) onError(wrapped);
|
|
1556
|
+
else log(`[inbox.batch] ${wrapped.message}`);
|
|
1557
|
+
break;
|
|
1558
|
+
}
|
|
1559
|
+
const batchSpawned = batch.entries.some((entry) => (
|
|
1560
|
+
entry.result?.outcome === 'posted'
|
|
1561
|
+
|| (entry.result?.outcome === 'no_action' && !entry.result?.reason)
|
|
1562
|
+
));
|
|
1563
|
+
if (batchSpawned) consecutiveSpawnFailures = 0;
|
|
1564
|
+
for (const entry of batch.entries) {
|
|
1565
|
+
const event = entry.event;
|
|
1566
|
+
if (entry.result?.reason !== 'duplicate-delivery') {
|
|
1567
|
+
recordHandledEvent(agentName, event._id);
|
|
1568
|
+
}
|
|
1569
|
+
try {
|
|
1570
|
+
const deliveryId = event.payload?.deliveryId;
|
|
1571
|
+
await client.post(`/api/agents/runtime/events/${event._id}/ack`, {
|
|
1572
|
+
result: entry.result,
|
|
1573
|
+
...(typeof deliveryId === 'string' && deliveryId ? { deliveryId } : {}),
|
|
1574
|
+
});
|
|
1575
|
+
} catch (ackErr) {
|
|
1576
|
+
onError?.(new Error(`Ack failed for ${event._id}: ${ackErr.message}`));
|
|
1577
|
+
}
|
|
1578
|
+
}
|
|
1579
|
+
continue;
|
|
1580
|
+
}
|
|
1581
|
+
const [event] = group;
|
|
1288
1582
|
let result;
|
|
1289
1583
|
if (wasEventHandled(agentName, event._id)) {
|
|
1290
1584
|
result = { outcome: 'no_action', reason: 'duplicate-delivery' };
|
package/src/commands/daemon.js
CHANGED
|
@@ -6,10 +6,20 @@
|
|
|
6
6
|
* later slices; this command never reads an agent runtime credential.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import { hostname } from 'os';
|
|
9
|
+
import { hostname, homedir } from 'os';
|
|
10
|
+
import { spawn } from 'child_process';
|
|
11
|
+
import { existsSync, mkdirSync, openSync } from 'fs';
|
|
12
|
+
import { join } from 'path';
|
|
10
13
|
import { createClient } from '../lib/api.js';
|
|
11
14
|
import { getToken, resolveInstanceUrl } from '../lib/config.js';
|
|
12
15
|
import { loadDaemonRecord, saveDaemonRecord } from '../lib/daemon-store.js';
|
|
16
|
+
import {
|
|
17
|
+
createDaemonSupervisor,
|
|
18
|
+
DEFAULT_HEARTBEAT_MS,
|
|
19
|
+
DEFAULT_POLL_MS,
|
|
20
|
+
} from '../lib/daemon-supervisor.js';
|
|
21
|
+
import { loadAgentToken, saveAgentToken } from './agent.js';
|
|
22
|
+
import { getAdapter } from '../lib/adapters/index.js';
|
|
13
23
|
|
|
14
24
|
const requireDaemonRecord = () => {
|
|
15
25
|
const record = loadDaemonRecord();
|
|
@@ -80,6 +90,20 @@ export const getDaemonMachineStatus = async ({ client }) => {
|
|
|
80
90
|
return response?.machine || null;
|
|
81
91
|
};
|
|
82
92
|
|
|
93
|
+
// The adapter names a binary on THIS machine — the one fact the server cannot
|
|
94
|
+
// know (same reasoning as `agent run`'s env bootstrap). A server-declared
|
|
95
|
+
// preference is honored when that CLI is installed; otherwise probe the known
|
|
96
|
+
// ones in order.
|
|
97
|
+
export const resolveAdapterForRuntime = async (runtime, registry = { getAdapter }) => {
|
|
98
|
+
const candidates = [runtime?.adapter, 'claude', 'codex'].filter(Boolean);
|
|
99
|
+
for (const name of candidates) {
|
|
100
|
+
const adapter = registry.getAdapter(name);
|
|
101
|
+
// eslint-disable-next-line no-await-in-loop
|
|
102
|
+
if (adapter && await adapter.detect()) return name;
|
|
103
|
+
}
|
|
104
|
+
return null;
|
|
105
|
+
};
|
|
106
|
+
|
|
83
107
|
export const registerDaemon = (program) => {
|
|
84
108
|
const daemon = program.command('daemon').description('Manage the local Commonly daemon');
|
|
85
109
|
|
|
@@ -147,6 +171,58 @@ Examples:
|
|
|
147
171
|
}
|
|
148
172
|
});
|
|
149
173
|
|
|
174
|
+
// ── run (ADR-026 Phase 2, slice 2) ────────────────────────────────────────
|
|
175
|
+
daemon
|
|
176
|
+
.command('run')
|
|
177
|
+
.description('Run the resident supervisor: adopt requested agents, keep bound agents running, report per-agent state')
|
|
178
|
+
.option('--poll <ms>', 'Work-list poll interval in ms', String(DEFAULT_POLL_MS))
|
|
179
|
+
.option('--heartbeat <ms>', 'Heartbeat interval in ms', String(DEFAULT_HEARTBEAT_MS))
|
|
180
|
+
.action(async (opts) => {
|
|
181
|
+
try {
|
|
182
|
+
const record = requireDaemonRecord();
|
|
183
|
+
const client = createClient({ instance: record.instanceUrl, token: record.daemonToken });
|
|
184
|
+
const logsDir = join(homedir(), '.commonly', 'logs', 'daemon');
|
|
185
|
+
if (!existsSync(logsDir)) mkdirSync(logsDir, { recursive: true });
|
|
186
|
+
const stampLog = (line) => console.log(`${new Date().toISOString()} ${line}`);
|
|
187
|
+
|
|
188
|
+
const supervisor = createDaemonSupervisor({
|
|
189
|
+
record,
|
|
190
|
+
client,
|
|
191
|
+
// One child per agent, logging to its own file. The child is the
|
|
192
|
+
// ordinary `commonly agent run <name>` — the daemon is its
|
|
193
|
+
// supervisor, never its replacement (D6).
|
|
194
|
+
spawnChild: (agentName) => {
|
|
195
|
+
const out = openSync(join(logsDir, `${agentName}.log`), 'a');
|
|
196
|
+
return spawn(process.execPath, [process.argv[1], 'agent', 'run', agentName], {
|
|
197
|
+
stdio: ['ignore', out, out],
|
|
198
|
+
});
|
|
199
|
+
},
|
|
200
|
+
loadToken: loadAgentToken,
|
|
201
|
+
saveToken: saveAgentToken,
|
|
202
|
+
resolveAdapter: (runtime) => resolveAdapterForRuntime(runtime),
|
|
203
|
+
log: stampLog,
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
stampLog(`daemon supervising for ${record.machineName} — poll ${opts.poll}ms, heartbeat ${opts.heartbeat}ms (ctrl+c to stop)`);
|
|
207
|
+
await supervisor.tick();
|
|
208
|
+
await supervisor.heartbeat();
|
|
209
|
+
const pollTimer = setInterval(() => supervisor.tick(), Number(opts.poll) || DEFAULT_POLL_MS);
|
|
210
|
+
const heartbeatTimer = setInterval(() => supervisor.heartbeat(), Number(opts.heartbeat) || DEFAULT_HEARTBEAT_MS);
|
|
211
|
+
const shutdown = () => {
|
|
212
|
+
stampLog('daemon stopping — terminating supervised agents');
|
|
213
|
+
clearInterval(pollTimer);
|
|
214
|
+
clearInterval(heartbeatTimer);
|
|
215
|
+
supervisor.stop();
|
|
216
|
+
process.exit(0);
|
|
217
|
+
};
|
|
218
|
+
process.on('SIGINT', shutdown);
|
|
219
|
+
process.on('SIGTERM', shutdown);
|
|
220
|
+
} catch (error) {
|
|
221
|
+
console.error(`Daemon run failed: ${error.message}`);
|
|
222
|
+
process.exitCode = 1;
|
|
223
|
+
}
|
|
224
|
+
});
|
|
225
|
+
|
|
150
226
|
daemon
|
|
151
227
|
.command('status')
|
|
152
228
|
.description('Show the server-derived liveness of this machine')
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ADR-026 Phase 2, slice 2: the resident supervision loop behind
|
|
3
|
+
* `commonly daemon run`.
|
|
4
|
+
*
|
|
5
|
+
* The server's work list (GET /api/agent-binding/assigned) is the source of
|
|
6
|
+
* truth (D2): a row `requested` gets adopted (the D3 CAS — the server refuses
|
|
7
|
+
* the loser of a race cleanly), a row `bound` gets provisioned (token file)
|
|
8
|
+
* and supervised (a `commonly agent run <name>` child), and a supervised
|
|
9
|
+
* agent that leaves the list gets stopped. Per-agent state rides every
|
|
10
|
+
* machine heartbeat (D5).
|
|
11
|
+
*
|
|
12
|
+
* D6 discipline: a replacement child is only ever scheduled from the previous
|
|
13
|
+
* child's 'exit' event — there is no code path that spawns a second runner
|
|
14
|
+
* for an agent whose child has not exited.
|
|
15
|
+
*
|
|
16
|
+
* All side effects (client, spawn, token file I/O, adapter detection, timers)
|
|
17
|
+
* are injected so the loop's decisions are testable without processes.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
export const DEFAULT_POLL_MS = 30_000;
|
|
21
|
+
export const DEFAULT_HEARTBEAT_MS = 30_000;
|
|
22
|
+
export const BACKOFF_BASE_MS = 5_000;
|
|
23
|
+
export const BACKOFF_MAX_MS = 60_000;
|
|
24
|
+
|
|
25
|
+
export const backoffMs = (restarts) => Math.min(
|
|
26
|
+
BACKOFF_MAX_MS,
|
|
27
|
+
BACKOFF_BASE_MS * 2 ** Math.max(0, Math.min(restarts, 10)),
|
|
28
|
+
);
|
|
29
|
+
|
|
30
|
+
const identityKey = (agentName, instanceId) => `${agentName} ${instanceId || 'default'}`;
|
|
31
|
+
|
|
32
|
+
export const createDaemonSupervisor = ({
|
|
33
|
+
record,
|
|
34
|
+
client,
|
|
35
|
+
spawnChild, // (agentName) => child emitting 'exit'; must expose .kill()
|
|
36
|
+
loadToken, // (agentName) => token record | null
|
|
37
|
+
saveToken, // (agentName, record) => void
|
|
38
|
+
resolveAdapter, // async (runtime) => adapter name for THIS machine
|
|
39
|
+
log = () => {},
|
|
40
|
+
setTimeoutFn = setTimeout,
|
|
41
|
+
clearTimeoutFn = clearTimeout,
|
|
42
|
+
}) => {
|
|
43
|
+
// key → { agentName, instanceId, child, state, restarts, backoffTimer, desired }
|
|
44
|
+
const seats = new Map();
|
|
45
|
+
let stopped = false;
|
|
46
|
+
|
|
47
|
+
const agentStates = () => Array.from(seats.values()).map((s) => ({
|
|
48
|
+
agentName: s.agentName,
|
|
49
|
+
instanceId: s.instanceId,
|
|
50
|
+
state: s.state,
|
|
51
|
+
restarts: s.restarts,
|
|
52
|
+
}));
|
|
53
|
+
|
|
54
|
+
const startChild = (seat) => {
|
|
55
|
+
if (stopped || !seat.desired || seat.child) return;
|
|
56
|
+
seat.child = spawnChild(seat.agentName);
|
|
57
|
+
seat.state = 'running';
|
|
58
|
+
log(`[${seat.agentName}] supervising (restarts so far: ${seat.restarts})`);
|
|
59
|
+
seat.child.on('exit', (code) => {
|
|
60
|
+
seat.child = null;
|
|
61
|
+
if (stopped || !seat.desired) {
|
|
62
|
+
seat.state = 'stopped';
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
seat.state = code === 0 ? 'stopped' : 'crashed';
|
|
66
|
+
seat.restarts += 1;
|
|
67
|
+
const delay = backoffMs(seat.restarts - 1);
|
|
68
|
+
log(`[${seat.agentName}] exited (code ${code}) — respawn in ${Math.round(delay / 1000)}s`);
|
|
69
|
+
seat.backoffTimer = setTimeoutFn(() => {
|
|
70
|
+
seat.backoffTimer = null;
|
|
71
|
+
startChild(seat);
|
|
72
|
+
}, delay);
|
|
73
|
+
});
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
const stopSeat = (seat) => {
|
|
77
|
+
seat.desired = false;
|
|
78
|
+
if (seat.backoffTimer) {
|
|
79
|
+
clearTimeoutFn(seat.backoffTimer);
|
|
80
|
+
seat.backoffTimer = null;
|
|
81
|
+
}
|
|
82
|
+
if (seat.child) {
|
|
83
|
+
log(`[${seat.agentName}] no longer assigned here — stopping`);
|
|
84
|
+
seat.child.kill('SIGTERM');
|
|
85
|
+
} else {
|
|
86
|
+
seat.state = 'stopped';
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
// Ensure ~/.commonly/tokens/<name>.json exists so `agent run` can boot.
|
|
91
|
+
// The mint refuses to clobber an existing token (409 token_exists); the
|
|
92
|
+
// binding to THIS machine is the owner's explicit takeover choice (D3), so
|
|
93
|
+
// that refusal is answered with rotate:true — loudly.
|
|
94
|
+
const ensureToken = async (row) => {
|
|
95
|
+
if (loadToken(row.agentName)) return true;
|
|
96
|
+
const body = { agentName: row.agentName, instanceId: row.instanceId };
|
|
97
|
+
let minted;
|
|
98
|
+
try {
|
|
99
|
+
minted = await client.post('/api/agent-binding/runtime-token', body);
|
|
100
|
+
} catch (error) {
|
|
101
|
+
if (error?.status === 409 && error?.body?.code === 'token_exists') {
|
|
102
|
+
log(`[${row.agentName}] a runtime token exists elsewhere — rotating it to this machine (the old token stops working)`);
|
|
103
|
+
try {
|
|
104
|
+
minted = await client.post('/api/agent-binding/runtime-token', { ...body, rotate: true });
|
|
105
|
+
} catch (rotateError) {
|
|
106
|
+
log(`[${row.agentName}] token rotation failed: ${rotateError.message}`);
|
|
107
|
+
return false;
|
|
108
|
+
}
|
|
109
|
+
} else {
|
|
110
|
+
log(`[${row.agentName}] token mint failed: ${error.message}`);
|
|
111
|
+
return false;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
if (!minted?.token) {
|
|
115
|
+
log(`[${row.agentName}] mint returned no token — skipping`);
|
|
116
|
+
return false;
|
|
117
|
+
}
|
|
118
|
+
const adapter = await resolveAdapter(row.runtime || null);
|
|
119
|
+
if (!adapter) {
|
|
120
|
+
log(`[${row.agentName}] no usable CLI adapter on this machine — install claude or codex, or attach manually`);
|
|
121
|
+
return false;
|
|
122
|
+
}
|
|
123
|
+
saveToken(row.agentName, {
|
|
124
|
+
agentName: row.agentName,
|
|
125
|
+
instanceId: row.instanceId,
|
|
126
|
+
runtimeToken: minted.token,
|
|
127
|
+
instanceUrl: record.instanceUrl,
|
|
128
|
+
podId: row.podIds?.[0] || null,
|
|
129
|
+
adapter,
|
|
130
|
+
});
|
|
131
|
+
log(`[${row.agentName}] provisioned runtime token (adapter: ${adapter})`);
|
|
132
|
+
return true;
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
const tick = async () => {
|
|
136
|
+
if (stopped) return;
|
|
137
|
+
let assigned;
|
|
138
|
+
try {
|
|
139
|
+
assigned = await client.get('/api/agent-binding/assigned');
|
|
140
|
+
} catch (error) {
|
|
141
|
+
log(`work-list fetch failed: ${error.message}`);
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
const rows = Array.isArray(assigned?.agents) ? assigned.agents : [];
|
|
145
|
+
|
|
146
|
+
const bound = [];
|
|
147
|
+
for (const row of rows) {
|
|
148
|
+
if (row.state === 'requested') {
|
|
149
|
+
try {
|
|
150
|
+
// eslint-disable-next-line no-await-in-loop
|
|
151
|
+
await client.post('/api/agent-binding/adopt', {
|
|
152
|
+
agentName: row.agentName, instanceId: row.instanceId,
|
|
153
|
+
});
|
|
154
|
+
log(`[${row.agentName}] adopted onto this machine`);
|
|
155
|
+
bound.push(row);
|
|
156
|
+
} catch (error) {
|
|
157
|
+
// A clean CAS refusal (409) means another machine won — drop it.
|
|
158
|
+
log(`[${row.agentName}] adopt refused: ${error.message}`);
|
|
159
|
+
}
|
|
160
|
+
} else {
|
|
161
|
+
bound.push(row);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const desiredKeys = new Set();
|
|
166
|
+
for (const row of bound) {
|
|
167
|
+
const key = identityKey(row.agentName, row.instanceId);
|
|
168
|
+
desiredKeys.add(key);
|
|
169
|
+
let seat = seats.get(key);
|
|
170
|
+
if (!seat) {
|
|
171
|
+
seat = {
|
|
172
|
+
agentName: row.agentName,
|
|
173
|
+
instanceId: row.instanceId || 'default',
|
|
174
|
+
child: null,
|
|
175
|
+
state: 'stopped',
|
|
176
|
+
restarts: 0,
|
|
177
|
+
backoffTimer: null,
|
|
178
|
+
desired: true,
|
|
179
|
+
};
|
|
180
|
+
seats.set(key, seat);
|
|
181
|
+
}
|
|
182
|
+
seat.desired = true;
|
|
183
|
+
if (!seat.child && !seat.backoffTimer) {
|
|
184
|
+
// eslint-disable-next-line no-await-in-loop
|
|
185
|
+
if (await ensureToken(row)) startChild(seat);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
for (const [key, seat] of seats) {
|
|
190
|
+
if (!desiredKeys.has(key) && seat.desired) stopSeat(seat);
|
|
191
|
+
}
|
|
192
|
+
};
|
|
193
|
+
|
|
194
|
+
const heartbeat = async () => {
|
|
195
|
+
if (stopped) return;
|
|
196
|
+
try {
|
|
197
|
+
await client.post(`/api/machines/${record.machineDbId}/heartbeat`, { agents: agentStates() });
|
|
198
|
+
} catch (error) {
|
|
199
|
+
log(`heartbeat failed: ${error.message}`);
|
|
200
|
+
}
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
const stop = () => {
|
|
204
|
+
stopped = true;
|
|
205
|
+
for (const seat of seats.values()) stopSeat(seat);
|
|
206
|
+
};
|
|
207
|
+
|
|
208
|
+
return {
|
|
209
|
+
tick, heartbeat, stop, agentStates,
|
|
210
|
+
};
|
|
211
|
+
};
|