@commonlyai/cli 0.1.209 → 0.1.210
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/commands/agent.js +53 -4
- package/src/lib/supervised-turn.js +351 -0
package/package.json
CHANGED
package/src/commands/agent.js
CHANGED
|
@@ -91,6 +91,7 @@ import { prepareMcpBeforeFirstPoll } from '../lib/mcp-first-run.js';
|
|
|
91
91
|
import { createSeatStop } from '../lib/seat-stop.js';
|
|
92
92
|
import { detectSeatbelt } from '../lib/sandbox/seatbelt.js';
|
|
93
93
|
import { describeUpstreamRefusal } from '../lib/upstream-refusal.js';
|
|
94
|
+
import { createTurnRunner, supervisedRunPolicy } from '../lib/supervised-turn.js';
|
|
94
95
|
import {
|
|
95
96
|
DEFAULT_HOOK_TIMEOUT_MS,
|
|
96
97
|
clampHookTimeoutMs,
|
|
@@ -1356,6 +1357,11 @@ const extractPrompt = (event) => {
|
|
|
1356
1357
|
return null;
|
|
1357
1358
|
};
|
|
1358
1359
|
|
|
1360
|
+
// The supervisor's policy for a run-loop spawn: lib/supervised-turn.js has the
|
|
1361
|
+
// stall windows per adapter (codex 15 min, claude off) and their evidence, the
|
|
1362
|
+
// cancel grace, and the wall-clock backstop past the adapter's own timeout.
|
|
1363
|
+
export { supervisedRunPolicy };
|
|
1364
|
+
|
|
1359
1365
|
/**
|
|
1360
1366
|
* Start the run loop. Polls /events, spawns the adapter per event, posts the
|
|
1361
1367
|
* result into the pod, acks.
|
|
@@ -1431,6 +1437,13 @@ export const performRun = ({
|
|
|
1431
1437
|
resolveOwnProviderKey = null,
|
|
1432
1438
|
runtimeReportClient = null,
|
|
1433
1439
|
refusalHandoff = null,
|
|
1440
|
+
// Off unless COMMONLY_SUPERVISE_RUNS=1: each spawn runs under the seats
|
|
1441
|
+
// supervisor (stall window, wall-clock backstop, a named failure on every
|
|
1442
|
+
// settle). The CLI still talks to its provider directly, and one metadata
|
|
1443
|
+
// line per turn goes to ~/.commonly/state/turns/<agent>.jsonl once
|
|
1444
|
+
// delivery settles. Off, the spawn path is the plain adapter call it has
|
|
1445
|
+
// always been, and nothing is written.
|
|
1446
|
+
superviseRuns = process.env.COMMONLY_SUPERVISE_RUNS === '1',
|
|
1434
1447
|
}) => {
|
|
1435
1448
|
reportRuntime = reportRuntime && WALKED_HARNESSES.includes(adapter?.name);
|
|
1436
1449
|
if (environment?.provider !== undefined && !adapter?.supportsProvider) {
|
|
@@ -1440,6 +1453,12 @@ export const performRun = ({
|
|
|
1440
1453
|
const effectiveTurnTimeoutMs = Number.isFinite(requestedTurnTimeout) && requestedTurnTimeout > 0
|
|
1441
1454
|
? Math.min(requestedTurnTimeout, 2_147_483_647) : 15 * 60 * 1000;
|
|
1442
1455
|
const client = createClient({ instance: instanceUrl, token });
|
|
1456
|
+
// The supervised run keeps its trace in memory, metadata only; no record of
|
|
1457
|
+
// a run is written to disk (lib/supervised-turn.js).
|
|
1458
|
+
const turnRunner = createTurnRunner({
|
|
1459
|
+
seat: agentName, turnTimeoutMs: effectiveTurnTimeoutMs, supervise: superviseRuns, log,
|
|
1460
|
+
});
|
|
1461
|
+
if (superviseRuns) log(turnRunner.describe(adapter?.name));
|
|
1443
1462
|
const runtimeReporter = createRuntimeReporter({
|
|
1444
1463
|
client: runtimeReportClient || { post: async () => {} },
|
|
1445
1464
|
harness: adapter.name, detect: () => adapter.detect(),
|
|
@@ -1852,8 +1871,21 @@ export const performRun = ({
|
|
|
1852
1871
|
|
|
1853
1872
|
// The spawn → post-decision half of a turn, split out so the claim release
|
|
1854
1873
|
// above is a plain finally instead of threading through every return path.
|
|
1855
|
-
|
|
1856
|
-
|
|
1874
|
+
// With superviseRuns, the turn's log line is written once delivery has
|
|
1875
|
+
// settled; anything thrown after a spawn that resolved means the reply never
|
|
1876
|
+
// reached the pod. Off, `turn` writes nothing.
|
|
1877
|
+
const runTurn = async (args) => {
|
|
1878
|
+
const turn = turnRunner.begin({ eventType: args.event.type });
|
|
1879
|
+
try {
|
|
1880
|
+
return await runTurnBody({ ...args, turn });
|
|
1881
|
+
} catch (error) {
|
|
1882
|
+
turn.finish('reply_lost');
|
|
1883
|
+
throw error;
|
|
1884
|
+
}
|
|
1885
|
+
};
|
|
1886
|
+
|
|
1887
|
+
const runTurnBody = async ({
|
|
1888
|
+
event, eventPodId, prompt, preSpawnIds, snapshotMessages, claimKeeper, trigger, turn,
|
|
1857
1889
|
}) => {
|
|
1858
1890
|
const placement = await placeTurn();
|
|
1859
1891
|
// A workspace turn placed public (its pod turned public, or its pods could
|
|
@@ -2076,7 +2108,8 @@ export const performRun = ({
|
|
|
2076
2108
|
reason: 'own-key-requires-scoped-spawn' };
|
|
2077
2109
|
}
|
|
2078
2110
|
}
|
|
2079
|
-
|
|
2111
|
+
const spawnPrompt = frameDecisionForkRule(promptForSpawn);
|
|
2112
|
+
const spawnCtx = {
|
|
2080
2113
|
onSpawn: reportRuntime ? (controls) => {
|
|
2081
2114
|
observedSpawn = true;
|
|
2082
2115
|
return runtimeReporter.spawn(controls);
|
|
@@ -2108,7 +2141,10 @@ export const performRun = ({
|
|
|
2108
2141
|
instanceUrl,
|
|
2109
2142
|
agentName,
|
|
2110
2143
|
metadata: { event },
|
|
2111
|
-
}
|
|
2144
|
+
};
|
|
2145
|
+
// Supervised or plain, a failure is thrown exactly as the adapter's own
|
|
2146
|
+
// would be, so the run loop's classification and retry apply unchanged.
|
|
2147
|
+
result = await turn.run(adapter, spawnPrompt, spawnCtx);
|
|
2112
2148
|
spawnCompleted = true;
|
|
2113
2149
|
} finally {
|
|
2114
2150
|
try {
|
|
@@ -2191,6 +2227,9 @@ export const performRun = ({
|
|
|
2191
2227
|
const upstream = result.upstream || null;
|
|
2192
2228
|
let delivered = agentPostedItself;
|
|
2193
2229
|
let deliveryRefusal = null;
|
|
2230
|
+
// What became of the reply, for the turn line (lib/supervised-turn.js
|
|
2231
|
+
// TURN_OUTCOMES). A throw below is reply_lost, finished by runTurn.
|
|
2232
|
+
let turnOutcome;
|
|
2194
2233
|
|
|
2195
2234
|
if (event.type === 'agent.ask') {
|
|
2196
2235
|
if (silentReply) {
|
|
@@ -2200,6 +2239,7 @@ export const performRun = ({
|
|
|
2200
2239
|
// having nothing to say.
|
|
2201
2240
|
if (upstream) log(`[${event.type}] ${describeUpstreamRefusal(upstream)} — no private response`);
|
|
2202
2241
|
else log(`[${event.type}] no private response (${reason})`);
|
|
2242
|
+
turnOutcome = upstream ? 'upstream_refused' : 'silent';
|
|
2203
2243
|
} else {
|
|
2204
2244
|
try {
|
|
2205
2245
|
await client.post(
|
|
@@ -2207,6 +2247,7 @@ export const performRun = ({
|
|
|
2207
2247
|
{ content: replyText },
|
|
2208
2248
|
);
|
|
2209
2249
|
delivered = true;
|
|
2250
|
+
turnOutcome = 'delivered';
|
|
2210
2251
|
log(`[${event.type}] routed private response (${Buffer.byteLength(replyText)} bytes)`);
|
|
2211
2252
|
} catch (err) {
|
|
2212
2253
|
// A tool-capable agent may have called commonly_respond_to_ask
|
|
@@ -2215,6 +2256,7 @@ export const performRun = ({
|
|
|
2215
2256
|
// the model forever.
|
|
2216
2257
|
if (err?.status === 409 && err?.body?.code === 'already_responded') {
|
|
2217
2258
|
delivered = true;
|
|
2259
|
+
turnOutcome = 'delivered';
|
|
2218
2260
|
log(`[${event.type}] response already routed by agent tool`);
|
|
2219
2261
|
} else {
|
|
2220
2262
|
throw err;
|
|
@@ -2242,6 +2284,7 @@ export const performRun = ({
|
|
|
2242
2284
|
// Report the two cases distinctly. `posted` is the load-bearing word: a
|
|
2243
2285
|
// reader scanning for whether a seat is contributing must not have to
|
|
2244
2286
|
// infer it from the absence of a different line.
|
|
2287
|
+
turnOutcome = agentPostedItself ? 'delivered' : upstream ? 'upstream_refused' : 'silent';
|
|
2245
2288
|
if (agentPostedItself) {
|
|
2246
2289
|
log(
|
|
2247
2290
|
`[${event.type}] posted via tool, then ${reason} — no echo needed `
|
|
@@ -2260,6 +2303,7 @@ export const performRun = ({
|
|
|
2260
2303
|
// Name the message that caused the suppression. A silently dropped reply
|
|
2261
2304
|
// is invisible to everyone; #757 went unnoticed precisely because this
|
|
2262
2305
|
// line said only "posted via tool" with no way to tell whose post it saw.
|
|
2306
|
+
turnOutcome = 'delivered';
|
|
2263
2307
|
log(
|
|
2264
2308
|
`[${event.type}] agent posted via tool this turn — not echoing CLI output `
|
|
2265
2309
|
+ `(avoids double-post; matched message ${suppressedBy.id} by ${suppressedBy.author} `
|
|
@@ -2269,6 +2313,7 @@ export const performRun = ({
|
|
|
2269
2313
|
// ADR-018 D3: the lease lapsed mid-turn and a peer re-won the message —
|
|
2270
2314
|
// that peer owns the conversation now, so posting would recreate the
|
|
2271
2315
|
// two-agents-one-message crossing the claim exists to prevent.
|
|
2316
|
+
turnOutcome = 'claim_lost';
|
|
2272
2317
|
log(
|
|
2273
2318
|
`[${event.type}] stood down — claim lost mid-turn to ${claimKeeper.getHolder()}; `
|
|
2274
2319
|
+ `reply suppressed (${Buffer.byteLength(replyText)} bytes not posted)`,
|
|
@@ -2311,6 +2356,7 @@ export const performRun = ({
|
|
|
2311
2356
|
// it like a refusal: re-posting the same text inside the window would
|
|
2312
2357
|
// only be skipped again.
|
|
2313
2358
|
deliveryRefusal = delivery;
|
|
2359
|
+
turnOutcome = 'server_skipped';
|
|
2314
2360
|
const windowMinutes = delivery.duplicate?.dedupeWindowMinutes;
|
|
2315
2361
|
const detail = [
|
|
2316
2362
|
windowMinutes ? `${windowMinutes} min window` : null,
|
|
@@ -2336,6 +2382,7 @@ export const performRun = ({
|
|
|
2336
2382
|
// guidance expressly says not to retry unchanged), while preserving
|
|
2337
2383
|
// the refusal and its partial-post count for the seat and event ledger.
|
|
2338
2384
|
deliveryRefusal = delivery;
|
|
2385
|
+
turnOutcome = 'server_refused';
|
|
2339
2386
|
const guidance = delivery.guidance || 'The server refused this message; do not retry it unchanged.';
|
|
2340
2387
|
const detail = `after ${delivery.messages}/${delivery.attemptedMessages} message${delivery.attemptedMessages === 1 ? '' : 's'}`;
|
|
2341
2388
|
log(`[${event.type}] wrapper delivery refused ${detail} (${delivery.reason}): ${guidance}`);
|
|
@@ -2351,12 +2398,14 @@ export const performRun = ({
|
|
|
2351
2398
|
));
|
|
2352
2399
|
} else {
|
|
2353
2400
|
delivered = true;
|
|
2401
|
+
turnOutcome = 'delivered';
|
|
2354
2402
|
log(
|
|
2355
2403
|
`[${event.type}] posted ${Buffer.byteLength(replyText)} bytes as `
|
|
2356
2404
|
+ `${delivery.messages} message${delivery.messages === 1 ? '' : 's'} (${delivery.mode})`,
|
|
2357
2405
|
);
|
|
2358
2406
|
}
|
|
2359
2407
|
}
|
|
2408
|
+
turn.finish(turnOutcome);
|
|
2360
2409
|
if (result.memorySummary && !memoryWithheld) {
|
|
2361
2410
|
try {
|
|
2362
2411
|
await syncBack(client, { summary: result.memorySummary });
|
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One model turn of `commonly agent run`, on one of two paths. The supervised
|
|
3
|
+
* path also writes one metadata line per turn (the turn log), once the
|
|
4
|
+
* turn's outcome has reached the pod or failed to.
|
|
5
|
+
*
|
|
6
|
+
* COMMONLY_SUPERVISE_RUNS=1 (read by performRun as `superviseRuns`)
|
|
7
|
+
* the turn runs under `superviseRun` from @commonlyai/seats: a stall
|
|
8
|
+
* window, a wall-clock backstop, and a guarantee the turn settles with a
|
|
9
|
+
* named failure. No provider relay: the CLI talks to its provider directly,
|
|
10
|
+
* exactly as on the plain path.
|
|
11
|
+
* anything else
|
|
12
|
+
* the plain path: `adapter.spawn(prompt, ctx)` with the caller's ctx object
|
|
13
|
+
* untouched, and nothing written: the flag-off run loop is main's.
|
|
14
|
+
*
|
|
15
|
+
* A turn is ok only when its outcome reached the pod: the reply was delivered
|
|
16
|
+
* or the agent chose silence. The run loop names the outcome after delivery
|
|
17
|
+
* settles (turn.finish); a spawn that threw is `failed` at once.
|
|
18
|
+
*
|
|
19
|
+
* NO RECORD WRITER. The supervisor needs a trace store. The one here
|
|
20
|
+
* (createMetadataStore) is in memory, lives for one turn, and keeps event type
|
|
21
|
+
* names, times and the recovery action word. It drops every event's data, so
|
|
22
|
+
* no agent text, prompt, reply, failure detail or credential is kept, and
|
|
23
|
+
* nothing is written to disk (a redaction review on #2701).
|
|
24
|
+
*
|
|
25
|
+
* Metrics: ~/.commonly/state/turns/<seat>.jsonl (COMMONLY_TURN_LOG_DIR
|
|
26
|
+
* overrides the directory). Whitelisted fields only; see metricLine.
|
|
27
|
+
*/
|
|
28
|
+
import { appendFileSync, mkdirSync } from 'fs';
|
|
29
|
+
import { homedir } from 'os';
|
|
30
|
+
import { dirname, join } from 'path';
|
|
31
|
+
import { superviseRun as defaultSuperviseRun } from '@commonlyai/seats/supervisor';
|
|
32
|
+
import { isEventType } from '@commonlyai/seats/trace/events';
|
|
33
|
+
import { failureKindOf } from '@commonlyai/seats/failures';
|
|
34
|
+
import { classifySpawnFailure } from './spawn-retry.js';
|
|
35
|
+
|
|
36
|
+
const MAX_TIMER_MS = 2_147_483_647;
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Stall windows per adapter, in ms; 0 is off. Unlisted adapters: off.
|
|
40
|
+
*
|
|
41
|
+
* codex: `codex exec --json` writes a JSONL event per item, so its stdout is a
|
|
42
|
+
* real progress signal, and the quiet stretch is the longest single item (a
|
|
43
|
+
* reasoning step or one shell command). Measured 2026-10-25 from per-turn
|
|
44
|
+
* timestamp gaps in October codex rollouts on one host (~/.codex/sessions):
|
|
45
|
+
* 9,703 turns, max quiet gap 162 s, p99 53 s; 9 turns would have tripped the
|
|
46
|
+
* package default of 120 s. 15 min is 5.5x the worst gap seen and still
|
|
47
|
+
* 30 min inside the seats' 45 min turn budget.
|
|
48
|
+
* claude: the adapter runs `claude -p --output-format text`, which writes
|
|
49
|
+
* nothing to stdout until the reply is done. A healthy 14-minute claude turn
|
|
50
|
+
* is 14 minutes of silence to the supervisor, so any window
|
|
51
|
+
* under the turn budget would be a turn-length cap, not a liveness check.
|
|
52
|
+
* Off; claude is guarded by the wall-clock backstop and the cancel ladder.
|
|
53
|
+
*/
|
|
54
|
+
export const STALL_DEFAULTS_MS = Object.freeze({ codex: 15 * 60 * 1000, claude: 0 });
|
|
55
|
+
// How long after the supervisor's abort before it SIGKILLs the child's tree
|
|
56
|
+
// (process-group.js). The claude and codex adapters do not act on the abort,
|
|
57
|
+
// so on a stall the child is alive at the kill and its whole tree goes with it:
|
|
58
|
+
// a SIGTERM to the CLI alone would let it exit first and orphan the MCP
|
|
59
|
+
// servers it started in groups of their own, which the tree kill then skips.
|
|
60
|
+
// An adapter that honours ctx.signal gets this long to settle.
|
|
61
|
+
export const CANCEL_GRACE_MS = 15_000;
|
|
62
|
+
// The wall clock sits past the adapter's own timeout, so the adapter's timeout
|
|
63
|
+
// (the same SIGTERM and error as on the plain path) fires first; the
|
|
64
|
+
// supervisor's only settles a child that outlived it.
|
|
65
|
+
export const WALL_BACKSTOP_MS = 60_000;
|
|
66
|
+
// After a failed turn, how long to wait for the child's exit status before
|
|
67
|
+
// writing the metric line anyway (the kill lands just before settle).
|
|
68
|
+
const EXIT_WAIT_MS = 3_000;
|
|
69
|
+
|
|
70
|
+
const envName = (prefix, adapterName) => `${prefix}_${String(adapterName || '').toUpperCase().replace(/[^A-Z0-9]/g, '_')}`;
|
|
71
|
+
|
|
72
|
+
const envMs = (env, name, warnings) => {
|
|
73
|
+
const raw = env?.[name];
|
|
74
|
+
if (raw === undefined || raw === '') return undefined;
|
|
75
|
+
const value = Number(raw);
|
|
76
|
+
if (!Number.isInteger(value) || value < 0) {
|
|
77
|
+
warnings.push(`${name} ignored (want a non-negative integer of ms)`);
|
|
78
|
+
return undefined;
|
|
79
|
+
}
|
|
80
|
+
return Math.min(value, MAX_TIMER_MS);
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* The supervisor policy for one adapter's turn. Env overrides, most specific
|
|
85
|
+
* first: COMMONLY_SUPERVISE_STALL_MS_<ADAPTER>, COMMONLY_SUPERVISE_STALL_MS,
|
|
86
|
+
* COMMONLY_SUPERVISE_CANCEL_GRACE_MS, COMMONLY_SUPERVISE_WALL_BACKSTOP_MS.
|
|
87
|
+
* 0 turns a window off. No retries: the run loop keeps those.
|
|
88
|
+
*/
|
|
89
|
+
export const supervisedRunPolicy = (turnTimeoutMs, { adapterName = null, env = process.env, warnings = [] } = {}) => {
|
|
90
|
+
const stallMs = envMs(env, envName('COMMONLY_SUPERVISE_STALL_MS', adapterName), warnings)
|
|
91
|
+
?? envMs(env, 'COMMONLY_SUPERVISE_STALL_MS', warnings)
|
|
92
|
+
?? STALL_DEFAULTS_MS[adapterName] ?? 0;
|
|
93
|
+
const cancelGraceMs = envMs(env, 'COMMONLY_SUPERVISE_CANCEL_GRACE_MS', warnings) ?? CANCEL_GRACE_MS;
|
|
94
|
+
const backstopMs = envMs(env, 'COMMONLY_SUPERVISE_WALL_BACKSTOP_MS', warnings) ?? WALL_BACKSTOP_MS;
|
|
95
|
+
return {
|
|
96
|
+
stallMs,
|
|
97
|
+
cancelGraceMs,
|
|
98
|
+
timeoutMs: Math.min(Number(turnTimeoutMs) + backstopMs, MAX_TIMER_MS),
|
|
99
|
+
};
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* A trace store that keeps no agent text and writes nothing. superviseRun
|
|
104
|
+
* calls startRun once per turn; each event keeps { type, at, action? }.
|
|
105
|
+
*/
|
|
106
|
+
export const createMetadataStore = ({ now = Date.now } = {}) => {
|
|
107
|
+
let current = null;
|
|
108
|
+
let seq = 0;
|
|
109
|
+
return {
|
|
110
|
+
startRun({ runId } = {}) {
|
|
111
|
+
seq += 1;
|
|
112
|
+
const events = [];
|
|
113
|
+
let settled = false;
|
|
114
|
+
current = {
|
|
115
|
+
runId: runId || `turn-${now().toString(36)}-${seq}`,
|
|
116
|
+
emit(type, data) {
|
|
117
|
+
if (!isEventType(type)) throw new Error(`unknown trace event type: ${type}`);
|
|
118
|
+
if (settled) return null;
|
|
119
|
+
const entry = { type, at: now() };
|
|
120
|
+
if (type === 'liveness.recovery' && typeof data?.action === 'string') entry.action = data.action;
|
|
121
|
+
events.push(entry);
|
|
122
|
+
if (type === 'run.settled') settled = true;
|
|
123
|
+
return entry;
|
|
124
|
+
},
|
|
125
|
+
get settled() { return settled; },
|
|
126
|
+
events: () => events.map((entry) => ({ ...entry })),
|
|
127
|
+
};
|
|
128
|
+
return current;
|
|
129
|
+
},
|
|
130
|
+
lastRun: () => current,
|
|
131
|
+
};
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
export const metricsPathFor = (seat, env = process.env) => {
|
|
135
|
+
const dir = env?.COMMONLY_TURN_LOG_DIR || join(homedir(), '.commonly', 'state', 'turns');
|
|
136
|
+
const safe = String(seat || 'unknown').replace(/[^A-Za-z0-9_.-]/g, '_').slice(0, 64) || 'unknown';
|
|
137
|
+
return join(dir, `${safe}.jsonl`);
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
/** A failure's name, never its text. RunFailure kinds are the supervisor's words. */
|
|
141
|
+
export const failureNameOf = (error) => {
|
|
142
|
+
const kind = failureKindOf(error) || (typeof error?.kind === 'string' && /^[a-z_]{1,40}$/.test(error.kind) ? error.kind : null);
|
|
143
|
+
if (kind) return { name: kind, source: 'supervisor' };
|
|
144
|
+
const message = String(error?.message ?? error ?? '');
|
|
145
|
+
if (/\btimed out after \d+ms\b/.test(message)) return { name: 'adapter_timeout', source: 'adapter' };
|
|
146
|
+
if (/\bexited with code\b/.test(message)) return { name: 'adapter_exit', source: 'adapter' };
|
|
147
|
+
return { name: 'adapter_error', source: 'adapter' };
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
const failureClassOf = (error) => {
|
|
151
|
+
try { return classifySpawnFailure(error); } catch { return null; }
|
|
152
|
+
};
|
|
153
|
+
|
|
154
|
+
// What happened to a turn, named by the run loop once delivery settles.
|
|
155
|
+
export const TURN_OUTCOMES = Object.freeze([
|
|
156
|
+
'delivered', // the reply reached the pod (wrapper post, the agent's own post, or an ask response)
|
|
157
|
+
'silent', // NO_REPLY, a heartbeat control word, or empty output: nothing to post
|
|
158
|
+
'reply_lost', // the model answered and every delivery rung threw
|
|
159
|
+
'claim_lost', // the lease lapsed mid-turn and a peer owns the message; reply suppressed
|
|
160
|
+
'server_skipped', // the server answered and created nothing (a duplicate, a kept heartbeat)
|
|
161
|
+
'server_refused', // the server refused the post (a run cap)
|
|
162
|
+
'upstream_refused', // the model route refused; the CLI exited with nothing to say
|
|
163
|
+
]);
|
|
164
|
+
export const OK_OUTCOMES = Object.freeze(['delivered', 'silent']);
|
|
165
|
+
|
|
166
|
+
const EVENT_TYPE_RE = /^[A-Za-z0-9._-]{1,40}$/;
|
|
167
|
+
const iso = (ms) => new Date(ms).toISOString();
|
|
168
|
+
const int = (value) => (Number.isInteger(value) ? value : null);
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* The metric line. Every field is listed here, each a number, boolean, null or
|
|
172
|
+
* a short fixed word; nothing from the prompt, the reply, a failure's detail or
|
|
173
|
+
* the environment can reach it.
|
|
174
|
+
*/
|
|
175
|
+
export const metricLine = (f) => {
|
|
176
|
+
const failed = f.outcome === 'failed';
|
|
177
|
+
const outcome = failed || TURN_OUTCOMES.includes(f.outcome) ? f.outcome : 'unknown';
|
|
178
|
+
return {
|
|
179
|
+
v: 1,
|
|
180
|
+
ts: iso(f.settledMs ?? f.endMs),
|
|
181
|
+
seat: String(f.seat || 'unknown').replace(/[^A-Za-z0-9_.-]/g, '_').slice(0, 64),
|
|
182
|
+
adapter: String(f.adapter || 'unknown').replace(/[^A-Za-z0-9_.-]/g, '_').slice(0, 32),
|
|
183
|
+
path: f.path === 'supervised' ? 'supervised' : 'legacy',
|
|
184
|
+
eventType: EVENT_TYPE_RE.test(String(f.eventType || '')) ? f.eventType : null,
|
|
185
|
+
turnStart: iso(f.startMs),
|
|
186
|
+
turnEnd: iso(f.endMs),
|
|
187
|
+
durationMs: Math.max(0, f.endMs - f.startMs),
|
|
188
|
+
outcome,
|
|
189
|
+
ok: OK_OUTCOMES.includes(outcome),
|
|
190
|
+
failure: failed ? (f.failureName || 'adapter_error') : null,
|
|
191
|
+
failureSource: failed ? (f.failureSource || null) : null,
|
|
192
|
+
failureClass: failed && typeof f.failureClass === 'string' ? f.failureClass : null,
|
|
193
|
+
stalled: !!f.stalled,
|
|
194
|
+
cancelled: !!f.cancelled,
|
|
195
|
+
killed: !!f.killed,
|
|
196
|
+
exitCode: int(f.exitCode),
|
|
197
|
+
signal: typeof f.signal === 'string' && /^SIG[A-Z0-9]+$/.test(f.signal) ? f.signal : null,
|
|
198
|
+
exitObserved: !!f.exitObserved,
|
|
199
|
+
childLeftRunning: typeof f.childLeftRunning === 'boolean' ? f.childLeftRunning : null,
|
|
200
|
+
attempts: int(f.attempts),
|
|
201
|
+
stallMs: int(f.policy?.stallMs),
|
|
202
|
+
cancelGraceMs: int(f.policy?.cancelGraceMs),
|
|
203
|
+
timeoutMs: int(f.policy?.timeoutMs),
|
|
204
|
+
};
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
const isChild = (value) => !!value && typeof value.once === 'function' && typeof value.kill === 'function';
|
|
208
|
+
const isAlive = (child) => !!child && child.exitCode === null && child.signalCode === null;
|
|
209
|
+
|
|
210
|
+
const waitForExit = (child, ms) => new Promise((resolve) => {
|
|
211
|
+
if (!isAlive(child)) { resolve(); return; }
|
|
212
|
+
const timer = setTimeout(resolve, ms);
|
|
213
|
+
timer.unref?.();
|
|
214
|
+
child.once('exit', () => { clearTimeout(timer); resolve(); });
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* @param {object} args
|
|
219
|
+
* @param {string} args.seat agent name; names the metrics file
|
|
220
|
+
* @param {number} args.turnTimeoutMs the run loop's effective turn budget
|
|
221
|
+
* @param {boolean} args.supervise performRun's superviseRuns
|
|
222
|
+
* @param {(line: string) => void} [args.log]
|
|
223
|
+
* @param {object} [args.env]
|
|
224
|
+
* @param {Function} [args.superviseRun] injectable for tests
|
|
225
|
+
*/
|
|
226
|
+
export const createTurnRunner = ({
|
|
227
|
+
seat,
|
|
228
|
+
turnTimeoutMs,
|
|
229
|
+
supervise = false,
|
|
230
|
+
log = () => {},
|
|
231
|
+
env = process.env,
|
|
232
|
+
now = Date.now,
|
|
233
|
+
superviseRun = defaultSuperviseRun,
|
|
234
|
+
} = {}) => {
|
|
235
|
+
const metricsPath = metricsPathFor(seat, env);
|
|
236
|
+
let metricsWarned = false;
|
|
237
|
+
|
|
238
|
+
const record = (fields) => {
|
|
239
|
+
try {
|
|
240
|
+
mkdirSync(dirname(metricsPath), { recursive: true, mode: 0o700 });
|
|
241
|
+
appendFileSync(metricsPath, `${JSON.stringify(metricLine({ seat, ...fields }))}\n`, { mode: 0o600 });
|
|
242
|
+
} catch (error) {
|
|
243
|
+
// Metrics never fail a turn. Said once per process, not per turn.
|
|
244
|
+
if (!metricsWarned) {
|
|
245
|
+
metricsWarned = true;
|
|
246
|
+
log(`turn metrics not written (${error?.code || 'error'}); turns are unaffected`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
};
|
|
250
|
+
|
|
251
|
+
/** The boot line for the supervised path; the plain path logs nothing new. */
|
|
252
|
+
const describe = (adapterName) => {
|
|
253
|
+
const warnings = [];
|
|
254
|
+
const policy = supervisedRunPolicy(turnTimeoutMs, { adapterName, env, warnings });
|
|
255
|
+
return `supervised runs ON for ${adapterName}: stall=${policy.stallMs ? `${policy.stallMs}ms` : 'off'}, `
|
|
256
|
+
+ `cancel grace=${policy.cancelGraceMs}ms, wall=${policy.timeoutMs}ms, `
|
|
257
|
+
+ 'trace=memory (no record written); '
|
|
258
|
+
+ `metrics → ${metricsPath}${warnings.length ? `; ${warnings.join('; ')}` : ''}`;
|
|
259
|
+
};
|
|
260
|
+
|
|
261
|
+
const runSupervised = async (adapter, prompt, ctx, eventType, hold) => {
|
|
262
|
+
const startMs = now();
|
|
263
|
+
const policy = supervisedRunPolicy(turnTimeoutMs, { adapterName: adapter?.name, env });
|
|
264
|
+
const store = createMetadataStore({ now });
|
|
265
|
+
let child = null;
|
|
266
|
+
let exit = null;
|
|
267
|
+
const callerOnSpawn = ctx?.onSpawn;
|
|
268
|
+
const onSpawn = (...args) => {
|
|
269
|
+
const proc = args.find(isChild);
|
|
270
|
+
if (proc) {
|
|
271
|
+
child = proc;
|
|
272
|
+
exit = null;
|
|
273
|
+
proc.once('exit', (code, signal) => { if (child === proc) exit = { code, signal }; });
|
|
274
|
+
}
|
|
275
|
+
return typeof callerOnSpawn === 'function' ? callerOnSpawn(...args) : undefined;
|
|
276
|
+
};
|
|
277
|
+
|
|
278
|
+
const supervised = await superviseRun({
|
|
279
|
+
adapter,
|
|
280
|
+
prompt,
|
|
281
|
+
ctx: { ...ctx, onSpawn },
|
|
282
|
+
store,
|
|
283
|
+
policy,
|
|
284
|
+
label: EVENT_TYPE_RE.test(String(eventType || '')) ? eventType : undefined,
|
|
285
|
+
});
|
|
286
|
+
|
|
287
|
+
const ok = !supervised.failure;
|
|
288
|
+
if (!ok) await waitForExit(child, EXIT_WAIT_MS);
|
|
289
|
+
const events = store.lastRun()?.events() || [];
|
|
290
|
+
const recovered = (action) => events.some((e) => e.type === 'liveness.recovery' && e.action === action);
|
|
291
|
+
const killed = recovered('kill');
|
|
292
|
+
const named = ok ? null : failureNameOf(supervised.failure);
|
|
293
|
+
const fields = {
|
|
294
|
+
adapter: adapter?.name,
|
|
295
|
+
path: 'supervised',
|
|
296
|
+
eventType,
|
|
297
|
+
startMs,
|
|
298
|
+
endMs: now(),
|
|
299
|
+
outcome: ok ? undefined : 'failed',
|
|
300
|
+
failureName: named?.name,
|
|
301
|
+
failureSource: named?.source,
|
|
302
|
+
failureClass: ok ? null : failureClassOf(supervised.failure),
|
|
303
|
+
stalled: events.some((e) => e.type === 'liveness.stalled'),
|
|
304
|
+
cancelled: recovered('cancel'),
|
|
305
|
+
killed,
|
|
306
|
+
exitCode: exit ? exit.code : null,
|
|
307
|
+
signal: exit ? exit.signal : null,
|
|
308
|
+
exitObserved: !!exit,
|
|
309
|
+
childLeftRunning: child ? isAlive(child) : null,
|
|
310
|
+
attempts: supervised.attempts,
|
|
311
|
+
policy,
|
|
312
|
+
};
|
|
313
|
+
// A spawn that resolved is not yet an ok turn: the line waits for the run
|
|
314
|
+
// loop to say what became of the reply.
|
|
315
|
+
if (ok) { hold(fields); return supervised.result; }
|
|
316
|
+
record(fields);
|
|
317
|
+
if (named.source === 'supervisor') {
|
|
318
|
+
// The name goes in the seat log. The throw takes the road a plain-path
|
|
319
|
+
// failure takes: unacked, the run loop's retry policy, onError.
|
|
320
|
+
log(`[${eventType}] supervised turn failed: ${named.name}${killed ? ' (child killed)' : ''}`
|
|
321
|
+
+ `${isAlive(child) ? ' (child still running)' : ''}`);
|
|
322
|
+
}
|
|
323
|
+
throw supervised.failure;
|
|
324
|
+
};
|
|
325
|
+
|
|
326
|
+
return {
|
|
327
|
+
supervise,
|
|
328
|
+
metricsPath,
|
|
329
|
+
describe,
|
|
330
|
+
/**
|
|
331
|
+
* One model turn. `run` has adapter.spawn's contract; `finish(outcome)`
|
|
332
|
+
* writes the line once delivery has settled. Only the first finish counts,
|
|
333
|
+
* and a finish with no spawned turn behind it writes nothing.
|
|
334
|
+
*/
|
|
335
|
+
begin({ eventType } = {}) {
|
|
336
|
+
let held = null;
|
|
337
|
+
let done = false;
|
|
338
|
+
return {
|
|
339
|
+
run(adapter, prompt, ctx) {
|
|
340
|
+
if (!supervise) return adapter.spawn(prompt, ctx);
|
|
341
|
+
return runSupervised(adapter, prompt, ctx, eventType, (fields) => { held = fields; });
|
|
342
|
+
},
|
|
343
|
+
finish(outcome) {
|
|
344
|
+
if (done || !held) return;
|
|
345
|
+
done = true;
|
|
346
|
+
record({ ...held, outcome, settledMs: now() });
|
|
347
|
+
},
|
|
348
|
+
};
|
|
349
|
+
},
|
|
350
|
+
};
|
|
351
|
+
};
|