@animalabs/agent-framework 0.6.4 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/src/agent.d.ts.map +1 -1
- package/dist/src/agent.js +20 -0
- package/dist/src/agent.js.map +1 -1
- package/dist/src/framework.d.ts +44 -1
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +264 -143
- package/dist/src/framework.js.map +1 -1
- package/dist/src/types/framework.d.ts +9 -0
- package/dist/src/types/framework.d.ts.map +1 -1
- package/dist/test/activation-request-sanitize.test.d.ts +2 -0
- package/dist/test/activation-request-sanitize.test.d.ts.map +1 -0
- package/dist/test/activation-request-sanitize.test.js +89 -0
- package/dist/test/activation-request-sanitize.test.js.map +1 -0
- package/dist/test/ephemeral-settle.test.d.ts +2 -0
- package/dist/test/ephemeral-settle.test.d.ts.map +1 -0
- package/dist/test/ephemeral-settle.test.js +351 -0
- package/dist/test/ephemeral-settle.test.js.map +1 -0
- package/dist/test/maintenance-timer.test.js +49 -0
- package/dist/test/maintenance-timer.test.js.map +1 -1
- package/dist/test/overbudget-drain-breaker.test.d.ts +2 -0
- package/dist/test/overbudget-drain-breaker.test.d.ts.map +1 -0
- package/dist/test/overbudget-drain-breaker.test.js +157 -0
- package/dist/test/overbudget-drain-breaker.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/agent.ts +22 -0
- package/src/framework.ts +295 -143
- package/src/types/framework.ts +10 -0
package/src/framework.ts
CHANGED
|
@@ -32,6 +32,7 @@ import type {
|
|
|
32
32
|
ProcessEvent,
|
|
33
33
|
EventResponse,
|
|
34
34
|
ModuleProcessResponse,
|
|
35
|
+
AgentSettleResult,
|
|
35
36
|
ToolCall,
|
|
36
37
|
ToolCallEvent,
|
|
37
38
|
ToolResult,
|
|
@@ -229,6 +230,23 @@ interface RewindRecord {
|
|
|
229
230
|
discordRef?: { channelId: string; messageId: string };
|
|
230
231
|
}
|
|
231
232
|
|
|
233
|
+
interface Deferred<T> {
|
|
234
|
+
promise: Promise<T>;
|
|
235
|
+
resolve: (value: T) => void;
|
|
236
|
+
reject: (reason?: unknown) => void;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** Per-run state for a framework-driven ephemeral agent: the settle signal
|
|
240
|
+
* the caller awaits plus the liveness/progress bookkeeping the watchdogs and
|
|
241
|
+
* the stream driver share. One map entry per run — created and torn down in
|
|
242
|
+
* runEphemeralToCompletion — so the pieces cannot desync. */
|
|
243
|
+
interface EphemeralRun {
|
|
244
|
+
settle: Deferred<AgentSettleResult>;
|
|
245
|
+
inferenceStarted: boolean;
|
|
246
|
+
lastActivity: number;
|
|
247
|
+
toolCallsCount: number;
|
|
248
|
+
}
|
|
249
|
+
|
|
232
250
|
/** Cap an API error message for inline use in a rewind marker: keep enough to
|
|
233
251
|
* identify the failure class without pasting a wall of provider JSON into the
|
|
234
252
|
* agent's context. */
|
|
@@ -275,6 +293,17 @@ export class AgentFramework {
|
|
|
275
293
|
private processLoggingBroadcast: boolean;
|
|
276
294
|
private activeStreams: Map<string, Promise<void>> = new Map();
|
|
277
295
|
private pendingAssistantBlocks: Map<string, ContentBlock[]> = new Map();
|
|
296
|
+
/** Streams the FRAMEWORK cancelled for non-terminal reasons, keyed
|
|
297
|
+
* `${agentName}:${streamId}`: an endTurn tool result or a context-budget
|
|
298
|
+
* restart. The membrane still delivers an `aborted` event for these, and
|
|
299
|
+
* without the marker the abort handler would treat it as a terminal
|
|
300
|
+
* failure — settling ephemerals with a rejection and emitting a spurious
|
|
301
|
+
* `inference:exhausted` (which also pollutes the failure streak). Kept
|
|
302
|
+
* separate from ephemeralRuns deliberately: endTurn/budget cancels happen
|
|
303
|
+
* for resident agents too, and the key is per-stream, not per-agent. */
|
|
304
|
+
private frameworkCancelledStreams: Map<string, 'turn_ended' | 'budget_restart'> = new Map();
|
|
305
|
+
/** Active runEphemeralToCompletion runs, keyed by agent name. */
|
|
306
|
+
private ephemeralRuns: Map<string, EphemeralRun> = new Map();
|
|
278
307
|
/** Per-agent count of consecutive exhausted inferences (reset on any success).
|
|
279
308
|
* Drives hard-down escalation — see noteInferenceExhausted. */
|
|
280
309
|
private consecutiveInferenceFailures: Map<string, number> = new Map();
|
|
@@ -303,6 +332,13 @@ export class AgentFramework {
|
|
|
303
332
|
* Bounds the automatic quarantine loop the same way refusalRewinds bounds
|
|
304
333
|
* the refusal loop, so the breaker can never shed the whole history. */
|
|
305
334
|
private exhaustionRewinds: Map<string, number> = new Map();
|
|
335
|
+
/** Agents with an OverBudget drain kick currently in flight. The breaker in
|
|
336
|
+
* noteInferenceExhausted fires on EVERY matching failure and the scenario it
|
|
337
|
+
* exists for is "every activation fails, repeatedly" — without this guard,
|
|
338
|
+
* overlapping kicks race the strategy's own pendingCompression gate and the
|
|
339
|
+
* tick counts in the success log stop meaning anything. One kick per agent
|
|
340
|
+
* at a time; cleared when the kick settles (success or failure). */
|
|
341
|
+
private overBudgetDrainInFlight: Set<string> = new Set();
|
|
306
342
|
/** Per-agent current rewind episode: the single consolidated marker's id and
|
|
307
343
|
* how many turns have been shed so far. One marker per episode, updated in
|
|
308
344
|
* place (see updateRewindMarker); cleared when the episode ends. */
|
|
@@ -732,9 +768,16 @@ export class AgentFramework {
|
|
|
732
768
|
private startQueuedMaintenance(): void {
|
|
733
769
|
if (this.maintenancePass || !this.running) return;
|
|
734
770
|
const pass = this.runQueuedMaintenance().catch((error) => {
|
|
771
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
735
772
|
console.error(
|
|
736
773
|
'[context-maintenance] pass failed:',
|
|
737
|
-
|
|
774
|
+
reason,
|
|
775
|
+
);
|
|
776
|
+
this.opsAlert(
|
|
777
|
+
'context-maintenance-failed',
|
|
778
|
+
'framework',
|
|
779
|
+
reason,
|
|
780
|
+
{ data: { scope: 'pass' } },
|
|
738
781
|
);
|
|
739
782
|
});
|
|
740
783
|
this.maintenancePass = pass;
|
|
@@ -789,6 +832,18 @@ export class AgentFramework {
|
|
|
789
832
|
} catch (error) {
|
|
790
833
|
record.error = error instanceof Error ? error.message : String(error);
|
|
791
834
|
console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
|
|
835
|
+
this.opsAlert(
|
|
836
|
+
'context-maintenance-failed',
|
|
837
|
+
agent.name,
|
|
838
|
+
record.error,
|
|
839
|
+
{
|
|
840
|
+
data: {
|
|
841
|
+
scope: 'agent',
|
|
842
|
+
...(record.pendingBefore ? { pending: record.pendingBefore } : {}),
|
|
843
|
+
...(record.progressBefore ? { progress: record.progressBefore } : {}),
|
|
844
|
+
},
|
|
845
|
+
},
|
|
846
|
+
);
|
|
792
847
|
} finally {
|
|
793
848
|
const pending = cm.getPendingWork()?.description;
|
|
794
849
|
record.finishedAt = Date.now();
|
|
@@ -1177,6 +1232,54 @@ export class AgentFramework {
|
|
|
1177
1232
|
return { agent, contextManager, cleanup };
|
|
1178
1233
|
}
|
|
1179
1234
|
|
|
1235
|
+
private createDeferred<T>(): Deferred<T> {
|
|
1236
|
+
let resolve!: (value: T) => void;
|
|
1237
|
+
let reject!: (reason?: unknown) => void;
|
|
1238
|
+
const promise = new Promise<T>((res, rej) => {
|
|
1239
|
+
resolve = res;
|
|
1240
|
+
reject = rej;
|
|
1241
|
+
});
|
|
1242
|
+
return { promise, resolve, reject };
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
/** Liveness ping for an active ephemeral run (no-op otherwise). */
|
|
1246
|
+
private touchEphemeralRun(agentName: string, inferenceStarted = false): void {
|
|
1247
|
+
const run = this.ephemeralRuns.get(agentName);
|
|
1248
|
+
if (!run) return;
|
|
1249
|
+
if (inferenceStarted) {
|
|
1250
|
+
run.inferenceStarted = true;
|
|
1251
|
+
}
|
|
1252
|
+
run.lastActivity = Date.now();
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
private recordEphemeralToolCalls(agentName: string, count: number): void {
|
|
1256
|
+
const run = this.ephemeralRuns.get(agentName);
|
|
1257
|
+
if (!run) return;
|
|
1258
|
+
run.toolCallsCount += count;
|
|
1259
|
+
run.lastActivity = Date.now();
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
/** Settle an active ephemeral run (no-op for resident agents). The tool-call
|
|
1263
|
+
* count comes from the run's own counter — the single source of truth,
|
|
1264
|
+
* which unlike a per-stream local survives budget restarts. */
|
|
1265
|
+
private settleAgent(
|
|
1266
|
+
agentName: string,
|
|
1267
|
+
outcome: { stopReason: AgentSettleResult['stopReason']; speech: string; error?: string },
|
|
1268
|
+
): void {
|
|
1269
|
+
const run = this.ephemeralRuns.get(agentName);
|
|
1270
|
+
if (!run) return;
|
|
1271
|
+
run.lastActivity = Date.now();
|
|
1272
|
+
if (outcome.stopReason === 'exhausted') {
|
|
1273
|
+
run.settle.reject(new Error(outcome.error ?? 'Unknown error'));
|
|
1274
|
+
return;
|
|
1275
|
+
}
|
|
1276
|
+
run.settle.resolve({
|
|
1277
|
+
stopReason: outcome.stopReason,
|
|
1278
|
+
speech: outcome.speech,
|
|
1279
|
+
toolCallsCount: run.toolCallsCount,
|
|
1280
|
+
});
|
|
1281
|
+
}
|
|
1282
|
+
|
|
1180
1283
|
/**
|
|
1181
1284
|
* Run an ephemeral agent to completion through the framework's event loop.
|
|
1182
1285
|
*
|
|
@@ -1190,42 +1293,31 @@ export class AgentFramework {
|
|
|
1190
1293
|
async runEphemeralToCompletion(
|
|
1191
1294
|
agent: Agent,
|
|
1192
1295
|
contextManager: ContextManager,
|
|
1296
|
+
watchdogs?: { startupTimeoutMs?: number; idleTimeoutMs?: number; idlePollMs?: number },
|
|
1193
1297
|
): Promise<{ speech: string; toolCallsCount: number }> {
|
|
1194
1298
|
// Register temporarily so the event loop can drive it
|
|
1195
1299
|
this.agents.set(agent.name, agent);
|
|
1300
|
+
const run: EphemeralRun = {
|
|
1301
|
+
settle: this.createDeferred<AgentSettleResult>(),
|
|
1302
|
+
inferenceStarted: false,
|
|
1303
|
+
lastActivity: Date.now(),
|
|
1304
|
+
toolCallsCount: 0,
|
|
1305
|
+
};
|
|
1306
|
+
this.ephemeralRuns.set(agent.name, run);
|
|
1196
1307
|
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
const STARTUP_TIMEOUT_MS = 30_000;
|
|
1205
|
-
// After inference has started, give it 15 minutes of activity-bounded
|
|
1206
|
-
// life. Each addressed trace event refreshes the deadline; only
|
|
1207
|
-
// sustained silence trips it. Subagents that legitimately stream for
|
|
1208
|
-
// 10+ minutes (e.g. long fork investigations) get the extra slack but
|
|
1209
|
-
// stalled streams that emit `inference:completed` and then go quiet —
|
|
1210
|
-
// the exact zombie shape we saw in production — are caught here.
|
|
1211
|
-
const COMPLETION_IDLE_TIMEOUT_MS = 15 * 60_000;
|
|
1212
|
-
|
|
1213
|
-
const cleanup = () => {
|
|
1214
|
-
if (settled) return;
|
|
1215
|
-
settled = true;
|
|
1216
|
-
clearTimeout(startupWatchdog);
|
|
1217
|
-
clearInterval(completionWatchdog);
|
|
1218
|
-
this.offTrace(traceListener);
|
|
1219
|
-
this.agents.delete(agent.name);
|
|
1220
|
-
this.evictTurnCheckpoints(agent.name);
|
|
1221
|
-
};
|
|
1308
|
+
const STARTUP_TIMEOUT_MS = watchdogs?.startupTimeoutMs ?? 30_000;
|
|
1309
|
+
// After inference has started, give it 15 minutes of activity-bounded
|
|
1310
|
+
// life. The stream driver refreshes the deadline as it makes progress;
|
|
1311
|
+
// sustained silence trips it.
|
|
1312
|
+
const COMPLETION_IDLE_TIMEOUT_MS = watchdogs?.idleTimeoutMs ?? 15 * 60_000;
|
|
1313
|
+
const IDLE_POLL_MS = watchdogs?.idlePollMs ?? 30_000;
|
|
1222
1314
|
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1315
|
+
let startupWatchdog: ReturnType<typeof setTimeout> | null = null;
|
|
1316
|
+
let completionWatchdog: ReturnType<typeof setInterval> | null = null;
|
|
1317
|
+
|
|
1318
|
+
const startupTimeout = new Promise<never>((_, reject) => {
|
|
1319
|
+
startupWatchdog = setTimeout(() => {
|
|
1320
|
+
if (!run.inferenceStarted) {
|
|
1229
1321
|
reject(new Error(
|
|
1230
1322
|
`Ephemeral agent "${agent.name}" failed to start inference within ` +
|
|
1231
1323
|
`${STARTUP_TIMEOUT_MS}ms — zombie detected. The event loop may have ` +
|
|
@@ -1233,104 +1325,48 @@ export class AgentFramework {
|
|
|
1233
1325
|
));
|
|
1234
1326
|
}
|
|
1235
1327
|
}, STARTUP_TIMEOUT_MS);
|
|
1328
|
+
});
|
|
1236
1329
|
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
// and keeps its concurrency slot pinned.
|
|
1242
|
-
const completionWatchdog = setInterval(() => {
|
|
1243
|
-
if (settled || !inferenceStarted) return;
|
|
1244
|
-
const idle = Date.now() - lastActivity;
|
|
1330
|
+
const completionIdleTimeout = new Promise<never>((_, reject) => {
|
|
1331
|
+
completionWatchdog = setInterval(() => {
|
|
1332
|
+
if (!run.inferenceStarted) return;
|
|
1333
|
+
const idle = Date.now() - run.lastActivity;
|
|
1245
1334
|
if (idle > COMPLETION_IDLE_TIMEOUT_MS) {
|
|
1246
|
-
cleanup();
|
|
1247
1335
|
reject(new Error(
|
|
1248
|
-
`Ephemeral agent "${agent.name}" stalled: no
|
|
1336
|
+
`Ephemeral agent "${agent.name}" stalled: no stream-driver activity for ` +
|
|
1249
1337
|
`${Math.round(idle / 1000)}s after inference started (threshold ` +
|
|
1250
1338
|
`${Math.round(COMPLETION_IDLE_TIMEOUT_MS / 1000)}s). Stream likely ` +
|
|
1251
1339
|
`dropped or terminal event was lost.`
|
|
1252
1340
|
));
|
|
1253
1341
|
}
|
|
1254
|
-
},
|
|
1255
|
-
|
|
1256
|
-
const traceListener = (event: TraceEvent) => {
|
|
1257
|
-
if (settled) return;
|
|
1258
|
-
// Only track events for our ephemeral agent
|
|
1259
|
-
const agentName = 'agentName' in event ? (event as { agentName: string }).agentName : null;
|
|
1260
|
-
if (agentName !== agent.name) return;
|
|
1261
|
-
|
|
1262
|
-
// Any addressed event counts as liveness for the completion watchdog.
|
|
1263
|
-
lastActivity = Date.now();
|
|
1264
|
-
|
|
1265
|
-
switch (event.type) {
|
|
1266
|
-
case 'inference:started':
|
|
1267
|
-
inferenceStarted = true;
|
|
1268
|
-
break;
|
|
1269
|
-
case 'inference:tokens': {
|
|
1270
|
-
inferenceStarted = true;
|
|
1271
|
-
const content = (event as { content?: string }).content;
|
|
1272
|
-
if (content) speech += content;
|
|
1273
|
-
break;
|
|
1274
|
-
}
|
|
1275
|
-
case 'inference:tool_calls_yielded': {
|
|
1276
|
-
const calls = (event as { calls?: Array<unknown> }).calls;
|
|
1277
|
-
if (calls) toolCallsCount += calls.length;
|
|
1278
|
-
// Reset speech buffer — tool calls break the text
|
|
1279
|
-
speech = '';
|
|
1280
|
-
break;
|
|
1281
|
-
}
|
|
1282
|
-
case 'inference:stream_resumed':
|
|
1283
|
-
speech = '';
|
|
1284
|
-
break;
|
|
1285
|
-
case 'inference:completed': {
|
|
1286
|
-
// `inference:completed` is terminal — driveStream only emits it
|
|
1287
|
-
// from `case 'complete'`, which fires after the full tool
|
|
1288
|
-
// round-trip has finished and the model has stopped. Intermediate
|
|
1289
|
-
// tool-cycle transitions use `inference:stream_resumed`, not
|
|
1290
|
-
// `inference:completed`. So we resolve unconditionally here.
|
|
1291
|
-
//
|
|
1292
|
-
// Previously this branch gated on `agent.state.status === 'idle'`,
|
|
1293
|
-
// which was correct in intent but order-fragile: the framework
|
|
1294
|
-
// emitted `inference:completed` BEFORE calling `agent.reset()`,
|
|
1295
|
-
// so synchronous listeners observed status === 'streaming' at
|
|
1296
|
-
// the trace boundary. The gate failed silently and the promise
|
|
1297
|
-
// hung forever — every production zombie subagent originated
|
|
1298
|
-
// here. The reset+emit order has been fixed in driveStream's
|
|
1299
|
-
// `case 'complete'` handler; the gate is dropped here as
|
|
1300
|
-
// belt-and-suspenders.
|
|
1301
|
-
cleanup();
|
|
1302
|
-
resolve({ speech, toolCallsCount });
|
|
1303
|
-
break;
|
|
1304
|
-
}
|
|
1305
|
-
case 'inference:turn_ended': {
|
|
1306
|
-
// endTurn from a tool result — agent is done
|
|
1307
|
-
cleanup();
|
|
1308
|
-
resolve({ speech, toolCallsCount });
|
|
1309
|
-
break;
|
|
1310
|
-
}
|
|
1311
|
-
case 'inference:exhausted': {
|
|
1312
|
-
// Retries exhausted — reject only after the error policy gives up.
|
|
1313
|
-
// Earlier inference:failed events are left alone so the framework
|
|
1314
|
-
// retry path in driveStream/startAgentStream can restart the stream
|
|
1315
|
-
// while this listener stays alive.
|
|
1316
|
-
const error = (event as { error?: string }).error ?? 'Unknown error';
|
|
1317
|
-
cleanup();
|
|
1318
|
-
reject(new Error(error));
|
|
1319
|
-
break;
|
|
1320
|
-
}
|
|
1321
|
-
}
|
|
1322
|
-
};
|
|
1323
|
-
|
|
1324
|
-
this.onTrace(traceListener);
|
|
1342
|
+
}, IDLE_POLL_MS);
|
|
1343
|
+
});
|
|
1325
1344
|
|
|
1326
|
-
|
|
1345
|
+
try {
|
|
1346
|
+
// Trigger inference after the settle promise is registered.
|
|
1327
1347
|
this.pendingRequests.push({
|
|
1328
1348
|
agentName: agent.name,
|
|
1329
1349
|
reason: 'ephemeral',
|
|
1330
1350
|
source: 'subagent',
|
|
1331
1351
|
timestamp: Date.now(),
|
|
1332
1352
|
});
|
|
1333
|
-
|
|
1353
|
+
|
|
1354
|
+
const result = await Promise.race([
|
|
1355
|
+
run.settle.promise,
|
|
1356
|
+
startupTimeout,
|
|
1357
|
+
completionIdleTimeout,
|
|
1358
|
+
]);
|
|
1359
|
+
return { speech: result.speech, toolCallsCount: result.toolCallsCount };
|
|
1360
|
+
} finally {
|
|
1361
|
+
if (startupWatchdog) clearTimeout(startupWatchdog);
|
|
1362
|
+
if (completionWatchdog) clearInterval(completionWatchdog);
|
|
1363
|
+
this.ephemeralRuns.delete(agent.name);
|
|
1364
|
+
this.agents.delete(agent.name);
|
|
1365
|
+
// Spawn-and-dispose bookkeeping (main, d453165/fee96a7): without this,
|
|
1366
|
+
// ephemeral agents leave checkpoint-tree keys and diagnostics map
|
|
1367
|
+
// entries behind for the life of the store/session.
|
|
1368
|
+
this.evictTurnCheckpoints(agent.name);
|
|
1369
|
+
}
|
|
1334
1370
|
}
|
|
1335
1371
|
|
|
1336
1372
|
/**
|
|
@@ -2619,6 +2655,9 @@ export class AgentFramework {
|
|
|
2619
2655
|
// messages between tool_use and tool_result, causing a 400 error.
|
|
2620
2656
|
if (event.type === 'tool-result') {
|
|
2621
2657
|
const agent = this.agents.get(event.agentName);
|
|
2658
|
+
if (agent) {
|
|
2659
|
+
this.touchEphemeralRun(event.agentName, true);
|
|
2660
|
+
}
|
|
2622
2661
|
if (!agent) {
|
|
2623
2662
|
console.warn(
|
|
2624
2663
|
`[framework] Dropping tool-result for unknown agent '${event.agentName}' (callId=${event.callId}). ` +
|
|
@@ -2693,12 +2732,28 @@ export class AgentFramework {
|
|
|
2693
2732
|
if (shouldEndTurn) {
|
|
2694
2733
|
// endTurn: messages already stored above, cancel stream, reset to idle.
|
|
2695
2734
|
if (currentState.stream) {
|
|
2696
|
-
agent.
|
|
2735
|
+
this.frameworkCancelledStreams.set(`${agent.name}:${agent.streamId}`, 'turn_ended');
|
|
2736
|
+
currentState.stream.cancel();
|
|
2697
2737
|
}
|
|
2698
2738
|
agent.reset();
|
|
2739
|
+
// The turn is over — release the gate here, not just in the abort
|
|
2740
|
+
// handler: the membrane's aborted event is asynchronous and the
|
|
2741
|
+
// agent must be wakeable the moment it goes idle. (Idempotent;
|
|
2742
|
+
// driveStream's finally is the backstop.)
|
|
2743
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
2744
|
+
this.settleAgent(agent.name, { stopReason: 'turn_ended', speech: '' });
|
|
2699
2745
|
this.emitTrace({ type: 'inference:turn_ended', agentName: agent.name });
|
|
2700
2746
|
} else if (overBudget) {
|
|
2701
|
-
// Context budget exceeded: break the stream, let compile() compress
|
|
2747
|
+
// Context budget exceeded: break the stream, let compile() compress.
|
|
2748
|
+
// Mark the cancel as framework-initiated BEFORE cancelling: the
|
|
2749
|
+
// membrane delivers `aborted` before the restart bumps streamId,
|
|
2750
|
+
// and without the marker the abort handler treats the restart as
|
|
2751
|
+
// a terminal failure — rejecting an ephemeral's promise mid-run
|
|
2752
|
+
// and emitting a spurious inference:exhausted (same race shape as
|
|
2753
|
+
// endTurn above).
|
|
2754
|
+
if (currentState.stream) {
|
|
2755
|
+
this.frameworkCancelledStreams.set(`${agent.name}:${agent.streamId}`, 'budget_restart');
|
|
2756
|
+
}
|
|
2702
2757
|
agent.cancelStream();
|
|
2703
2758
|
this.emitTrace({
|
|
2704
2759
|
type: 'inference:stream_restarted',
|
|
@@ -3364,6 +3419,7 @@ export class AgentFramework {
|
|
|
3364
3419
|
this.activeTriggerChannels.delete(agent.name);
|
|
3365
3420
|
}
|
|
3366
3421
|
|
|
3422
|
+
this.touchEphemeralRun(agent.name, true);
|
|
3367
3423
|
this.emitTrace({ type: 'inference:started', agentName: agent.name });
|
|
3368
3424
|
this.eventGate?.onInferenceStarted(agent.name);
|
|
3369
3425
|
this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), startedAt: Date.now() });
|
|
@@ -3428,17 +3484,23 @@ export class AgentFramework {
|
|
|
3428
3484
|
await new Promise((resolve) => setTimeout(resolve, action.delayMs));
|
|
3429
3485
|
await this.startAgentStream(agent, trigger, attempt + 1);
|
|
3430
3486
|
} else {
|
|
3487
|
+
this.settleAgent(agent.name, {
|
|
3488
|
+
stopReason: 'exhausted',
|
|
3489
|
+
speech: '',
|
|
3490
|
+
error: err.message,
|
|
3491
|
+
});
|
|
3431
3492
|
this.emitTrace({
|
|
3432
3493
|
type: 'inference:exhausted',
|
|
3433
3494
|
agentName: agent.name,
|
|
3434
3495
|
error: err.message,
|
|
3435
|
-
// Drives the poison-history breaker
|
|
3436
|
-
//
|
|
3437
|
-
//
|
|
3438
|
-
//
|
|
3439
|
-
//
|
|
3440
|
-
|
|
3441
|
-
|
|
3496
|
+
// Drives the poison-history breaker (`invalid_request`: a 400-class
|
|
3497
|
+
// rejection of the history itself — retrying the same context can
|
|
3498
|
+
// never succeed; auth/abort/context_length are also non-retryable
|
|
3499
|
+
// but must NOT cost history) and the OverBudget drain breaker
|
|
3500
|
+
// (`over_budget`: compile refused to fit the hard budget — this is
|
|
3501
|
+
// the site that sees it, since compile runs before the stream
|
|
3502
|
+
// exists). `retryable` is kept for observability.
|
|
3503
|
+
...this.classifyInferenceError(err),
|
|
3442
3504
|
});
|
|
3443
3505
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
3444
3506
|
if (action.emit) {
|
|
@@ -3469,6 +3531,7 @@ export class AgentFramework {
|
|
|
3469
3531
|
|
|
3470
3532
|
try {
|
|
3471
3533
|
for await (const event of stream) {
|
|
3534
|
+
this.touchEphemeralRun(agent.name, true);
|
|
3472
3535
|
switch (event.type) {
|
|
3473
3536
|
case 'tokens':
|
|
3474
3537
|
this.emitTrace({
|
|
@@ -3494,6 +3557,7 @@ export class AgentFramework {
|
|
|
3494
3557
|
|
|
3495
3558
|
case 'tool-calls': {
|
|
3496
3559
|
hadToolCalls = true;
|
|
3560
|
+
this.recordEphemeralToolCalls(agent.name, event.calls.length);
|
|
3497
3561
|
this.emitTrace({
|
|
3498
3562
|
type: 'inference:tool_calls_yielded',
|
|
3499
3563
|
agentName: agent.name,
|
|
@@ -3593,13 +3657,15 @@ export class AgentFramework {
|
|
|
3593
3657
|
b.type === 'tool_use' || b.type === 'tool_result' ? i : last,
|
|
3594
3658
|
-1
|
|
3595
3659
|
);
|
|
3660
|
+
const terminalContent = lastToolIdx >= 0
|
|
3661
|
+
? response.content.slice(lastToolIdx + 1)
|
|
3662
|
+
: response.content;
|
|
3596
3663
|
if (lastToolIdx >= 0) {
|
|
3597
|
-
|
|
3598
|
-
|
|
3599
|
-
agent.addAssistantResponse(trailingContent);
|
|
3664
|
+
if (terminalContent.length > 0) {
|
|
3665
|
+
agent.addAssistantResponse(terminalContent);
|
|
3600
3666
|
}
|
|
3601
3667
|
} else {
|
|
3602
|
-
agent.addAssistantResponse(
|
|
3668
|
+
agent.addAssistantResponse(terminalContent);
|
|
3603
3669
|
}
|
|
3604
3670
|
|
|
3605
3671
|
// Run afterInference hooks (no-op if no MCPL servers)
|
|
@@ -3658,17 +3724,20 @@ export class AgentFramework {
|
|
|
3658
3724
|
}
|
|
3659
3725
|
: undefined;
|
|
3660
3726
|
|
|
3661
|
-
// Reset agent state
|
|
3662
|
-
// synchronous listeners
|
|
3663
|
-
//
|
|
3664
|
-
//
|
|
3665
|
-
//
|
|
3666
|
-
// trace boundary, failed its idle gate, and the promise hung
|
|
3667
|
-
// forever — every "zombie subagent" in production tracked back
|
|
3668
|
-
// here. Speech dispatch happens after but doesn't depend on the
|
|
3669
|
-
// status field.
|
|
3727
|
+
// Reset agent state before emitting inference:completed. Traces are
|
|
3728
|
+
// observability-only, but external synchronous listeners should
|
|
3729
|
+
// still see the terminal state at the terminal trace boundary.
|
|
3730
|
+
// Speech dispatch happens after but doesn't depend on the status
|
|
3731
|
+
// field.
|
|
3670
3732
|
agent.reset();
|
|
3671
3733
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
3734
|
+
this.settleAgent(agent.name, {
|
|
3735
|
+
stopReason: 'completed',
|
|
3736
|
+
speech: terminalContent
|
|
3737
|
+
.filter((block: ContentBlock): block is ContentBlock & { type: 'text' } => block.type === 'text')
|
|
3738
|
+
.map((block) => block.text)
|
|
3739
|
+
.join('\n'),
|
|
3740
|
+
});
|
|
3672
3741
|
|
|
3673
3742
|
this.emitTrace({
|
|
3674
3743
|
type: 'inference:completed',
|
|
@@ -3907,9 +3976,8 @@ export class AgentFramework {
|
|
|
3907
3976
|
}
|
|
3908
3977
|
}
|
|
3909
3978
|
}
|
|
3910
|
-
// NOTE: agent.reset() + onInferenceEnded() already ran above,
|
|
3911
|
-
// dispatchSpeech
|
|
3912
|
-
// must see status === 'idle'). Locus routing is speech dispatch and
|
|
3979
|
+
// NOTE: agent.reset() + onInferenceEnded() already ran above,
|
|
3980
|
+
// BEFORE dispatchSpeech. Locus routing is speech dispatch and
|
|
3913
3981
|
// doesn't depend on the status field, so it correctly runs after.
|
|
3914
3982
|
|
|
3915
3983
|
break;
|
|
@@ -3945,16 +4013,20 @@ export class AgentFramework {
|
|
|
3945
4013
|
await new Promise((resolve) => setTimeout(resolve, action.delayMs));
|
|
3946
4014
|
await this.startAgentStream(agent, trigger, attempt + 1);
|
|
3947
4015
|
} else {
|
|
4016
|
+
this.settleAgent(agent.name, {
|
|
4017
|
+
stopReason: 'exhausted',
|
|
4018
|
+
speech: '',
|
|
4019
|
+
error: err.message,
|
|
4020
|
+
});
|
|
3948
4021
|
this.emitTrace({
|
|
3949
4022
|
type: 'inference:exhausted',
|
|
3950
4023
|
agentName: agent.name,
|
|
3951
4024
|
error: err.message,
|
|
3952
4025
|
// Drives the poison-history breaker. `retryable` is kept for
|
|
3953
|
-
// observability, but the
|
|
4026
|
+
// observability, but the breakers gate on `errorType` — only an
|
|
3954
4027
|
// `invalid_request` (a 400-class rejection of the history
|
|
3955
4028
|
// itself) means retrying the same context can never succeed.
|
|
3956
|
-
|
|
3957
|
-
errorType: err instanceof MembraneError ? err.type : undefined,
|
|
4029
|
+
...this.classifyInferenceError(err),
|
|
3958
4030
|
});
|
|
3959
4031
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
3960
4032
|
if (action.emit) {
|
|
@@ -3965,12 +4037,28 @@ export class AgentFramework {
|
|
|
3965
4037
|
}
|
|
3966
4038
|
|
|
3967
4039
|
case 'aborted': {
|
|
4040
|
+
// Framework-initiated, non-terminal cancels (endTurn tool result,
|
|
4041
|
+
// context-budget restart) also surface here as `aborted` — but
|
|
4042
|
+
// the turn either already settled (endTurn) or a replacement
|
|
4043
|
+
// stream is queued (budget restart). Neither is a failure: no
|
|
4044
|
+
// reset, no settle, no spurious inference:exhausted (which would
|
|
4045
|
+
// reject an ephemeral's promise mid-run and bump the failure
|
|
4046
|
+
// streak). Gate release + stream teardown happen in `finally`.
|
|
4047
|
+
if (this.frameworkCancelledStreams.delete(`${agent.name}:${myStreamId}`)) {
|
|
4048
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
4049
|
+
return;
|
|
4050
|
+
}
|
|
3968
4051
|
const reason = event.reason ?? 'unknown';
|
|
3969
4052
|
// Only reset if this is still the active stream (a budget restart
|
|
3970
4053
|
// may have already started a new stream, bumping streamId)
|
|
3971
4054
|
if (agent.streamId === myStreamId) {
|
|
3972
4055
|
const durationMs = Date.now() - startTime;
|
|
3973
4056
|
agent.reset();
|
|
4057
|
+
this.settleAgent(agent.name, {
|
|
4058
|
+
stopReason: 'exhausted',
|
|
4059
|
+
speech: '',
|
|
4060
|
+
error: `Stream aborted: ${reason}`,
|
|
4061
|
+
});
|
|
3974
4062
|
this.emitTrace({
|
|
3975
4063
|
type: 'inference:exhausted',
|
|
3976
4064
|
agentName: agent.name,
|
|
@@ -4046,10 +4134,16 @@ export class AgentFramework {
|
|
|
4046
4134
|
error: err.message,
|
|
4047
4135
|
stack: err.stack,
|
|
4048
4136
|
});
|
|
4137
|
+
this.settleAgent(agent.name, {
|
|
4138
|
+
stopReason: 'exhausted',
|
|
4139
|
+
speech: '',
|
|
4140
|
+
error: err.message,
|
|
4141
|
+
});
|
|
4049
4142
|
this.emitTrace({
|
|
4050
4143
|
type: 'inference:exhausted',
|
|
4051
4144
|
agentName: agent.name,
|
|
4052
4145
|
error: err.message,
|
|
4146
|
+
...this.classifyInferenceError(err),
|
|
4053
4147
|
});
|
|
4054
4148
|
// Postmortem 2026-05-28 P2 #7: catch-path failures (stream itself
|
|
4055
4149
|
// threw) were previously only visible via in-memory trace listeners.
|
|
@@ -4079,6 +4173,7 @@ export class AgentFramework {
|
|
|
4079
4173
|
// Stop the typing indicator on every exit path (complete, error,
|
|
4080
4174
|
// exhausted, abort) so it never sticks after the turn ends.
|
|
4081
4175
|
this.channelRegistry?.stopTyping();
|
|
4176
|
+
this.frameworkCancelledStreams.delete(`${agent.name}:${myStreamId}`);
|
|
4082
4177
|
this.activeStreams.delete(agent.name);
|
|
4083
4178
|
this.pendingAssistantBlocks.delete(agent.name);
|
|
4084
4179
|
|
|
@@ -4667,6 +4762,28 @@ export class AgentFramework {
|
|
|
4667
4762
|
return streak;
|
|
4668
4763
|
}
|
|
4669
4764
|
|
|
4765
|
+
/**
|
|
4766
|
+
* Classify a terminal inference error for the `inference:exhausted` trace.
|
|
4767
|
+
* Single-sourced for every emit site: the downstream breakers (poison-history
|
|
4768
|
+
* quarantine, OverBudget drain kick) gate on `errorType`, so classification
|
|
4769
|
+
* drift between sites would silently disable a safety net.
|
|
4770
|
+
*
|
|
4771
|
+
* context-manager's OverBudgetError is recognized by `err.name`: CM does not
|
|
4772
|
+
* export the class from its package root, so a cross-package `instanceof` is
|
|
4773
|
+
* unavailable — but `name` is set in its constructor and survives the package
|
|
4774
|
+
* boundary. Deliberately NOT a message match: the message wording belongs to
|
|
4775
|
+
* CM and can be reworded without warning.
|
|
4776
|
+
*/
|
|
4777
|
+
private classifyInferenceError(err: Error): { retryable?: boolean; errorType?: string } {
|
|
4778
|
+
if (err instanceof MembraneError) {
|
|
4779
|
+
return { retryable: err.retryable, errorType: err.type };
|
|
4780
|
+
}
|
|
4781
|
+
if (err.name === 'OverBudgetError') {
|
|
4782
|
+
return { errorType: 'over_budget' };
|
|
4783
|
+
}
|
|
4784
|
+
return {};
|
|
4785
|
+
}
|
|
4786
|
+
|
|
4670
4787
|
private noteInferenceExhausted(
|
|
4671
4788
|
agentName: string,
|
|
4672
4789
|
reason: string,
|
|
@@ -4707,6 +4824,41 @@ export class AgentFramework {
|
|
|
4707
4824
|
}
|
|
4708
4825
|
}
|
|
4709
4826
|
|
|
4827
|
+
// (2b) OverBudget deadlock breaker. When compile fails with
|
|
4828
|
+
// OverBudgetError, the normal compression drain never runs: it is driven
|
|
4829
|
+
// by successful activity, which the over-budget state prevents - a closed
|
|
4830
|
+
// loop with no internal exit. Field data (2026-07-10, resident agent):
|
|
4831
|
+
// 36 minutes hard-down, zero self-rescue; only an operator raising the
|
|
4832
|
+
// budget externally could break the loop. Break it here instead: kick the
|
|
4833
|
+
// strategy drain directly so folding/merging frees space for the next
|
|
4834
|
+
// compile. Bounded ticks, best-effort, never throws, one kick per agent
|
|
4835
|
+
// at a time (see overBudgetDrainInFlight).
|
|
4836
|
+
//
|
|
4837
|
+
// Gates on the classified errorType (see classifyInferenceError); the
|
|
4838
|
+
// message match is only a fallback for paths that lost the Error object
|
|
4839
|
+
// (e.g. a reason string that crossed a serialization boundary). It matches
|
|
4840
|
+
// CM's current OverBudgetError wording and MAY rot if CM rewords it — the
|
|
4841
|
+
// errorType gate is the one that's load-bearing.
|
|
4842
|
+
const overBudget = errorType === 'over_budget' || /exceed hard budget/i.test(reason);
|
|
4843
|
+
if (agent && overBudget && !this.overBudgetDrainInFlight.has(agentName)) {
|
|
4844
|
+
this.overBudgetDrainInFlight.add(agentName);
|
|
4845
|
+
void (async () => {
|
|
4846
|
+
let ticks = 0;
|
|
4847
|
+
try {
|
|
4848
|
+
const cm = agent.getContextManager();
|
|
4849
|
+
while (ticks < 8) {
|
|
4850
|
+
await cm.tick();
|
|
4851
|
+
ticks++;
|
|
4852
|
+
}
|
|
4853
|
+
console.error(`[inference-failed] drain kicked for ${agentName} (OverBudget breaker, ${ticks} ticks)`);
|
|
4854
|
+
} catch (err) {
|
|
4855
|
+
console.error(`[inference-failed] drain kick failed for ${agentName} after ${ticks} ticks:`, err);
|
|
4856
|
+
} finally {
|
|
4857
|
+
this.overBudgetDrainInFlight.delete(agentName);
|
|
4858
|
+
}
|
|
4859
|
+
})();
|
|
4860
|
+
}
|
|
4861
|
+
|
|
4710
4862
|
// (3) Hard-down escalation on repeated identical failure.
|
|
4711
4863
|
if (streak >= this.inferenceFailureEscalationThreshold) {
|
|
4712
4864
|
console.error(
|
package/src/types/framework.ts
CHANGED
|
@@ -181,6 +181,16 @@ export type ErrorAction =
|
|
|
181
181
|
| { retry: true; delayMs: number }
|
|
182
182
|
| { retry: false; emit?: ProcessEvent };
|
|
183
183
|
|
|
184
|
+
/**
|
|
185
|
+
* Terminal result for a framework-driven ephemeral agent run.
|
|
186
|
+
*/
|
|
187
|
+
export interface AgentSettleResult {
|
|
188
|
+
stopReason: 'completed' | 'turn_ended' | 'exhausted';
|
|
189
|
+
speech: string;
|
|
190
|
+
toolCallsCount: number;
|
|
191
|
+
error?: string;
|
|
192
|
+
}
|
|
193
|
+
|
|
184
194
|
/**
|
|
185
195
|
* Framework state exposed to policies.
|
|
186
196
|
*/
|