@animalabs/agent-framework 0.6.3 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/src/agent.d.ts +3 -1
- package/dist/src/agent.d.ts.map +1 -1
- package/dist/src/agent.js +27 -2
- package/dist/src/agent.js.map +1 -1
- package/dist/src/framework.d.ts +46 -1
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +280 -146
- package/dist/src/framework.js.map +1 -1
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/modules/discord/index.d.ts +1 -0
- package/dist/src/modules/discord/index.d.ts.map +1 -1
- package/dist/src/modules/discord/index.js +4 -1
- package/dist/src/modules/discord/index.js.map +1 -1
- package/dist/src/modules/discord/types.d.ts +2 -0
- package/dist/src/modules/discord/types.d.ts.map +1 -1
- package/dist/src/modules/health/index.d.ts +2 -0
- package/dist/src/modules/health/index.d.ts.map +1 -1
- package/dist/src/modules/health/index.js +15 -4
- package/dist/src/modules/health/index.js.map +1 -1
- package/dist/src/timezone.d.ts +14 -0
- package/dist/src/timezone.d.ts.map +1 -0
- package/dist/src/timezone.js +74 -0
- package/dist/src/timezone.js.map +1 -0
- package/dist/src/types/agent.d.ts +5 -2
- package/dist/src/types/agent.d.ts.map +1 -1
- package/dist/src/types/framework.d.ts +15 -0
- package/dist/src/types/framework.d.ts.map +1 -1
- package/dist/test/activation-request-sanitize.test.d.ts +2 -0
- package/dist/test/activation-request-sanitize.test.d.ts.map +1 -0
- package/dist/test/activation-request-sanitize.test.js +89 -0
- package/dist/test/activation-request-sanitize.test.js.map +1 -0
- package/dist/test/agent-cache-ttl.test.js +18 -2
- package/dist/test/agent-cache-ttl.test.js.map +1 -1
- package/dist/test/ephemeral-settle.test.d.ts +2 -0
- package/dist/test/ephemeral-settle.test.d.ts.map +1 -0
- package/dist/test/ephemeral-settle.test.js +351 -0
- package/dist/test/ephemeral-settle.test.js.map +1 -0
- package/dist/test/health-snapshot.test.d.ts +2 -0
- package/dist/test/health-snapshot.test.d.ts.map +1 -0
- package/dist/test/health-snapshot.test.js +74 -0
- package/dist/test/health-snapshot.test.js.map +1 -0
- package/dist/test/maintenance-timer.test.js +49 -0
- package/dist/test/maintenance-timer.test.js.map +1 -1
- package/dist/test/overbudget-drain-breaker.test.d.ts +2 -0
- package/dist/test/overbudget-drain-breaker.test.d.ts.map +1 -0
- package/dist/test/overbudget-drain-breaker.test.js +157 -0
- package/dist/test/overbudget-drain-breaker.test.js.map +1 -0
- package/dist/test/timezone.test.d.ts +2 -0
- package/dist/test/timezone.test.d.ts.map +1 -0
- package/dist/test/timezone.test.js +18 -0
- package/dist/test/timezone.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/agent.ts +30 -3
- package/src/framework.ts +313 -146
- package/src/index.ts +1 -0
- package/src/modules/discord/index.ts +4 -1
- package/src/modules/discord/types.ts +3 -0
- package/src/modules/health/index.ts +20 -4
- package/src/timezone.ts +90 -0
- package/src/types/agent.ts +6 -2
- package/src/types/framework.ts +17 -0
package/src/framework.ts
CHANGED
|
@@ -32,6 +32,7 @@ import type {
|
|
|
32
32
|
ProcessEvent,
|
|
33
33
|
EventResponse,
|
|
34
34
|
ModuleProcessResponse,
|
|
35
|
+
AgentSettleResult,
|
|
35
36
|
ToolCall,
|
|
36
37
|
ToolCallEvent,
|
|
37
38
|
ToolResult,
|
|
@@ -79,6 +80,7 @@ import type {
|
|
|
79
80
|
ChannelsIncomingParams,
|
|
80
81
|
} from './mcpl/types.js';
|
|
81
82
|
import type { ContextInjection } from '@animalabs/context-manager';
|
|
83
|
+
import { formatZonedDateTime, resolveTimeZone } from './timezone.js';
|
|
82
84
|
|
|
83
85
|
const FRAMEWORK_STATE_ID = 'framework/state';
|
|
84
86
|
const CONVERSATION_ROUTER_STATE_ID = 'framework/conversation-router';
|
|
@@ -228,6 +230,23 @@ interface RewindRecord {
|
|
|
228
230
|
discordRef?: { channelId: string; messageId: string };
|
|
229
231
|
}
|
|
230
232
|
|
|
233
|
+
interface Deferred<T> {
|
|
234
|
+
promise: Promise<T>;
|
|
235
|
+
resolve: (value: T) => void;
|
|
236
|
+
reject: (reason?: unknown) => void;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** Per-run state for a framework-driven ephemeral agent: the settle signal
|
|
240
|
+
* the caller awaits plus the liveness/progress bookkeeping the watchdogs and
|
|
241
|
+
* the stream driver share. One map entry per run — created and torn down in
|
|
242
|
+
* runEphemeralToCompletion — so the pieces cannot desync. */
|
|
243
|
+
interface EphemeralRun {
|
|
244
|
+
settle: Deferred<AgentSettleResult>;
|
|
245
|
+
inferenceStarted: boolean;
|
|
246
|
+
lastActivity: number;
|
|
247
|
+
toolCallsCount: number;
|
|
248
|
+
}
|
|
249
|
+
|
|
231
250
|
/** Cap an API error message for inline use in a rewind marker: keep enough to
|
|
232
251
|
* identify the failure class without pasting a wall of provider JSON into the
|
|
233
252
|
* agent's context. */
|
|
@@ -274,6 +293,17 @@ export class AgentFramework {
|
|
|
274
293
|
private processLoggingBroadcast: boolean;
|
|
275
294
|
private activeStreams: Map<string, Promise<void>> = new Map();
|
|
276
295
|
private pendingAssistantBlocks: Map<string, ContentBlock[]> = new Map();
|
|
296
|
+
/** Streams the FRAMEWORK cancelled for non-terminal reasons, keyed
|
|
297
|
+
* `${agentName}:${streamId}`: an endTurn tool result or a context-budget
|
|
298
|
+
* restart. The membrane still delivers an `aborted` event for these, and
|
|
299
|
+
* without the marker the abort handler would treat it as a terminal
|
|
300
|
+
* failure — settling ephemerals with a rejection and emitting a spurious
|
|
301
|
+
* `inference:exhausted` (which also pollutes the failure streak). Kept
|
|
302
|
+
* separate from ephemeralRuns deliberately: endTurn/budget cancels happen
|
|
303
|
+
* for resident agents too, and the key is per-stream, not per-agent. */
|
|
304
|
+
private frameworkCancelledStreams: Map<string, 'turn_ended' | 'budget_restart'> = new Map();
|
|
305
|
+
/** Active runEphemeralToCompletion runs, keyed by agent name. */
|
|
306
|
+
private ephemeralRuns: Map<string, EphemeralRun> = new Map();
|
|
277
307
|
/** Per-agent count of consecutive exhausted inferences (reset on any success).
|
|
278
308
|
* Drives hard-down escalation — see noteInferenceExhausted. */
|
|
279
309
|
private consecutiveInferenceFailures: Map<string, number> = new Map();
|
|
@@ -302,6 +332,13 @@ export class AgentFramework {
|
|
|
302
332
|
* Bounds the automatic quarantine loop the same way refusalRewinds bounds
|
|
303
333
|
* the refusal loop, so the breaker can never shed the whole history. */
|
|
304
334
|
private exhaustionRewinds: Map<string, number> = new Map();
|
|
335
|
+
/** Agents with an OverBudget drain kick currently in flight. The breaker in
|
|
336
|
+
* noteInferenceExhausted fires on EVERY matching failure and the scenario it
|
|
337
|
+
* exists for is "every activation fails, repeatedly" — without this guard,
|
|
338
|
+
* overlapping kicks race the strategy's own pendingCompression gate and the
|
|
339
|
+
* tick counts in the success log stop meaning anything. One kick per agent
|
|
340
|
+
* at a time; cleared when the kick settles (success or failure). */
|
|
341
|
+
private overBudgetDrainInFlight: Set<string> = new Set();
|
|
305
342
|
/** Per-agent current rewind episode: the single consolidated marker's id and
|
|
306
343
|
* how many turns have been shed so far. One marker per episode, updated in
|
|
307
344
|
* place (see updateRewindMarker); cleared when the episode ends. */
|
|
@@ -374,6 +411,8 @@ export class AgentFramework {
|
|
|
374
411
|
|
|
375
412
|
// Session-level token usage tracking (always-on)
|
|
376
413
|
private usageTracker: UsageTracker;
|
|
414
|
+
/** Presentation-only wall-clock zone; persistence remains UTC/epoch. */
|
|
415
|
+
private readonly timeZone: string;
|
|
377
416
|
|
|
378
417
|
private constructor(
|
|
379
418
|
store: JsStore,
|
|
@@ -384,7 +423,8 @@ export class AgentFramework {
|
|
|
384
423
|
syncIntervalMs: number,
|
|
385
424
|
maintenanceIntervalMs: number,
|
|
386
425
|
processLoggingPersist: boolean,
|
|
387
|
-
processLoggingBroadcast: boolean
|
|
426
|
+
processLoggingBroadcast: boolean,
|
|
427
|
+
timeZone: string,
|
|
388
428
|
) {
|
|
389
429
|
this.store = store;
|
|
390
430
|
this.ownsStore = ownsStore;
|
|
@@ -395,6 +435,7 @@ export class AgentFramework {
|
|
|
395
435
|
this.maintenanceIntervalMs = maintenanceIntervalMs;
|
|
396
436
|
this.processLoggingPersist = processLoggingPersist;
|
|
397
437
|
this.processLoggingBroadcast = processLoggingBroadcast;
|
|
438
|
+
this.timeZone = timeZone;
|
|
398
439
|
this.queue = new ProcessQueueImpl();
|
|
399
440
|
this.usageTracker = new UsageTracker({
|
|
400
441
|
emitTrace: (e: UsageUpdatedEvent) => this.emitTrace({ ...e }),
|
|
@@ -485,7 +526,8 @@ export class AgentFramework {
|
|
|
485
526
|
config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS,
|
|
486
527
|
config.maintenanceIntervalMs ?? DEFAULT_MAINTENANCE_INTERVAL_MS,
|
|
487
528
|
processLoggingPersist,
|
|
488
|
-
processLoggingBroadcast
|
|
529
|
+
processLoggingBroadcast,
|
|
530
|
+
resolveTimeZone(config.timeZone),
|
|
489
531
|
);
|
|
490
532
|
|
|
491
533
|
// Restore persisted usage data (if any) from prior session
|
|
@@ -726,9 +768,16 @@ export class AgentFramework {
|
|
|
726
768
|
private startQueuedMaintenance(): void {
|
|
727
769
|
if (this.maintenancePass || !this.running) return;
|
|
728
770
|
const pass = this.runQueuedMaintenance().catch((error) => {
|
|
771
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
729
772
|
console.error(
|
|
730
773
|
'[context-maintenance] pass failed:',
|
|
731
|
-
|
|
774
|
+
reason,
|
|
775
|
+
);
|
|
776
|
+
this.opsAlert(
|
|
777
|
+
'context-maintenance-failed',
|
|
778
|
+
'framework',
|
|
779
|
+
reason,
|
|
780
|
+
{ data: { scope: 'pass' } },
|
|
732
781
|
);
|
|
733
782
|
});
|
|
734
783
|
this.maintenancePass = pass;
|
|
@@ -783,6 +832,18 @@ export class AgentFramework {
|
|
|
783
832
|
} catch (error) {
|
|
784
833
|
record.error = error instanceof Error ? error.message : String(error);
|
|
785
834
|
console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
|
|
835
|
+
this.opsAlert(
|
|
836
|
+
'context-maintenance-failed',
|
|
837
|
+
agent.name,
|
|
838
|
+
record.error,
|
|
839
|
+
{
|
|
840
|
+
data: {
|
|
841
|
+
scope: 'agent',
|
|
842
|
+
...(record.pendingBefore ? { pending: record.pendingBefore } : {}),
|
|
843
|
+
...(record.progressBefore ? { progress: record.progressBefore } : {}),
|
|
844
|
+
},
|
|
845
|
+
},
|
|
846
|
+
);
|
|
786
847
|
} finally {
|
|
787
848
|
const pending = cm.getPendingWork()?.description;
|
|
788
849
|
record.finishedAt = Date.now();
|
|
@@ -1171,6 +1232,54 @@ export class AgentFramework {
|
|
|
1171
1232
|
return { agent, contextManager, cleanup };
|
|
1172
1233
|
}
|
|
1173
1234
|
|
|
1235
|
+
private createDeferred<T>(): Deferred<T> {
|
|
1236
|
+
let resolve!: (value: T) => void;
|
|
1237
|
+
let reject!: (reason?: unknown) => void;
|
|
1238
|
+
const promise = new Promise<T>((res, rej) => {
|
|
1239
|
+
resolve = res;
|
|
1240
|
+
reject = rej;
|
|
1241
|
+
});
|
|
1242
|
+
return { promise, resolve, reject };
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
/** Liveness ping for an active ephemeral run (no-op otherwise). */
|
|
1246
|
+
private touchEphemeralRun(agentName: string, inferenceStarted = false): void {
|
|
1247
|
+
const run = this.ephemeralRuns.get(agentName);
|
|
1248
|
+
if (!run) return;
|
|
1249
|
+
if (inferenceStarted) {
|
|
1250
|
+
run.inferenceStarted = true;
|
|
1251
|
+
}
|
|
1252
|
+
run.lastActivity = Date.now();
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
private recordEphemeralToolCalls(agentName: string, count: number): void {
|
|
1256
|
+
const run = this.ephemeralRuns.get(agentName);
|
|
1257
|
+
if (!run) return;
|
|
1258
|
+
run.toolCallsCount += count;
|
|
1259
|
+
run.lastActivity = Date.now();
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
/** Settle an active ephemeral run (no-op for resident agents). The tool-call
|
|
1263
|
+
* count comes from the run's own counter — the single source of truth,
|
|
1264
|
+
* which unlike a per-stream local survives budget restarts. */
|
|
1265
|
+
private settleAgent(
|
|
1266
|
+
agentName: string,
|
|
1267
|
+
outcome: { stopReason: AgentSettleResult['stopReason']; speech: string; error?: string },
|
|
1268
|
+
): void {
|
|
1269
|
+
const run = this.ephemeralRuns.get(agentName);
|
|
1270
|
+
if (!run) return;
|
|
1271
|
+
run.lastActivity = Date.now();
|
|
1272
|
+
if (outcome.stopReason === 'exhausted') {
|
|
1273
|
+
run.settle.reject(new Error(outcome.error ?? 'Unknown error'));
|
|
1274
|
+
return;
|
|
1275
|
+
}
|
|
1276
|
+
run.settle.resolve({
|
|
1277
|
+
stopReason: outcome.stopReason,
|
|
1278
|
+
speech: outcome.speech,
|
|
1279
|
+
toolCallsCount: run.toolCallsCount,
|
|
1280
|
+
});
|
|
1281
|
+
}
|
|
1282
|
+
|
|
1174
1283
|
/**
|
|
1175
1284
|
* Run an ephemeral agent to completion through the framework's event loop.
|
|
1176
1285
|
*
|
|
@@ -1184,42 +1293,31 @@ export class AgentFramework {
|
|
|
1184
1293
|
async runEphemeralToCompletion(
|
|
1185
1294
|
agent: Agent,
|
|
1186
1295
|
contextManager: ContextManager,
|
|
1296
|
+
watchdogs?: { startupTimeoutMs?: number; idleTimeoutMs?: number; idlePollMs?: number },
|
|
1187
1297
|
): Promise<{ speech: string; toolCallsCount: number }> {
|
|
1188
1298
|
// Register temporarily so the event loop can drive it
|
|
1189
1299
|
this.agents.set(agent.name, agent);
|
|
1300
|
+
const run: EphemeralRun = {
|
|
1301
|
+
settle: this.createDeferred<AgentSettleResult>(),
|
|
1302
|
+
inferenceStarted: false,
|
|
1303
|
+
lastActivity: Date.now(),
|
|
1304
|
+
toolCallsCount: 0,
|
|
1305
|
+
};
|
|
1306
|
+
this.ephemeralRuns.set(agent.name, run);
|
|
1190
1307
|
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
const STARTUP_TIMEOUT_MS = 30_000;
|
|
1199
|
-
// After inference has started, give it 15 minutes of activity-bounded
|
|
1200
|
-
// life. Each addressed trace event refreshes the deadline; only
|
|
1201
|
-
// sustained silence trips it. Subagents that legitimately stream for
|
|
1202
|
-
// 10+ minutes (e.g. long fork investigations) get the extra slack but
|
|
1203
|
-
// stalled streams that emit `inference:completed` and then go quiet —
|
|
1204
|
-
// the exact zombie shape we saw in production — are caught here.
|
|
1205
|
-
const COMPLETION_IDLE_TIMEOUT_MS = 15 * 60_000;
|
|
1206
|
-
|
|
1207
|
-
const cleanup = () => {
|
|
1208
|
-
if (settled) return;
|
|
1209
|
-
settled = true;
|
|
1210
|
-
clearTimeout(startupWatchdog);
|
|
1211
|
-
clearInterval(completionWatchdog);
|
|
1212
|
-
this.offTrace(traceListener);
|
|
1213
|
-
this.agents.delete(agent.name);
|
|
1214
|
-
this.evictTurnCheckpoints(agent.name);
|
|
1215
|
-
};
|
|
1308
|
+
const STARTUP_TIMEOUT_MS = watchdogs?.startupTimeoutMs ?? 30_000;
|
|
1309
|
+
// After inference has started, give it 15 minutes of activity-bounded
|
|
1310
|
+
// life. The stream driver refreshes the deadline as it makes progress;
|
|
1311
|
+
// sustained silence trips it.
|
|
1312
|
+
const COMPLETION_IDLE_TIMEOUT_MS = watchdogs?.idleTimeoutMs ?? 15 * 60_000;
|
|
1313
|
+
const IDLE_POLL_MS = watchdogs?.idlePollMs ?? 30_000;
|
|
1216
1314
|
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1315
|
+
let startupWatchdog: ReturnType<typeof setTimeout> | null = null;
|
|
1316
|
+
let completionWatchdog: ReturnType<typeof setInterval> | null = null;
|
|
1317
|
+
|
|
1318
|
+
const startupTimeout = new Promise<never>((_, reject) => {
|
|
1319
|
+
startupWatchdog = setTimeout(() => {
|
|
1320
|
+
if (!run.inferenceStarted) {
|
|
1223
1321
|
reject(new Error(
|
|
1224
1322
|
`Ephemeral agent "${agent.name}" failed to start inference within ` +
|
|
1225
1323
|
`${STARTUP_TIMEOUT_MS}ms — zombie detected. The event loop may have ` +
|
|
@@ -1227,104 +1325,48 @@ export class AgentFramework {
|
|
|
1227
1325
|
));
|
|
1228
1326
|
}
|
|
1229
1327
|
}, STARTUP_TIMEOUT_MS);
|
|
1328
|
+
});
|
|
1230
1329
|
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
// and keeps its concurrency slot pinned.
|
|
1236
|
-
const completionWatchdog = setInterval(() => {
|
|
1237
|
-
if (settled || !inferenceStarted) return;
|
|
1238
|
-
const idle = Date.now() - lastActivity;
|
|
1330
|
+
const completionIdleTimeout = new Promise<never>((_, reject) => {
|
|
1331
|
+
completionWatchdog = setInterval(() => {
|
|
1332
|
+
if (!run.inferenceStarted) return;
|
|
1333
|
+
const idle = Date.now() - run.lastActivity;
|
|
1239
1334
|
if (idle > COMPLETION_IDLE_TIMEOUT_MS) {
|
|
1240
|
-
cleanup();
|
|
1241
1335
|
reject(new Error(
|
|
1242
|
-
`Ephemeral agent "${agent.name}" stalled: no
|
|
1336
|
+
`Ephemeral agent "${agent.name}" stalled: no stream-driver activity for ` +
|
|
1243
1337
|
`${Math.round(idle / 1000)}s after inference started (threshold ` +
|
|
1244
1338
|
`${Math.round(COMPLETION_IDLE_TIMEOUT_MS / 1000)}s). Stream likely ` +
|
|
1245
1339
|
`dropped or terminal event was lost.`
|
|
1246
1340
|
));
|
|
1247
1341
|
}
|
|
1248
|
-
},
|
|
1249
|
-
|
|
1250
|
-
const traceListener = (event: TraceEvent) => {
|
|
1251
|
-
if (settled) return;
|
|
1252
|
-
// Only track events for our ephemeral agent
|
|
1253
|
-
const agentName = 'agentName' in event ? (event as { agentName: string }).agentName : null;
|
|
1254
|
-
if (agentName !== agent.name) return;
|
|
1255
|
-
|
|
1256
|
-
// Any addressed event counts as liveness for the completion watchdog.
|
|
1257
|
-
lastActivity = Date.now();
|
|
1258
|
-
|
|
1259
|
-
switch (event.type) {
|
|
1260
|
-
case 'inference:started':
|
|
1261
|
-
inferenceStarted = true;
|
|
1262
|
-
break;
|
|
1263
|
-
case 'inference:tokens': {
|
|
1264
|
-
inferenceStarted = true;
|
|
1265
|
-
const content = (event as { content?: string }).content;
|
|
1266
|
-
if (content) speech += content;
|
|
1267
|
-
break;
|
|
1268
|
-
}
|
|
1269
|
-
case 'inference:tool_calls_yielded': {
|
|
1270
|
-
const calls = (event as { calls?: Array<unknown> }).calls;
|
|
1271
|
-
if (calls) toolCallsCount += calls.length;
|
|
1272
|
-
// Reset speech buffer — tool calls break the text
|
|
1273
|
-
speech = '';
|
|
1274
|
-
break;
|
|
1275
|
-
}
|
|
1276
|
-
case 'inference:stream_resumed':
|
|
1277
|
-
speech = '';
|
|
1278
|
-
break;
|
|
1279
|
-
case 'inference:completed': {
|
|
1280
|
-
// `inference:completed` is terminal — driveStream only emits it
|
|
1281
|
-
// from `case 'complete'`, which fires after the full tool
|
|
1282
|
-
// round-trip has finished and the model has stopped. Intermediate
|
|
1283
|
-
// tool-cycle transitions use `inference:stream_resumed`, not
|
|
1284
|
-
// `inference:completed`. So we resolve unconditionally here.
|
|
1285
|
-
//
|
|
1286
|
-
// Previously this branch gated on `agent.state.status === 'idle'`,
|
|
1287
|
-
// which was correct in intent but order-fragile: the framework
|
|
1288
|
-
// emitted `inference:completed` BEFORE calling `agent.reset()`,
|
|
1289
|
-
// so synchronous listeners observed status === 'streaming' at
|
|
1290
|
-
// the trace boundary. The gate failed silently and the promise
|
|
1291
|
-
// hung forever — every production zombie subagent originated
|
|
1292
|
-
// here. The reset+emit order has been fixed in driveStream's
|
|
1293
|
-
// `case 'complete'` handler; the gate is dropped here as
|
|
1294
|
-
// belt-and-suspenders.
|
|
1295
|
-
cleanup();
|
|
1296
|
-
resolve({ speech, toolCallsCount });
|
|
1297
|
-
break;
|
|
1298
|
-
}
|
|
1299
|
-
case 'inference:turn_ended': {
|
|
1300
|
-
// endTurn from a tool result — agent is done
|
|
1301
|
-
cleanup();
|
|
1302
|
-
resolve({ speech, toolCallsCount });
|
|
1303
|
-
break;
|
|
1304
|
-
}
|
|
1305
|
-
case 'inference:exhausted': {
|
|
1306
|
-
// Retries exhausted — reject only after the error policy gives up.
|
|
1307
|
-
// Earlier inference:failed events are left alone so the framework
|
|
1308
|
-
// retry path in driveStream/startAgentStream can restart the stream
|
|
1309
|
-
// while this listener stays alive.
|
|
1310
|
-
const error = (event as { error?: string }).error ?? 'Unknown error';
|
|
1311
|
-
cleanup();
|
|
1312
|
-
reject(new Error(error));
|
|
1313
|
-
break;
|
|
1314
|
-
}
|
|
1315
|
-
}
|
|
1316
|
-
};
|
|
1317
|
-
|
|
1318
|
-
this.onTrace(traceListener);
|
|
1342
|
+
}, IDLE_POLL_MS);
|
|
1343
|
+
});
|
|
1319
1344
|
|
|
1320
|
-
|
|
1345
|
+
try {
|
|
1346
|
+
// Trigger inference after the settle promise is registered.
|
|
1321
1347
|
this.pendingRequests.push({
|
|
1322
1348
|
agentName: agent.name,
|
|
1323
1349
|
reason: 'ephemeral',
|
|
1324
1350
|
source: 'subagent',
|
|
1325
1351
|
timestamp: Date.now(),
|
|
1326
1352
|
});
|
|
1327
|
-
|
|
1353
|
+
|
|
1354
|
+
const result = await Promise.race([
|
|
1355
|
+
run.settle.promise,
|
|
1356
|
+
startupTimeout,
|
|
1357
|
+
completionIdleTimeout,
|
|
1358
|
+
]);
|
|
1359
|
+
return { speech: result.speech, toolCallsCount: result.toolCallsCount };
|
|
1360
|
+
} finally {
|
|
1361
|
+
if (startupWatchdog) clearTimeout(startupWatchdog);
|
|
1362
|
+
if (completionWatchdog) clearInterval(completionWatchdog);
|
|
1363
|
+
this.ephemeralRuns.delete(agent.name);
|
|
1364
|
+
this.agents.delete(agent.name);
|
|
1365
|
+
// Spawn-and-dispose bookkeeping (main, d453165/fee96a7): without this,
|
|
1366
|
+
// ephemeral agents leave checkpoint-tree keys and diagnostics map
|
|
1367
|
+
// entries behind for the life of the store/session.
|
|
1368
|
+
this.evictTurnCheckpoints(agent.name);
|
|
1369
|
+
}
|
|
1328
1370
|
}
|
|
1329
1371
|
|
|
1330
1372
|
/**
|
|
@@ -2613,6 +2655,9 @@ export class AgentFramework {
|
|
|
2613
2655
|
// messages between tool_use and tool_result, causing a 400 error.
|
|
2614
2656
|
if (event.type === 'tool-result') {
|
|
2615
2657
|
const agent = this.agents.get(event.agentName);
|
|
2658
|
+
if (agent) {
|
|
2659
|
+
this.touchEphemeralRun(event.agentName, true);
|
|
2660
|
+
}
|
|
2616
2661
|
if (!agent) {
|
|
2617
2662
|
console.warn(
|
|
2618
2663
|
`[framework] Dropping tool-result for unknown agent '${event.agentName}' (callId=${event.callId}). ` +
|
|
@@ -2687,12 +2732,28 @@ export class AgentFramework {
|
|
|
2687
2732
|
if (shouldEndTurn) {
|
|
2688
2733
|
// endTurn: messages already stored above, cancel stream, reset to idle.
|
|
2689
2734
|
if (currentState.stream) {
|
|
2690
|
-
agent.
|
|
2735
|
+
this.frameworkCancelledStreams.set(`${agent.name}:${agent.streamId}`, 'turn_ended');
|
|
2736
|
+
currentState.stream.cancel();
|
|
2691
2737
|
}
|
|
2692
2738
|
agent.reset();
|
|
2739
|
+
// The turn is over — release the gate here, not just in the abort
|
|
2740
|
+
// handler: the membrane's aborted event is asynchronous and the
|
|
2741
|
+
// agent must be wakeable the moment it goes idle. (Idempotent;
|
|
2742
|
+
// driveStream's finally is the backstop.)
|
|
2743
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
2744
|
+
this.settleAgent(agent.name, { stopReason: 'turn_ended', speech: '' });
|
|
2693
2745
|
this.emitTrace({ type: 'inference:turn_ended', agentName: agent.name });
|
|
2694
2746
|
} else if (overBudget) {
|
|
2695
|
-
// Context budget exceeded: break the stream, let compile() compress
|
|
2747
|
+
// Context budget exceeded: break the stream, let compile() compress.
|
|
2748
|
+
// Mark the cancel as framework-initiated BEFORE cancelling: the
|
|
2749
|
+
// membrane delivers `aborted` before the restart bumps streamId,
|
|
2750
|
+
// and without the marker the abort handler treats the restart as
|
|
2751
|
+
// a terminal failure — rejecting an ephemeral's promise mid-run
|
|
2752
|
+
// and emitting a spurious inference:exhausted (same race shape as
|
|
2753
|
+
// endTurn above).
|
|
2754
|
+
if (currentState.stream) {
|
|
2755
|
+
this.frameworkCancelledStreams.set(`${agent.name}:${agent.streamId}`, 'budget_restart');
|
|
2756
|
+
}
|
|
2696
2757
|
agent.cancelStream();
|
|
2697
2758
|
this.emitTrace({
|
|
2698
2759
|
type: 'inference:stream_restarted',
|
|
@@ -3358,6 +3419,7 @@ export class AgentFramework {
|
|
|
3358
3419
|
this.activeTriggerChannels.delete(agent.name);
|
|
3359
3420
|
}
|
|
3360
3421
|
|
|
3422
|
+
this.touchEphemeralRun(agent.name, true);
|
|
3361
3423
|
this.emitTrace({ type: 'inference:started', agentName: agent.name });
|
|
3362
3424
|
this.eventGate?.onInferenceStarted(agent.name);
|
|
3363
3425
|
this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), startedAt: Date.now() });
|
|
@@ -3422,17 +3484,23 @@ export class AgentFramework {
|
|
|
3422
3484
|
await new Promise((resolve) => setTimeout(resolve, action.delayMs));
|
|
3423
3485
|
await this.startAgentStream(agent, trigger, attempt + 1);
|
|
3424
3486
|
} else {
|
|
3487
|
+
this.settleAgent(agent.name, {
|
|
3488
|
+
stopReason: 'exhausted',
|
|
3489
|
+
speech: '',
|
|
3490
|
+
error: err.message,
|
|
3491
|
+
});
|
|
3425
3492
|
this.emitTrace({
|
|
3426
3493
|
type: 'inference:exhausted',
|
|
3427
3494
|
agentName: agent.name,
|
|
3428
3495
|
error: err.message,
|
|
3429
|
-
// Drives the poison-history breaker
|
|
3430
|
-
//
|
|
3431
|
-
//
|
|
3432
|
-
//
|
|
3433
|
-
//
|
|
3434
|
-
|
|
3435
|
-
|
|
3496
|
+
// Drives the poison-history breaker (`invalid_request`: a 400-class
|
|
3497
|
+
// rejection of the history itself — retrying the same context can
|
|
3498
|
+
// never succeed; auth/abort/context_length are also non-retryable
|
|
3499
|
+
// but must NOT cost history) and the OverBudget drain breaker
|
|
3500
|
+
// (`over_budget`: compile refused to fit the hard budget — this is
|
|
3501
|
+
// the site that sees it, since compile runs before the stream
|
|
3502
|
+
// exists). `retryable` is kept for observability.
|
|
3503
|
+
...this.classifyInferenceError(err),
|
|
3436
3504
|
});
|
|
3437
3505
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
3438
3506
|
if (action.emit) {
|
|
@@ -3463,6 +3531,7 @@ export class AgentFramework {
|
|
|
3463
3531
|
|
|
3464
3532
|
try {
|
|
3465
3533
|
for await (const event of stream) {
|
|
3534
|
+
this.touchEphemeralRun(agent.name, true);
|
|
3466
3535
|
switch (event.type) {
|
|
3467
3536
|
case 'tokens':
|
|
3468
3537
|
this.emitTrace({
|
|
@@ -3488,6 +3557,7 @@ export class AgentFramework {
|
|
|
3488
3557
|
|
|
3489
3558
|
case 'tool-calls': {
|
|
3490
3559
|
hadToolCalls = true;
|
|
3560
|
+
this.recordEphemeralToolCalls(agent.name, event.calls.length);
|
|
3491
3561
|
this.emitTrace({
|
|
3492
3562
|
type: 'inference:tool_calls_yielded',
|
|
3493
3563
|
agentName: agent.name,
|
|
@@ -3587,13 +3657,15 @@ export class AgentFramework {
|
|
|
3587
3657
|
b.type === 'tool_use' || b.type === 'tool_result' ? i : last,
|
|
3588
3658
|
-1
|
|
3589
3659
|
);
|
|
3660
|
+
const terminalContent = lastToolIdx >= 0
|
|
3661
|
+
? response.content.slice(lastToolIdx + 1)
|
|
3662
|
+
: response.content;
|
|
3590
3663
|
if (lastToolIdx >= 0) {
|
|
3591
|
-
|
|
3592
|
-
|
|
3593
|
-
agent.addAssistantResponse(trailingContent);
|
|
3664
|
+
if (terminalContent.length > 0) {
|
|
3665
|
+
agent.addAssistantResponse(terminalContent);
|
|
3594
3666
|
}
|
|
3595
3667
|
} else {
|
|
3596
|
-
agent.addAssistantResponse(
|
|
3668
|
+
agent.addAssistantResponse(terminalContent);
|
|
3597
3669
|
}
|
|
3598
3670
|
|
|
3599
3671
|
// Run afterInference hooks (no-op if no MCPL servers)
|
|
@@ -3652,17 +3724,20 @@ export class AgentFramework {
|
|
|
3652
3724
|
}
|
|
3653
3725
|
: undefined;
|
|
3654
3726
|
|
|
3655
|
-
// Reset agent state
|
|
3656
|
-
// synchronous listeners
|
|
3657
|
-
//
|
|
3658
|
-
//
|
|
3659
|
-
//
|
|
3660
|
-
// trace boundary, failed its idle gate, and the promise hung
|
|
3661
|
-
// forever — every "zombie subagent" in production tracked back
|
|
3662
|
-
// here. Speech dispatch happens after but doesn't depend on the
|
|
3663
|
-
// status field.
|
|
3727
|
+
// Reset agent state before emitting inference:completed. Traces are
|
|
3728
|
+
// observability-only, but external synchronous listeners should
|
|
3729
|
+
// still see the terminal state at the terminal trace boundary.
|
|
3730
|
+
// Speech dispatch happens after but doesn't depend on the status
|
|
3731
|
+
// field.
|
|
3664
3732
|
agent.reset();
|
|
3665
3733
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
3734
|
+
this.settleAgent(agent.name, {
|
|
3735
|
+
stopReason: 'completed',
|
|
3736
|
+
speech: terminalContent
|
|
3737
|
+
.filter((block: ContentBlock): block is ContentBlock & { type: 'text' } => block.type === 'text')
|
|
3738
|
+
.map((block) => block.text)
|
|
3739
|
+
.join('\n'),
|
|
3740
|
+
});
|
|
3666
3741
|
|
|
3667
3742
|
this.emitTrace({
|
|
3668
3743
|
type: 'inference:completed',
|
|
@@ -3901,9 +3976,8 @@ export class AgentFramework {
|
|
|
3901
3976
|
}
|
|
3902
3977
|
}
|
|
3903
3978
|
}
|
|
3904
|
-
// NOTE: agent.reset() + onInferenceEnded() already ran above,
|
|
3905
|
-
// dispatchSpeech
|
|
3906
|
-
// must see status === 'idle'). Locus routing is speech dispatch and
|
|
3979
|
+
// NOTE: agent.reset() + onInferenceEnded() already ran above,
|
|
3980
|
+
// BEFORE dispatchSpeech. Locus routing is speech dispatch and
|
|
3907
3981
|
// doesn't depend on the status field, so it correctly runs after.
|
|
3908
3982
|
|
|
3909
3983
|
break;
|
|
@@ -3939,16 +4013,20 @@ export class AgentFramework {
|
|
|
3939
4013
|
await new Promise((resolve) => setTimeout(resolve, action.delayMs));
|
|
3940
4014
|
await this.startAgentStream(agent, trigger, attempt + 1);
|
|
3941
4015
|
} else {
|
|
4016
|
+
this.settleAgent(agent.name, {
|
|
4017
|
+
stopReason: 'exhausted',
|
|
4018
|
+
speech: '',
|
|
4019
|
+
error: err.message,
|
|
4020
|
+
});
|
|
3942
4021
|
this.emitTrace({
|
|
3943
4022
|
type: 'inference:exhausted',
|
|
3944
4023
|
agentName: agent.name,
|
|
3945
4024
|
error: err.message,
|
|
3946
4025
|
// Drives the poison-history breaker. `retryable` is kept for
|
|
3947
|
-
// observability, but the
|
|
4026
|
+
// observability, but the breakers gate on `errorType` — only an
|
|
3948
4027
|
// `invalid_request` (a 400-class rejection of the history
|
|
3949
4028
|
// itself) means retrying the same context can never succeed.
|
|
3950
|
-
|
|
3951
|
-
errorType: err instanceof MembraneError ? err.type : undefined,
|
|
4029
|
+
...this.classifyInferenceError(err),
|
|
3952
4030
|
});
|
|
3953
4031
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
3954
4032
|
if (action.emit) {
|
|
@@ -3959,12 +4037,28 @@ export class AgentFramework {
|
|
|
3959
4037
|
}
|
|
3960
4038
|
|
|
3961
4039
|
case 'aborted': {
|
|
4040
|
+
// Framework-initiated, non-terminal cancels (endTurn tool result,
|
|
4041
|
+
// context-budget restart) also surface here as `aborted` — but
|
|
4042
|
+
// the turn either already settled (endTurn) or a replacement
|
|
4043
|
+
// stream is queued (budget restart). Neither is a failure: no
|
|
4044
|
+
// reset, no settle, no spurious inference:exhausted (which would
|
|
4045
|
+
// reject an ephemeral's promise mid-run and bump the failure
|
|
4046
|
+
// streak). Gate release + stream teardown happen in `finally`.
|
|
4047
|
+
if (this.frameworkCancelledStreams.delete(`${agent.name}:${myStreamId}`)) {
|
|
4048
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
4049
|
+
return;
|
|
4050
|
+
}
|
|
3962
4051
|
const reason = event.reason ?? 'unknown';
|
|
3963
4052
|
// Only reset if this is still the active stream (a budget restart
|
|
3964
4053
|
// may have already started a new stream, bumping streamId)
|
|
3965
4054
|
if (agent.streamId === myStreamId) {
|
|
3966
4055
|
const durationMs = Date.now() - startTime;
|
|
3967
4056
|
agent.reset();
|
|
4057
|
+
this.settleAgent(agent.name, {
|
|
4058
|
+
stopReason: 'exhausted',
|
|
4059
|
+
speech: '',
|
|
4060
|
+
error: `Stream aborted: ${reason}`,
|
|
4061
|
+
});
|
|
3968
4062
|
this.emitTrace({
|
|
3969
4063
|
type: 'inference:exhausted',
|
|
3970
4064
|
agentName: agent.name,
|
|
@@ -4040,10 +4134,16 @@ export class AgentFramework {
|
|
|
4040
4134
|
error: err.message,
|
|
4041
4135
|
stack: err.stack,
|
|
4042
4136
|
});
|
|
4137
|
+
this.settleAgent(agent.name, {
|
|
4138
|
+
stopReason: 'exhausted',
|
|
4139
|
+
speech: '',
|
|
4140
|
+
error: err.message,
|
|
4141
|
+
});
|
|
4043
4142
|
this.emitTrace({
|
|
4044
4143
|
type: 'inference:exhausted',
|
|
4045
4144
|
agentName: agent.name,
|
|
4046
4145
|
error: err.message,
|
|
4146
|
+
...this.classifyInferenceError(err),
|
|
4047
4147
|
});
|
|
4048
4148
|
// Postmortem 2026-05-28 P2 #7: catch-path failures (stream itself
|
|
4049
4149
|
// threw) were previously only visible via in-memory trace listeners.
|
|
@@ -4073,6 +4173,7 @@ export class AgentFramework {
|
|
|
4073
4173
|
// Stop the typing indicator on every exit path (complete, error,
|
|
4074
4174
|
// exhausted, abort) so it never sticks after the turn ends.
|
|
4075
4175
|
this.channelRegistry?.stopTyping();
|
|
4176
|
+
this.frameworkCancelledStreams.delete(`${agent.name}:${myStreamId}`);
|
|
4076
4177
|
this.activeStreams.delete(agent.name);
|
|
4077
4178
|
this.pendingAssistantBlocks.delete(agent.name);
|
|
4078
4179
|
|
|
@@ -4661,6 +4762,28 @@ export class AgentFramework {
|
|
|
4661
4762
|
return streak;
|
|
4662
4763
|
}
|
|
4663
4764
|
|
|
4765
|
+
/**
|
|
4766
|
+
* Classify a terminal inference error for the `inference:exhausted` trace.
|
|
4767
|
+
* Single-sourced for every emit site: the downstream breakers (poison-history
|
|
4768
|
+
* quarantine, OverBudget drain kick) gate on `errorType`, so classification
|
|
4769
|
+
* drift between sites would silently disable a safety net.
|
|
4770
|
+
*
|
|
4771
|
+
* context-manager's OverBudgetError is recognized by `err.name`: CM does not
|
|
4772
|
+
* export the class from its package root, so a cross-package `instanceof` is
|
|
4773
|
+
* unavailable — but `name` is set in its constructor and survives the package
|
|
4774
|
+
* boundary. Deliberately NOT a message match: the message wording belongs to
|
|
4775
|
+
* CM and can be reworded without warning.
|
|
4776
|
+
*/
|
|
4777
|
+
private classifyInferenceError(err: Error): { retryable?: boolean; errorType?: string } {
|
|
4778
|
+
if (err instanceof MembraneError) {
|
|
4779
|
+
return { retryable: err.retryable, errorType: err.type };
|
|
4780
|
+
}
|
|
4781
|
+
if (err.name === 'OverBudgetError') {
|
|
4782
|
+
return { errorType: 'over_budget' };
|
|
4783
|
+
}
|
|
4784
|
+
return {};
|
|
4785
|
+
}
|
|
4786
|
+
|
|
4664
4787
|
private noteInferenceExhausted(
|
|
4665
4788
|
agentName: string,
|
|
4666
4789
|
reason: string,
|
|
@@ -4701,6 +4824,41 @@ export class AgentFramework {
|
|
|
4701
4824
|
}
|
|
4702
4825
|
}
|
|
4703
4826
|
|
|
4827
|
+
// (2b) OverBudget deadlock breaker. When compile fails with
|
|
4828
|
+
// OverBudgetError, the normal compression drain never runs: it is driven
|
|
4829
|
+
// by successful activity, which the over-budget state prevents - a closed
|
|
4830
|
+
// loop with no internal exit. Field data (2026-07-10, resident agent):
|
|
4831
|
+
// 36 minutes hard-down, zero self-rescue; only an operator raising the
|
|
4832
|
+
// budget externally could break the loop. Break it here instead: kick the
|
|
4833
|
+
// strategy drain directly so folding/merging frees space for the next
|
|
4834
|
+
// compile. Bounded ticks, best-effort, never throws, one kick per agent
|
|
4835
|
+
// at a time (see overBudgetDrainInFlight).
|
|
4836
|
+
//
|
|
4837
|
+
// Gates on the classified errorType (see classifyInferenceError); the
|
|
4838
|
+
// message match is only a fallback for paths that lost the Error object
|
|
4839
|
+
// (e.g. a reason string that crossed a serialization boundary). It matches
|
|
4840
|
+
// CM's current OverBudgetError wording and MAY rot if CM rewords it — the
|
|
4841
|
+
// errorType gate is the one that's load-bearing.
|
|
4842
|
+
const overBudget = errorType === 'over_budget' || /exceed hard budget/i.test(reason);
|
|
4843
|
+
if (agent && overBudget && !this.overBudgetDrainInFlight.has(agentName)) {
|
|
4844
|
+
this.overBudgetDrainInFlight.add(agentName);
|
|
4845
|
+
void (async () => {
|
|
4846
|
+
let ticks = 0;
|
|
4847
|
+
try {
|
|
4848
|
+
const cm = agent.getContextManager();
|
|
4849
|
+
while (ticks < 8) {
|
|
4850
|
+
await cm.tick();
|
|
4851
|
+
ticks++;
|
|
4852
|
+
}
|
|
4853
|
+
console.error(`[inference-failed] drain kicked for ${agentName} (OverBudget breaker, ${ticks} ticks)`);
|
|
4854
|
+
} catch (err) {
|
|
4855
|
+
console.error(`[inference-failed] drain kick failed for ${agentName} after ${ticks} ticks:`, err);
|
|
4856
|
+
} finally {
|
|
4857
|
+
this.overBudgetDrainInFlight.delete(agentName);
|
|
4858
|
+
}
|
|
4859
|
+
})();
|
|
4860
|
+
}
|
|
4861
|
+
|
|
4704
4862
|
// (3) Hard-down escalation on repeated identical failure.
|
|
4705
4863
|
if (streak >= this.inferenceFailureEscalationThreshold) {
|
|
4706
4864
|
console.error(
|
|
@@ -5764,7 +5922,16 @@ export class AgentFramework {
|
|
|
5764
5922
|
|
|
5765
5923
|
console.error(`[sleep] agent=${agentName} seconds=${seconds} announce=${announce} until=${new Date(until).toISOString()}`);
|
|
5766
5924
|
// endTurn: the agent stops here and goes idle for the duration.
|
|
5767
|
-
finish({
|
|
5925
|
+
finish({
|
|
5926
|
+
success: true,
|
|
5927
|
+
data: {
|
|
5928
|
+
sleepingFor: human,
|
|
5929
|
+
until,
|
|
5930
|
+
untilLocal: formatZonedDateTime(until, this.timeZone),
|
|
5931
|
+
timeZone: this.timeZone,
|
|
5932
|
+
},
|
|
5933
|
+
endTurn: true,
|
|
5934
|
+
});
|
|
5768
5935
|
}
|
|
5769
5936
|
|
|
5770
5937
|
/** Aggregate the event-tag vocabulary: reserved chat:* core + each connected
|