@animalabs/agent-framework 0.6.4 → 0.6.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -147,6 +147,17 @@ export class AgentFramework {
147
147
  processLoggingBroadcast;
148
148
  activeStreams = new Map();
149
149
  pendingAssistantBlocks = new Map();
150
+ /** Streams the FRAMEWORK cancelled for non-terminal reasons, keyed
151
+ * `${agentName}:${streamId}`: an endTurn tool result or a context-budget
152
+ * restart. The membrane still delivers an `aborted` event for these, and
153
+ * without the marker the abort handler would treat it as a terminal
154
+ * failure — settling ephemerals with a rejection and emitting a spurious
155
+ * `inference:exhausted` (which also pollutes the failure streak). Kept
156
+ * separate from ephemeralRuns deliberately: endTurn/budget cancels happen
157
+ * for resident agents too, and the key is per-stream, not per-agent. */
158
+ frameworkCancelledStreams = new Map();
159
+ /** Active runEphemeralToCompletion runs, keyed by agent name. */
160
+ ephemeralRuns = new Map();
150
161
  /** Per-agent count of consecutive exhausted inferences (reset on any success).
151
162
  * Drives hard-down escalation — see noteInferenceExhausted. */
152
163
  consecutiveInferenceFailures = new Map();
@@ -175,6 +186,13 @@ export class AgentFramework {
175
186
  * Bounds the automatic quarantine loop the same way refusalRewinds bounds
176
187
  * the refusal loop, so the breaker can never shed the whole history. */
177
188
  exhaustionRewinds = new Map();
189
+ /** Agents with an OverBudget drain kick currently in flight. The breaker in
190
+ * noteInferenceExhausted fires on EVERY matching failure and the scenario it
191
+ * exists for is "every activation fails, repeatedly" — without this guard,
192
+ * overlapping kicks race the strategy's own pendingCompression gate and the
193
+ * tick counts in the success log stop meaning anything. One kick per agent
194
+ * at a time; cleared when the kick settles (success or failure). */
195
+ overBudgetDrainInFlight = new Set();
178
196
  /** Per-agent current rewind episode: the single consolidated marker's id and
179
197
  * how many turns have been shed so far. One marker per episode, updated in
180
198
  * place (see updateRewindMarker); cleared when the episode ends. */
@@ -537,7 +555,9 @@ export class AgentFramework {
537
555
  if (this.maintenancePass || !this.running)
538
556
  return;
539
557
  const pass = this.runQueuedMaintenance().catch((error) => {
540
- console.error('[context-maintenance] pass failed:', error instanceof Error ? error.message : error);
558
+ const reason = error instanceof Error ? error.message : String(error);
559
+ console.error('[context-maintenance] pass failed:', reason);
560
+ this.opsAlert('context-maintenance-failed', 'framework', reason, { data: { scope: 'pass' } });
541
561
  });
542
562
  this.maintenancePass = pass;
543
563
  void pass.finally(() => {
@@ -588,6 +608,13 @@ export class AgentFramework {
588
608
  catch (error) {
589
609
  record.error = error instanceof Error ? error.message : String(error);
590
610
  console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
611
+ this.opsAlert('context-maintenance-failed', agent.name, record.error, {
612
+ data: {
613
+ scope: 'agent',
614
+ ...(record.pendingBefore ? { pending: record.pendingBefore } : {}),
615
+ ...(record.progressBefore ? { progress: record.progressBefore } : {}),
616
+ },
617
+ });
591
618
  }
592
619
  finally {
593
620
  const pending = cm.getPendingWork()?.description;
@@ -934,6 +961,50 @@ export class AgentFramework {
934
961
  };
935
962
  return { agent, contextManager, cleanup };
936
963
  }
964
+ createDeferred() {
965
+ let resolve;
966
+ let reject;
967
+ const promise = new Promise((res, rej) => {
968
+ resolve = res;
969
+ reject = rej;
970
+ });
971
+ return { promise, resolve, reject };
972
+ }
973
+ /** Liveness ping for an active ephemeral run (no-op otherwise). */
974
+ touchEphemeralRun(agentName, inferenceStarted = false) {
975
+ const run = this.ephemeralRuns.get(agentName);
976
+ if (!run)
977
+ return;
978
+ if (inferenceStarted) {
979
+ run.inferenceStarted = true;
980
+ }
981
+ run.lastActivity = Date.now();
982
+ }
983
+ recordEphemeralToolCalls(agentName, count) {
984
+ const run = this.ephemeralRuns.get(agentName);
985
+ if (!run)
986
+ return;
987
+ run.toolCallsCount += count;
988
+ run.lastActivity = Date.now();
989
+ }
990
+ /** Settle an active ephemeral run (no-op for resident agents). The tool-call
991
+ * count comes from the run's own counter — the single source of truth,
992
+ * which unlike a per-stream local survives budget restarts. */
993
+ settleAgent(agentName, outcome) {
994
+ const run = this.ephemeralRuns.get(agentName);
995
+ if (!run)
996
+ return;
997
+ run.lastActivity = Date.now();
998
+ if (outcome.stopReason === 'exhausted') {
999
+ run.settle.reject(new Error(outcome.error ?? 'Unknown error'));
1000
+ return;
1001
+ }
1002
+ run.settle.resolve({
1003
+ stopReason: outcome.stopReason,
1004
+ speech: outcome.speech,
1005
+ toolCallsCount: run.toolCallsCount,
1006
+ });
1007
+ }
937
1008
  /**
938
1009
  * Run an ephemeral agent to completion through the framework's event loop.
939
1010
  *
@@ -944,139 +1015,73 @@ export class AgentFramework {
944
1015
  * The caller provides a pre-created agent + contextManager (from createEphemeralAgent).
945
1016
  * The task message should already be in the context manager.
946
1017
  */
947
- async runEphemeralToCompletion(agent, contextManager) {
1018
+ async runEphemeralToCompletion(agent, contextManager, watchdogs) {
948
1019
  // Register temporarily so the event loop can drive it
949
1020
  this.agents.set(agent.name, agent);
950
- return new Promise((resolve, reject) => {
951
- let speech = '';
952
- let toolCallsCount = 0;
953
- let settled = false;
954
- let inferenceStarted = false;
955
- let lastActivity = Date.now();
956
- const STARTUP_TIMEOUT_MS = 30_000;
957
- // After inference has started, give it 15 minutes of activity-bounded
958
- // life. Each addressed trace event refreshes the deadline; only
959
- // sustained silence trips it. Subagents that legitimately stream for
960
- // 10+ minutes (e.g. long fork investigations) get the extra slack but
961
- // stalled streams that emit `inference:completed` and then go quiet —
962
- // the exact zombie shape we saw in production — are caught here.
963
- const COMPLETION_IDLE_TIMEOUT_MS = 15 * 60_000;
964
- const cleanup = () => {
965
- if (settled)
966
- return;
967
- settled = true;
968
- clearTimeout(startupWatchdog);
969
- clearInterval(completionWatchdog);
970
- this.offTrace(traceListener);
971
- this.agents.delete(agent.name);
972
- this.evictTurnCheckpoints(agent.name);
973
- };
974
- // Watchdog: if inference never starts within 30s, the agent is a zombie.
975
- // This catches cases where the event loop stalled, the agent was
976
- // deregistered, or processInferenceRequests() skipped the request.
977
- const startupWatchdog = setTimeout(() => {
978
- if (!inferenceStarted && !settled) {
979
- cleanup();
1021
+ const run = {
1022
+ settle: this.createDeferred(),
1023
+ inferenceStarted: false,
1024
+ lastActivity: Date.now(),
1025
+ toolCallsCount: 0,
1026
+ };
1027
+ this.ephemeralRuns.set(agent.name, run);
1028
+ const STARTUP_TIMEOUT_MS = watchdogs?.startupTimeoutMs ?? 30_000;
1029
+ // After inference has started, give it 15 minutes of activity-bounded
1030
+ // life. The stream driver refreshes the deadline as it makes progress;
1031
+ // sustained silence trips it.
1032
+ const COMPLETION_IDLE_TIMEOUT_MS = watchdogs?.idleTimeoutMs ?? 15 * 60_000;
1033
+ const IDLE_POLL_MS = watchdogs?.idlePollMs ?? 30_000;
1034
+ let startupWatchdog = null;
1035
+ let completionWatchdog = null;
1036
+ const startupTimeout = new Promise((_, reject) => {
1037
+ startupWatchdog = setTimeout(() => {
1038
+ if (!run.inferenceStarted) {
980
1039
  reject(new Error(`Ephemeral agent "${agent.name}" failed to start inference within ` +
981
1040
  `${STARTUP_TIMEOUT_MS}ms — zombie detected. The event loop may have ` +
982
1041
  `stalled or the inference request was dropped.`));
983
1042
  }
984
1043
  }, STARTUP_TIMEOUT_MS);
985
- // Completion watchdog: once inference has started, reject if no trace
986
- // activity for COMPLETION_IDLE_TIMEOUT_MS. Polls every 30s. Without
987
- // this, an ephemeral that loses its terminal event (e.g. due to a
988
- // future ordering regression or an upstream stream drop) hangs forever
989
- // and keeps its concurrency slot pinned.
990
- const completionWatchdog = setInterval(() => {
991
- if (settled || !inferenceStarted)
1044
+ });
1045
+ const completionIdleTimeout = new Promise((_, reject) => {
1046
+ completionWatchdog = setInterval(() => {
1047
+ if (!run.inferenceStarted)
992
1048
  return;
993
- const idle = Date.now() - lastActivity;
1049
+ const idle = Date.now() - run.lastActivity;
994
1050
  if (idle > COMPLETION_IDLE_TIMEOUT_MS) {
995
- cleanup();
996
- reject(new Error(`Ephemeral agent "${agent.name}" stalled: no trace activity for ` +
1051
+ reject(new Error(`Ephemeral agent "${agent.name}" stalled: no stream-driver activity for ` +
997
1052
  `${Math.round(idle / 1000)}s after inference started (threshold ` +
998
1053
  `${Math.round(COMPLETION_IDLE_TIMEOUT_MS / 1000)}s). Stream likely ` +
999
1054
  `dropped or terminal event was lost.`));
1000
1055
  }
1001
- }, 30_000);
1002
- const traceListener = (event) => {
1003
- if (settled)
1004
- return;
1005
- // Only track events for our ephemeral agent
1006
- const agentName = 'agentName' in event ? event.agentName : null;
1007
- if (agentName !== agent.name)
1008
- return;
1009
- // Any addressed event counts as liveness for the completion watchdog.
1010
- lastActivity = Date.now();
1011
- switch (event.type) {
1012
- case 'inference:started':
1013
- inferenceStarted = true;
1014
- break;
1015
- case 'inference:tokens': {
1016
- inferenceStarted = true;
1017
- const content = event.content;
1018
- if (content)
1019
- speech += content;
1020
- break;
1021
- }
1022
- case 'inference:tool_calls_yielded': {
1023
- const calls = event.calls;
1024
- if (calls)
1025
- toolCallsCount += calls.length;
1026
- // Reset speech buffer — tool calls break the text
1027
- speech = '';
1028
- break;
1029
- }
1030
- case 'inference:stream_resumed':
1031
- speech = '';
1032
- break;
1033
- case 'inference:completed': {
1034
- // `inference:completed` is terminal — driveStream only emits it
1035
- // from `case 'complete'`, which fires after the full tool
1036
- // round-trip has finished and the model has stopped. Intermediate
1037
- // tool-cycle transitions use `inference:stream_resumed`, not
1038
- // `inference:completed`. So we resolve unconditionally here.
1039
- //
1040
- // Previously this branch gated on `agent.state.status === 'idle'`,
1041
- // which was correct in intent but order-fragile: the framework
1042
- // emitted `inference:completed` BEFORE calling `agent.reset()`,
1043
- // so synchronous listeners observed status === 'streaming' at
1044
- // the trace boundary. The gate failed silently and the promise
1045
- // hung forever — every production zombie subagent originated
1046
- // here. The reset+emit order has been fixed in driveStream's
1047
- // `case 'complete'` handler; the gate is dropped here as
1048
- // belt-and-suspenders.
1049
- cleanup();
1050
- resolve({ speech, toolCallsCount });
1051
- break;
1052
- }
1053
- case 'inference:turn_ended': {
1054
- // endTurn from a tool result — agent is done
1055
- cleanup();
1056
- resolve({ speech, toolCallsCount });
1057
- break;
1058
- }
1059
- case 'inference:exhausted': {
1060
- // Retries exhausted — reject only after the error policy gives up.
1061
- // Earlier inference:failed events are left alone so the framework
1062
- // retry path in driveStream/startAgentStream can restart the stream
1063
- // while this listener stays alive.
1064
- const error = event.error ?? 'Unknown error';
1065
- cleanup();
1066
- reject(new Error(error));
1067
- break;
1068
- }
1069
- }
1070
- };
1071
- this.onTrace(traceListener);
1072
- // Trigger inference
1056
+ }, IDLE_POLL_MS);
1057
+ });
1058
+ try {
1059
+ // Trigger inference after the settle promise is registered.
1073
1060
  this.pendingRequests.push({
1074
1061
  agentName: agent.name,
1075
1062
  reason: 'ephemeral',
1076
1063
  source: 'subagent',
1077
1064
  timestamp: Date.now(),
1078
1065
  });
1079
- });
1066
+ const result = await Promise.race([
1067
+ run.settle.promise,
1068
+ startupTimeout,
1069
+ completionIdleTimeout,
1070
+ ]);
1071
+ return { speech: result.speech, toolCallsCount: result.toolCallsCount };
1072
+ }
1073
+ finally {
1074
+ if (startupWatchdog)
1075
+ clearTimeout(startupWatchdog);
1076
+ if (completionWatchdog)
1077
+ clearInterval(completionWatchdog);
1078
+ this.ephemeralRuns.delete(agent.name);
1079
+ this.agents.delete(agent.name);
1080
+ // Spawn-and-dispose bookkeeping (main, d453165/fee96a7): without this,
1081
+ // ephemeral agents leave checkpoint-tree keys and diagnostics map
1082
+ // entries behind for the life of the store/session.
1083
+ this.evictTurnCheckpoints(agent.name);
1084
+ }
1080
1085
  }
1081
1086
  /**
1082
1087
  * Get queue depth.
@@ -2196,6 +2201,9 @@ export class AgentFramework {
2196
2201
  // messages between tool_use and tool_result, causing a 400 error.
2197
2202
  if (event.type === 'tool-result') {
2198
2203
  const agent = this.agents.get(event.agentName);
2204
+ if (agent) {
2205
+ this.touchEphemeralRun(event.agentName, true);
2206
+ }
2199
2207
  if (!agent) {
2200
2208
  console.warn(`[framework] Dropping tool-result for unknown agent '${event.agentName}' (callId=${event.callId}). ` +
2201
2209
  `Agent may have been destroyed while tool was executing.`);
@@ -2260,13 +2268,29 @@ export class AgentFramework {
2260
2268
  if (shouldEndTurn) {
2261
2269
  // endTurn: messages already stored above, cancel stream, reset to idle.
2262
2270
  if (currentState.stream) {
2263
- agent.cancelStream();
2271
+ this.frameworkCancelledStreams.set(`${agent.name}:${agent.streamId}`, 'turn_ended');
2272
+ currentState.stream.cancel();
2264
2273
  }
2265
2274
  agent.reset();
2275
+ // The turn is over — release the gate here, not just in the abort
2276
+ // handler: the membrane's aborted event is asynchronous and the
2277
+ // agent must be wakeable the moment it goes idle. (Idempotent;
2278
+ // driveStream's finally is the backstop.)
2279
+ this.eventGate?.onInferenceEnded(agent.name);
2280
+ this.settleAgent(agent.name, { stopReason: 'turn_ended', speech: '' });
2266
2281
  this.emitTrace({ type: 'inference:turn_ended', agentName: agent.name });
2267
2282
  }
2268
2283
  else if (overBudget) {
2269
- // Context budget exceeded: break the stream, let compile() compress
2284
+ // Context budget exceeded: break the stream, let compile() compress.
2285
+ // Mark the cancel as framework-initiated BEFORE cancelling: the
2286
+ // membrane delivers `aborted` before the restart bumps streamId,
2287
+ // and without the marker the abort handler treats the restart as
2288
+ // a terminal failure — rejecting an ephemeral's promise mid-run
2289
+ // and emitting a spurious inference:exhausted (same race shape as
2290
+ // endTurn above).
2291
+ if (currentState.stream) {
2292
+ this.frameworkCancelledStreams.set(`${agent.name}:${agent.streamId}`, 'budget_restart');
2293
+ }
2270
2294
  agent.cancelStream();
2271
2295
  this.emitTrace({
2272
2296
  type: 'inference:stream_restarted',
@@ -2825,6 +2849,7 @@ export class AgentFramework {
2825
2849
  else {
2826
2850
  this.activeTriggerChannels.delete(agent.name);
2827
2851
  }
2852
+ this.touchEphemeralRun(agent.name, true);
2828
2853
  this.emitTrace({ type: 'inference:started', agentName: agent.name });
2829
2854
  this.eventGate?.onInferenceStarted(agent.name);
2830
2855
  this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), startedAt: Date.now() });
@@ -2880,17 +2905,23 @@ export class AgentFramework {
2880
2905
  await this.startAgentStream(agent, trigger, attempt + 1);
2881
2906
  }
2882
2907
  else {
2908
+ this.settleAgent(agent.name, {
2909
+ stopReason: 'exhausted',
2910
+ speech: '',
2911
+ error: err.message,
2912
+ });
2883
2913
  this.emitTrace({
2884
2914
  type: 'inference:exhausted',
2885
2915
  agentName: agent.name,
2886
2916
  error: err.message,
2887
- // Drives the poison-history breaker. `retryable` is kept for
2888
- // observability, but the breaker gates on `errorType` — only an
2889
- // `invalid_request` (a 400-class rejection of the history itself)
2890
- // means retrying the same context can never succeed. auth/abort/
2891
- // context_length are also non-retryable but must NOT cost history.
2892
- retryable: err instanceof MembraneError ? err.retryable : undefined,
2893
- errorType: err instanceof MembraneError ? err.type : undefined,
2917
+ // Drives the poison-history breaker (`invalid_request`: a 400-class
2918
+ // rejection of the history itself — retrying the same context can
2919
+ // never succeed; auth/abort/context_length are also non-retryable
2920
+ // but must NOT cost history) and the OverBudget drain breaker
2921
+ // (`over_budget`: compile refused to fit the hard budget — this is
2922
+ // the site that sees it, since compile runs before the stream
2923
+ // exists). `retryable` is kept for observability.
2924
+ ...this.classifyInferenceError(err),
2894
2925
  });
2895
2926
  this.eventGate?.onInferenceEnded(agent.name);
2896
2927
  if (action.emit) {
@@ -2913,6 +2944,7 @@ export class AgentFramework {
2913
2944
  this.channelRegistry.startTyping(typingChannel);
2914
2945
  try {
2915
2946
  for await (const event of stream) {
2947
+ this.touchEphemeralRun(agent.name, true);
2916
2948
  switch (event.type) {
2917
2949
  case 'tokens':
2918
2950
  this.emitTrace({
@@ -2936,6 +2968,7 @@ export class AgentFramework {
2936
2968
  }
2937
2969
  case 'tool-calls': {
2938
2970
  hadToolCalls = true;
2971
+ this.recordEphemeralToolCalls(agent.name, event.calls.length);
2939
2972
  this.emitTrace({
2940
2973
  type: 'inference:tool_calls_yielded',
2941
2974
  agentName: agent.name,
@@ -3021,14 +3054,16 @@ export class AgentFramework {
3021
3054
  // thinking blocks from their tool_use turn (the API requires
3022
3055
  // thinking to precede tool_use in the same assistant turn).
3023
3056
  const lastToolIdx = response.content.reduce((last, b, i) => b.type === 'tool_use' || b.type === 'tool_result' ? i : last, -1);
3057
+ const terminalContent = lastToolIdx >= 0
3058
+ ? response.content.slice(lastToolIdx + 1)
3059
+ : response.content;
3024
3060
  if (lastToolIdx >= 0) {
3025
- const trailingContent = response.content.slice(lastToolIdx + 1);
3026
- if (trailingContent.length > 0) {
3027
- agent.addAssistantResponse(trailingContent);
3061
+ if (terminalContent.length > 0) {
3062
+ agent.addAssistantResponse(terminalContent);
3028
3063
  }
3029
3064
  }
3030
3065
  else {
3031
- agent.addAssistantResponse(response.content);
3066
+ agent.addAssistantResponse(terminalContent);
3032
3067
  }
3033
3068
  // Run afterInference hooks (no-op if no MCPL servers)
3034
3069
  if (this.hookOrchestrator) {
@@ -3080,17 +3115,20 @@ export class AgentFramework {
3080
3115
  cacheRead: du.cacheReadTokens,
3081
3116
  }
3082
3117
  : undefined;
3083
- // Reset agent state BEFORE emitting inference:completed so that
3084
- // synchronous listeners (e.g. runEphemeralToCompletion) observe
3085
- // status === 'idle' the moment the event fires. Previously
3086
- // agent.reset() ran after `await dispatchSpeech`, which meant the
3087
- // ephemeral-completion listener saw status === 'streaming' at the
3088
- // trace boundary, failed its idle gate, and the promise hung
3089
- // forever — every "zombie subagent" in production tracked back
3090
- // here. Speech dispatch happens after but doesn't depend on the
3091
- // status field.
3118
+ // Reset agent state before emitting inference:completed. Traces are
3119
+ // observability-only, but external synchronous listeners should
3120
+ // still see the terminal state at the terminal trace boundary.
3121
+ // Speech dispatch happens after but doesn't depend on the status
3122
+ // field.
3092
3123
  agent.reset();
3093
3124
  this.eventGate?.onInferenceEnded(agent.name);
3125
+ this.settleAgent(agent.name, {
3126
+ stopReason: 'completed',
3127
+ speech: terminalContent
3128
+ .filter((block) => block.type === 'text')
3129
+ .map((block) => block.text)
3130
+ .join('\n'),
3131
+ });
3094
3132
  this.emitTrace({
3095
3133
  type: 'inference:completed',
3096
3134
  agentName: agent.name,
@@ -3301,9 +3339,8 @@ export class AgentFramework {
3301
3339
  }
3302
3340
  }
3303
3341
  }
3304
- // NOTE: agent.reset() + onInferenceEnded() already ran above, BEFORE
3305
- // dispatchSpeech (main's zombie-subagent fix, PR #32 — sync listeners
3306
- // must see status === 'idle'). Locus routing is speech dispatch and
3342
+ // NOTE: agent.reset() + onInferenceEnded() already ran above,
3343
+ // BEFORE dispatchSpeech. Locus routing is speech dispatch and
3307
3344
  // doesn't depend on the status field, so it correctly runs after.
3308
3345
  break;
3309
3346
  }
@@ -3335,16 +3372,20 @@ export class AgentFramework {
3335
3372
  await this.startAgentStream(agent, trigger, attempt + 1);
3336
3373
  }
3337
3374
  else {
3375
+ this.settleAgent(agent.name, {
3376
+ stopReason: 'exhausted',
3377
+ speech: '',
3378
+ error: err.message,
3379
+ });
3338
3380
  this.emitTrace({
3339
3381
  type: 'inference:exhausted',
3340
3382
  agentName: agent.name,
3341
3383
  error: err.message,
3342
3384
  // Drives the poison-history breaker. `retryable` is kept for
3343
- // observability, but the breaker gates on `errorType` — only an
3385
+ // observability, but the breakers gate on `errorType` — only an
3344
3386
  // `invalid_request` (a 400-class rejection of the history
3345
3387
  // itself) means retrying the same context can never succeed.
3346
- retryable: err instanceof MembraneError ? err.retryable : undefined,
3347
- errorType: err instanceof MembraneError ? err.type : undefined,
3388
+ ...this.classifyInferenceError(err),
3348
3389
  });
3349
3390
  this.eventGate?.onInferenceEnded(agent.name);
3350
3391
  if (action.emit) {
@@ -3354,12 +3395,28 @@ export class AgentFramework {
3354
3395
  break;
3355
3396
  }
3356
3397
  case 'aborted': {
3398
+ // Framework-initiated, non-terminal cancels (endTurn tool result,
3399
+ // context-budget restart) also surface here as `aborted` — but
3400
+ // the turn either already settled (endTurn) or a replacement
3401
+ // stream is queued (budget restart). Neither is a failure: no
3402
+ // reset, no settle, no spurious inference:exhausted (which would
3403
+ // reject an ephemeral's promise mid-run and bump the failure
3404
+ // streak). Gate release + stream teardown happen in `finally`.
3405
+ if (this.frameworkCancelledStreams.delete(`${agent.name}:${myStreamId}`)) {
3406
+ this.eventGate?.onInferenceEnded(agent.name);
3407
+ return;
3408
+ }
3357
3409
  const reason = event.reason ?? 'unknown';
3358
3410
  // Only reset if this is still the active stream (a budget restart
3359
3411
  // may have already started a new stream, bumping streamId)
3360
3412
  if (agent.streamId === myStreamId) {
3361
3413
  const durationMs = Date.now() - startTime;
3362
3414
  agent.reset();
3415
+ this.settleAgent(agent.name, {
3416
+ stopReason: 'exhausted',
3417
+ speech: '',
3418
+ error: `Stream aborted: ${reason}`,
3419
+ });
3363
3420
  this.emitTrace({
3364
3421
  type: 'inference:exhausted',
3365
3422
  agentName: agent.name,
@@ -3429,10 +3486,16 @@ export class AgentFramework {
3429
3486
  error: err.message,
3430
3487
  stack: err.stack,
3431
3488
  });
3489
+ this.settleAgent(agent.name, {
3490
+ stopReason: 'exhausted',
3491
+ speech: '',
3492
+ error: err.message,
3493
+ });
3432
3494
  this.emitTrace({
3433
3495
  type: 'inference:exhausted',
3434
3496
  agentName: agent.name,
3435
3497
  error: err.message,
3498
+ ...this.classifyInferenceError(err),
3436
3499
  });
3437
3500
  // Postmortem 2026-05-28 P2 #7: catch-path failures (stream itself
3438
3501
  // threw) were previously only visible via in-memory trace listeners.
@@ -3462,6 +3525,7 @@ export class AgentFramework {
3462
3525
  // Stop the typing indicator on every exit path (complete, error,
3463
3526
  // exhausted, abort) so it never sticks after the turn ends.
3464
3527
  this.channelRegistry?.stopTyping();
3528
+ this.frameworkCancelledStreams.delete(`${agent.name}:${myStreamId}`);
3465
3529
  this.activeStreams.delete(agent.name);
3466
3530
  this.pendingAssistantBlocks.delete(agent.name);
3467
3531
  // A conversation fork whose TTL closure turn just finished is done for
@@ -3979,6 +4043,27 @@ export class AgentFramework {
3979
4043
  }
3980
4044
  return streak;
3981
4045
  }
4046
+ /**
4047
+ * Classify a terminal inference error for the `inference:exhausted` trace.
4048
+ * Single-sourced for every emit site: the downstream breakers (poison-history
4049
+ * quarantine, OverBudget drain kick) gate on `errorType`, so classification
4050
+ * drift between sites would silently disable a safety net.
4051
+ *
4052
+ * context-manager's OverBudgetError is recognized by `err.name`: CM does not
4053
+ * export the class from its package root, so a cross-package `instanceof` is
4054
+ * unavailable — but `name` is set in its constructor and survives the package
4055
+ * boundary. Deliberately NOT a message match: the message wording belongs to
4056
+ * CM and can be reworded without warning.
4057
+ */
4058
+ classifyInferenceError(err) {
4059
+ if (err instanceof MembraneError) {
4060
+ return { retryable: err.retryable, errorType: err.type };
4061
+ }
4062
+ if (err.name === 'OverBudgetError') {
4063
+ return { errorType: 'over_budget' };
4064
+ }
4065
+ return {};
4066
+ }
3982
4067
  noteInferenceExhausted(agentName, reason, retryable, errorType) {
3983
4068
  const streak = (this.consecutiveInferenceFailures.get(agentName) ?? 0) + 1;
3984
4069
  this.consecutiveInferenceFailures.set(agentName, streak);
@@ -4006,6 +4091,42 @@ export class AgentFramework {
4006
4091
  console.error(`[inference-failed] could not record chronicle marker for ${agentName}:`, err);
4007
4092
  }
4008
4093
  }
4094
+ // (2b) OverBudget deadlock breaker. When compile fails with
4095
+ // OverBudgetError, the normal compression drain never runs: it is driven
4096
+ // by successful activity, which the over-budget state prevents - a closed
4097
+ // loop with no internal exit. Field data (2026-07-10, resident agent):
4098
+ // 36 minutes hard-down, zero self-rescue; only an operator raising the
4099
+ // budget externally could break the loop. Break it here instead: kick the
4100
+ // strategy drain directly so folding/merging frees space for the next
4101
+ // compile. Bounded ticks, best-effort, never throws, one kick per agent
4102
+ // at a time (see overBudgetDrainInFlight).
4103
+ //
4104
+ // Gates on the classified errorType (see classifyInferenceError); the
4105
+ // message match is only a fallback for paths that lost the Error object
4106
+ // (e.g. a reason string that crossed a serialization boundary). It matches
4107
+ // CM's current OverBudgetError wording and MAY rot if CM rewords it — the
4108
+ // errorType gate is the one that's load-bearing.
4109
+ const overBudget = errorType === 'over_budget' || /exceed hard budget/i.test(reason);
4110
+ if (agent && overBudget && !this.overBudgetDrainInFlight.has(agentName)) {
4111
+ this.overBudgetDrainInFlight.add(agentName);
4112
+ void (async () => {
4113
+ let ticks = 0;
4114
+ try {
4115
+ const cm = agent.getContextManager();
4116
+ while (ticks < 8) {
4117
+ await cm.tick();
4118
+ ticks++;
4119
+ }
4120
+ console.error(`[inference-failed] drain kicked for ${agentName} (OverBudget breaker, ${ticks} ticks)`);
4121
+ }
4122
+ catch (err) {
4123
+ console.error(`[inference-failed] drain kick failed for ${agentName} after ${ticks} ticks:`, err);
4124
+ }
4125
+ finally {
4126
+ this.overBudgetDrainInFlight.delete(agentName);
4127
+ }
4128
+ })();
4129
+ }
4009
4130
  // (3) Hard-down escalation on repeated identical failure.
4010
4131
  if (streak >= this.inferenceFailureEscalationThreshold) {
4011
4132
  console.error(`[inference-hard-down] agent=${agentName} has FAILED ${streak} consecutive ` +