@animalabs/agent-framework 0.6.4 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/src/agent.d.ts.map +1 -1
- package/dist/src/agent.js +20 -0
- package/dist/src/agent.js.map +1 -1
- package/dist/src/framework.d.ts +44 -1
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +264 -143
- package/dist/src/framework.js.map +1 -1
- package/dist/src/types/framework.d.ts +9 -0
- package/dist/src/types/framework.d.ts.map +1 -1
- package/dist/test/activation-request-sanitize.test.d.ts +2 -0
- package/dist/test/activation-request-sanitize.test.d.ts.map +1 -0
- package/dist/test/activation-request-sanitize.test.js +89 -0
- package/dist/test/activation-request-sanitize.test.js.map +1 -0
- package/dist/test/ephemeral-settle.test.d.ts +2 -0
- package/dist/test/ephemeral-settle.test.d.ts.map +1 -0
- package/dist/test/ephemeral-settle.test.js +351 -0
- package/dist/test/ephemeral-settle.test.js.map +1 -0
- package/dist/test/maintenance-timer.test.js +49 -0
- package/dist/test/maintenance-timer.test.js.map +1 -1
- package/dist/test/overbudget-drain-breaker.test.d.ts +2 -0
- package/dist/test/overbudget-drain-breaker.test.d.ts.map +1 -0
- package/dist/test/overbudget-drain-breaker.test.js +157 -0
- package/dist/test/overbudget-drain-breaker.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/agent.ts +22 -0
- package/src/framework.ts +295 -143
- package/src/types/framework.ts +10 -0
package/dist/src/framework.js
CHANGED
|
@@ -147,6 +147,17 @@ export class AgentFramework {
|
|
|
147
147
|
processLoggingBroadcast;
|
|
148
148
|
activeStreams = new Map();
|
|
149
149
|
pendingAssistantBlocks = new Map();
|
|
150
|
+
/** Streams the FRAMEWORK cancelled for non-terminal reasons, keyed
|
|
151
|
+
* `${agentName}:${streamId}`: an endTurn tool result or a context-budget
|
|
152
|
+
* restart. The membrane still delivers an `aborted` event for these, and
|
|
153
|
+
* without the marker the abort handler would treat it as a terminal
|
|
154
|
+
* failure — settling ephemerals with a rejection and emitting a spurious
|
|
155
|
+
* `inference:exhausted` (which also pollutes the failure streak). Kept
|
|
156
|
+
* separate from ephemeralRuns deliberately: endTurn/budget cancels happen
|
|
157
|
+
* for resident agents too, and the key is per-stream, not per-agent. */
|
|
158
|
+
frameworkCancelledStreams = new Map();
|
|
159
|
+
/** Active runEphemeralToCompletion runs, keyed by agent name. */
|
|
160
|
+
ephemeralRuns = new Map();
|
|
150
161
|
/** Per-agent count of consecutive exhausted inferences (reset on any success).
|
|
151
162
|
* Drives hard-down escalation — see noteInferenceExhausted. */
|
|
152
163
|
consecutiveInferenceFailures = new Map();
|
|
@@ -175,6 +186,13 @@ export class AgentFramework {
|
|
|
175
186
|
* Bounds the automatic quarantine loop the same way refusalRewinds bounds
|
|
176
187
|
* the refusal loop, so the breaker can never shed the whole history. */
|
|
177
188
|
exhaustionRewinds = new Map();
|
|
189
|
+
/** Agents with an OverBudget drain kick currently in flight. The breaker in
|
|
190
|
+
* noteInferenceExhausted fires on EVERY matching failure and the scenario it
|
|
191
|
+
* exists for is "every activation fails, repeatedly" — without this guard,
|
|
192
|
+
* overlapping kicks race the strategy's own pendingCompression gate and the
|
|
193
|
+
* tick counts in the success log stop meaning anything. One kick per agent
|
|
194
|
+
* at a time; cleared when the kick settles (success or failure). */
|
|
195
|
+
overBudgetDrainInFlight = new Set();
|
|
178
196
|
/** Per-agent current rewind episode: the single consolidated marker's id and
|
|
179
197
|
* how many turns have been shed so far. One marker per episode, updated in
|
|
180
198
|
* place (see updateRewindMarker); cleared when the episode ends. */
|
|
@@ -537,7 +555,9 @@ export class AgentFramework {
|
|
|
537
555
|
if (this.maintenancePass || !this.running)
|
|
538
556
|
return;
|
|
539
557
|
const pass = this.runQueuedMaintenance().catch((error) => {
|
|
540
|
-
|
|
558
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
559
|
+
console.error('[context-maintenance] pass failed:', reason);
|
|
560
|
+
this.opsAlert('context-maintenance-failed', 'framework', reason, { data: { scope: 'pass' } });
|
|
541
561
|
});
|
|
542
562
|
this.maintenancePass = pass;
|
|
543
563
|
void pass.finally(() => {
|
|
@@ -588,6 +608,13 @@ export class AgentFramework {
|
|
|
588
608
|
catch (error) {
|
|
589
609
|
record.error = error instanceof Error ? error.message : String(error);
|
|
590
610
|
console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
|
|
611
|
+
this.opsAlert('context-maintenance-failed', agent.name, record.error, {
|
|
612
|
+
data: {
|
|
613
|
+
scope: 'agent',
|
|
614
|
+
...(record.pendingBefore ? { pending: record.pendingBefore } : {}),
|
|
615
|
+
...(record.progressBefore ? { progress: record.progressBefore } : {}),
|
|
616
|
+
},
|
|
617
|
+
});
|
|
591
618
|
}
|
|
592
619
|
finally {
|
|
593
620
|
const pending = cm.getPendingWork()?.description;
|
|
@@ -934,6 +961,50 @@ export class AgentFramework {
|
|
|
934
961
|
};
|
|
935
962
|
return { agent, contextManager, cleanup };
|
|
936
963
|
}
|
|
964
|
+
createDeferred() {
|
|
965
|
+
let resolve;
|
|
966
|
+
let reject;
|
|
967
|
+
const promise = new Promise((res, rej) => {
|
|
968
|
+
resolve = res;
|
|
969
|
+
reject = rej;
|
|
970
|
+
});
|
|
971
|
+
return { promise, resolve, reject };
|
|
972
|
+
}
|
|
973
|
+
/** Liveness ping for an active ephemeral run (no-op otherwise). */
|
|
974
|
+
touchEphemeralRun(agentName, inferenceStarted = false) {
|
|
975
|
+
const run = this.ephemeralRuns.get(agentName);
|
|
976
|
+
if (!run)
|
|
977
|
+
return;
|
|
978
|
+
if (inferenceStarted) {
|
|
979
|
+
run.inferenceStarted = true;
|
|
980
|
+
}
|
|
981
|
+
run.lastActivity = Date.now();
|
|
982
|
+
}
|
|
983
|
+
recordEphemeralToolCalls(agentName, count) {
|
|
984
|
+
const run = this.ephemeralRuns.get(agentName);
|
|
985
|
+
if (!run)
|
|
986
|
+
return;
|
|
987
|
+
run.toolCallsCount += count;
|
|
988
|
+
run.lastActivity = Date.now();
|
|
989
|
+
}
|
|
990
|
+
/** Settle an active ephemeral run (no-op for resident agents). The tool-call
|
|
991
|
+
* count comes from the run's own counter — the single source of truth,
|
|
992
|
+
* which unlike a per-stream local survives budget restarts. */
|
|
993
|
+
settleAgent(agentName, outcome) {
|
|
994
|
+
const run = this.ephemeralRuns.get(agentName);
|
|
995
|
+
if (!run)
|
|
996
|
+
return;
|
|
997
|
+
run.lastActivity = Date.now();
|
|
998
|
+
if (outcome.stopReason === 'exhausted') {
|
|
999
|
+
run.settle.reject(new Error(outcome.error ?? 'Unknown error'));
|
|
1000
|
+
return;
|
|
1001
|
+
}
|
|
1002
|
+
run.settle.resolve({
|
|
1003
|
+
stopReason: outcome.stopReason,
|
|
1004
|
+
speech: outcome.speech,
|
|
1005
|
+
toolCallsCount: run.toolCallsCount,
|
|
1006
|
+
});
|
|
1007
|
+
}
|
|
937
1008
|
/**
|
|
938
1009
|
* Run an ephemeral agent to completion through the framework's event loop.
|
|
939
1010
|
*
|
|
@@ -944,139 +1015,73 @@ export class AgentFramework {
|
|
|
944
1015
|
* The caller provides a pre-created agent + contextManager (from createEphemeralAgent).
|
|
945
1016
|
* The task message should already be in the context manager.
|
|
946
1017
|
*/
|
|
947
|
-
async runEphemeralToCompletion(agent, contextManager) {
|
|
1018
|
+
async runEphemeralToCompletion(agent, contextManager, watchdogs) {
|
|
948
1019
|
// Register temporarily so the event loop can drive it
|
|
949
1020
|
this.agents.set(agent.name, agent);
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
clearTimeout(startupWatchdog);
|
|
969
|
-
clearInterval(completionWatchdog);
|
|
970
|
-
this.offTrace(traceListener);
|
|
971
|
-
this.agents.delete(agent.name);
|
|
972
|
-
this.evictTurnCheckpoints(agent.name);
|
|
973
|
-
};
|
|
974
|
-
// Watchdog: if inference never starts within 30s, the agent is a zombie.
|
|
975
|
-
// This catches cases where the event loop stalled, the agent was
|
|
976
|
-
// deregistered, or processInferenceRequests() skipped the request.
|
|
977
|
-
const startupWatchdog = setTimeout(() => {
|
|
978
|
-
if (!inferenceStarted && !settled) {
|
|
979
|
-
cleanup();
|
|
1021
|
+
const run = {
|
|
1022
|
+
settle: this.createDeferred(),
|
|
1023
|
+
inferenceStarted: false,
|
|
1024
|
+
lastActivity: Date.now(),
|
|
1025
|
+
toolCallsCount: 0,
|
|
1026
|
+
};
|
|
1027
|
+
this.ephemeralRuns.set(agent.name, run);
|
|
1028
|
+
const STARTUP_TIMEOUT_MS = watchdogs?.startupTimeoutMs ?? 30_000;
|
|
1029
|
+
// After inference has started, give it 15 minutes of activity-bounded
|
|
1030
|
+
// life. The stream driver refreshes the deadline as it makes progress;
|
|
1031
|
+
// sustained silence trips it.
|
|
1032
|
+
const COMPLETION_IDLE_TIMEOUT_MS = watchdogs?.idleTimeoutMs ?? 15 * 60_000;
|
|
1033
|
+
const IDLE_POLL_MS = watchdogs?.idlePollMs ?? 30_000;
|
|
1034
|
+
let startupWatchdog = null;
|
|
1035
|
+
let completionWatchdog = null;
|
|
1036
|
+
const startupTimeout = new Promise((_, reject) => {
|
|
1037
|
+
startupWatchdog = setTimeout(() => {
|
|
1038
|
+
if (!run.inferenceStarted) {
|
|
980
1039
|
reject(new Error(`Ephemeral agent "${agent.name}" failed to start inference within ` +
|
|
981
1040
|
`${STARTUP_TIMEOUT_MS}ms — zombie detected. The event loop may have ` +
|
|
982
1041
|
`stalled or the inference request was dropped.`));
|
|
983
1042
|
}
|
|
984
1043
|
}, STARTUP_TIMEOUT_MS);
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
// and keeps its concurrency slot pinned.
|
|
990
|
-
const completionWatchdog = setInterval(() => {
|
|
991
|
-
if (settled || !inferenceStarted)
|
|
1044
|
+
});
|
|
1045
|
+
const completionIdleTimeout = new Promise((_, reject) => {
|
|
1046
|
+
completionWatchdog = setInterval(() => {
|
|
1047
|
+
if (!run.inferenceStarted)
|
|
992
1048
|
return;
|
|
993
|
-
const idle = Date.now() - lastActivity;
|
|
1049
|
+
const idle = Date.now() - run.lastActivity;
|
|
994
1050
|
if (idle > COMPLETION_IDLE_TIMEOUT_MS) {
|
|
995
|
-
|
|
996
|
-
reject(new Error(`Ephemeral agent "${agent.name}" stalled: no trace activity for ` +
|
|
1051
|
+
reject(new Error(`Ephemeral agent "${agent.name}" stalled: no stream-driver activity for ` +
|
|
997
1052
|
`${Math.round(idle / 1000)}s after inference started (threshold ` +
|
|
998
1053
|
`${Math.round(COMPLETION_IDLE_TIMEOUT_MS / 1000)}s). Stream likely ` +
|
|
999
1054
|
`dropped or terminal event was lost.`));
|
|
1000
1055
|
}
|
|
1001
|
-
},
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
// Only track events for our ephemeral agent
|
|
1006
|
-
const agentName = 'agentName' in event ? event.agentName : null;
|
|
1007
|
-
if (agentName !== agent.name)
|
|
1008
|
-
return;
|
|
1009
|
-
// Any addressed event counts as liveness for the completion watchdog.
|
|
1010
|
-
lastActivity = Date.now();
|
|
1011
|
-
switch (event.type) {
|
|
1012
|
-
case 'inference:started':
|
|
1013
|
-
inferenceStarted = true;
|
|
1014
|
-
break;
|
|
1015
|
-
case 'inference:tokens': {
|
|
1016
|
-
inferenceStarted = true;
|
|
1017
|
-
const content = event.content;
|
|
1018
|
-
if (content)
|
|
1019
|
-
speech += content;
|
|
1020
|
-
break;
|
|
1021
|
-
}
|
|
1022
|
-
case 'inference:tool_calls_yielded': {
|
|
1023
|
-
const calls = event.calls;
|
|
1024
|
-
if (calls)
|
|
1025
|
-
toolCallsCount += calls.length;
|
|
1026
|
-
// Reset speech buffer — tool calls break the text
|
|
1027
|
-
speech = '';
|
|
1028
|
-
break;
|
|
1029
|
-
}
|
|
1030
|
-
case 'inference:stream_resumed':
|
|
1031
|
-
speech = '';
|
|
1032
|
-
break;
|
|
1033
|
-
case 'inference:completed': {
|
|
1034
|
-
// `inference:completed` is terminal — driveStream only emits it
|
|
1035
|
-
// from `case 'complete'`, which fires after the full tool
|
|
1036
|
-
// round-trip has finished and the model has stopped. Intermediate
|
|
1037
|
-
// tool-cycle transitions use `inference:stream_resumed`, not
|
|
1038
|
-
// `inference:completed`. So we resolve unconditionally here.
|
|
1039
|
-
//
|
|
1040
|
-
// Previously this branch gated on `agent.state.status === 'idle'`,
|
|
1041
|
-
// which was correct in intent but order-fragile: the framework
|
|
1042
|
-
// emitted `inference:completed` BEFORE calling `agent.reset()`,
|
|
1043
|
-
// so synchronous listeners observed status === 'streaming' at
|
|
1044
|
-
// the trace boundary. The gate failed silently and the promise
|
|
1045
|
-
// hung forever — every production zombie subagent originated
|
|
1046
|
-
// here. The reset+emit order has been fixed in driveStream's
|
|
1047
|
-
// `case 'complete'` handler; the gate is dropped here as
|
|
1048
|
-
// belt-and-suspenders.
|
|
1049
|
-
cleanup();
|
|
1050
|
-
resolve({ speech, toolCallsCount });
|
|
1051
|
-
break;
|
|
1052
|
-
}
|
|
1053
|
-
case 'inference:turn_ended': {
|
|
1054
|
-
// endTurn from a tool result — agent is done
|
|
1055
|
-
cleanup();
|
|
1056
|
-
resolve({ speech, toolCallsCount });
|
|
1057
|
-
break;
|
|
1058
|
-
}
|
|
1059
|
-
case 'inference:exhausted': {
|
|
1060
|
-
// Retries exhausted — reject only after the error policy gives up.
|
|
1061
|
-
// Earlier inference:failed events are left alone so the framework
|
|
1062
|
-
// retry path in driveStream/startAgentStream can restart the stream
|
|
1063
|
-
// while this listener stays alive.
|
|
1064
|
-
const error = event.error ?? 'Unknown error';
|
|
1065
|
-
cleanup();
|
|
1066
|
-
reject(new Error(error));
|
|
1067
|
-
break;
|
|
1068
|
-
}
|
|
1069
|
-
}
|
|
1070
|
-
};
|
|
1071
|
-
this.onTrace(traceListener);
|
|
1072
|
-
// Trigger inference
|
|
1056
|
+
}, IDLE_POLL_MS);
|
|
1057
|
+
});
|
|
1058
|
+
try {
|
|
1059
|
+
// Trigger inference after the settle promise is registered.
|
|
1073
1060
|
this.pendingRequests.push({
|
|
1074
1061
|
agentName: agent.name,
|
|
1075
1062
|
reason: 'ephemeral',
|
|
1076
1063
|
source: 'subagent',
|
|
1077
1064
|
timestamp: Date.now(),
|
|
1078
1065
|
});
|
|
1079
|
-
|
|
1066
|
+
const result = await Promise.race([
|
|
1067
|
+
run.settle.promise,
|
|
1068
|
+
startupTimeout,
|
|
1069
|
+
completionIdleTimeout,
|
|
1070
|
+
]);
|
|
1071
|
+
return { speech: result.speech, toolCallsCount: result.toolCallsCount };
|
|
1072
|
+
}
|
|
1073
|
+
finally {
|
|
1074
|
+
if (startupWatchdog)
|
|
1075
|
+
clearTimeout(startupWatchdog);
|
|
1076
|
+
if (completionWatchdog)
|
|
1077
|
+
clearInterval(completionWatchdog);
|
|
1078
|
+
this.ephemeralRuns.delete(agent.name);
|
|
1079
|
+
this.agents.delete(agent.name);
|
|
1080
|
+
// Spawn-and-dispose bookkeeping (main, d453165/fee96a7): without this,
|
|
1081
|
+
// ephemeral agents leave checkpoint-tree keys and diagnostics map
|
|
1082
|
+
// entries behind for the life of the store/session.
|
|
1083
|
+
this.evictTurnCheckpoints(agent.name);
|
|
1084
|
+
}
|
|
1080
1085
|
}
|
|
1081
1086
|
/**
|
|
1082
1087
|
* Get queue depth.
|
|
@@ -2196,6 +2201,9 @@ export class AgentFramework {
|
|
|
2196
2201
|
// messages between tool_use and tool_result, causing a 400 error.
|
|
2197
2202
|
if (event.type === 'tool-result') {
|
|
2198
2203
|
const agent = this.agents.get(event.agentName);
|
|
2204
|
+
if (agent) {
|
|
2205
|
+
this.touchEphemeralRun(event.agentName, true);
|
|
2206
|
+
}
|
|
2199
2207
|
if (!agent) {
|
|
2200
2208
|
console.warn(`[framework] Dropping tool-result for unknown agent '${event.agentName}' (callId=${event.callId}). ` +
|
|
2201
2209
|
`Agent may have been destroyed while tool was executing.`);
|
|
@@ -2260,13 +2268,29 @@ export class AgentFramework {
|
|
|
2260
2268
|
if (shouldEndTurn) {
|
|
2261
2269
|
// endTurn: messages already stored above, cancel stream, reset to idle.
|
|
2262
2270
|
if (currentState.stream) {
|
|
2263
|
-
agent.
|
|
2271
|
+
this.frameworkCancelledStreams.set(`${agent.name}:${agent.streamId}`, 'turn_ended');
|
|
2272
|
+
currentState.stream.cancel();
|
|
2264
2273
|
}
|
|
2265
2274
|
agent.reset();
|
|
2275
|
+
// The turn is over — release the gate here, not just in the abort
|
|
2276
|
+
// handler: the membrane's aborted event is asynchronous and the
|
|
2277
|
+
// agent must be wakeable the moment it goes idle. (Idempotent;
|
|
2278
|
+
// driveStream's finally is the backstop.)
|
|
2279
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
2280
|
+
this.settleAgent(agent.name, { stopReason: 'turn_ended', speech: '' });
|
|
2266
2281
|
this.emitTrace({ type: 'inference:turn_ended', agentName: agent.name });
|
|
2267
2282
|
}
|
|
2268
2283
|
else if (overBudget) {
|
|
2269
|
-
// Context budget exceeded: break the stream, let compile() compress
|
|
2284
|
+
// Context budget exceeded: break the stream, let compile() compress.
|
|
2285
|
+
// Mark the cancel as framework-initiated BEFORE cancelling: the
|
|
2286
|
+
// membrane delivers `aborted` before the restart bumps streamId,
|
|
2287
|
+
// and without the marker the abort handler treats the restart as
|
|
2288
|
+
// a terminal failure — rejecting an ephemeral's promise mid-run
|
|
2289
|
+
// and emitting a spurious inference:exhausted (same race shape as
|
|
2290
|
+
// endTurn above).
|
|
2291
|
+
if (currentState.stream) {
|
|
2292
|
+
this.frameworkCancelledStreams.set(`${agent.name}:${agent.streamId}`, 'budget_restart');
|
|
2293
|
+
}
|
|
2270
2294
|
agent.cancelStream();
|
|
2271
2295
|
this.emitTrace({
|
|
2272
2296
|
type: 'inference:stream_restarted',
|
|
@@ -2825,6 +2849,7 @@ export class AgentFramework {
|
|
|
2825
2849
|
else {
|
|
2826
2850
|
this.activeTriggerChannels.delete(agent.name);
|
|
2827
2851
|
}
|
|
2852
|
+
this.touchEphemeralRun(agent.name, true);
|
|
2828
2853
|
this.emitTrace({ type: 'inference:started', agentName: agent.name });
|
|
2829
2854
|
this.eventGate?.onInferenceStarted(agent.name);
|
|
2830
2855
|
this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), startedAt: Date.now() });
|
|
@@ -2880,17 +2905,23 @@ export class AgentFramework {
|
|
|
2880
2905
|
await this.startAgentStream(agent, trigger, attempt + 1);
|
|
2881
2906
|
}
|
|
2882
2907
|
else {
|
|
2908
|
+
this.settleAgent(agent.name, {
|
|
2909
|
+
stopReason: 'exhausted',
|
|
2910
|
+
speech: '',
|
|
2911
|
+
error: err.message,
|
|
2912
|
+
});
|
|
2883
2913
|
this.emitTrace({
|
|
2884
2914
|
type: 'inference:exhausted',
|
|
2885
2915
|
agentName: agent.name,
|
|
2886
2916
|
error: err.message,
|
|
2887
|
-
// Drives the poison-history breaker
|
|
2888
|
-
//
|
|
2889
|
-
//
|
|
2890
|
-
//
|
|
2891
|
-
//
|
|
2892
|
-
|
|
2893
|
-
|
|
2917
|
+
// Drives the poison-history breaker (`invalid_request`: a 400-class
|
|
2918
|
+
// rejection of the history itself — retrying the same context can
|
|
2919
|
+
// never succeed; auth/abort/context_length are also non-retryable
|
|
2920
|
+
// but must NOT cost history) and the OverBudget drain breaker
|
|
2921
|
+
// (`over_budget`: compile refused to fit the hard budget — this is
|
|
2922
|
+
// the site that sees it, since compile runs before the stream
|
|
2923
|
+
// exists). `retryable` is kept for observability.
|
|
2924
|
+
...this.classifyInferenceError(err),
|
|
2894
2925
|
});
|
|
2895
2926
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
2896
2927
|
if (action.emit) {
|
|
@@ -2913,6 +2944,7 @@ export class AgentFramework {
|
|
|
2913
2944
|
this.channelRegistry.startTyping(typingChannel);
|
|
2914
2945
|
try {
|
|
2915
2946
|
for await (const event of stream) {
|
|
2947
|
+
this.touchEphemeralRun(agent.name, true);
|
|
2916
2948
|
switch (event.type) {
|
|
2917
2949
|
case 'tokens':
|
|
2918
2950
|
this.emitTrace({
|
|
@@ -2936,6 +2968,7 @@ export class AgentFramework {
|
|
|
2936
2968
|
}
|
|
2937
2969
|
case 'tool-calls': {
|
|
2938
2970
|
hadToolCalls = true;
|
|
2971
|
+
this.recordEphemeralToolCalls(agent.name, event.calls.length);
|
|
2939
2972
|
this.emitTrace({
|
|
2940
2973
|
type: 'inference:tool_calls_yielded',
|
|
2941
2974
|
agentName: agent.name,
|
|
@@ -3021,14 +3054,16 @@ export class AgentFramework {
|
|
|
3021
3054
|
// thinking blocks from their tool_use turn (the API requires
|
|
3022
3055
|
// thinking to precede tool_use in the same assistant turn).
|
|
3023
3056
|
const lastToolIdx = response.content.reduce((last, b, i) => b.type === 'tool_use' || b.type === 'tool_result' ? i : last, -1);
|
|
3057
|
+
const terminalContent = lastToolIdx >= 0
|
|
3058
|
+
? response.content.slice(lastToolIdx + 1)
|
|
3059
|
+
: response.content;
|
|
3024
3060
|
if (lastToolIdx >= 0) {
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
agent.addAssistantResponse(trailingContent);
|
|
3061
|
+
if (terminalContent.length > 0) {
|
|
3062
|
+
agent.addAssistantResponse(terminalContent);
|
|
3028
3063
|
}
|
|
3029
3064
|
}
|
|
3030
3065
|
else {
|
|
3031
|
-
agent.addAssistantResponse(
|
|
3066
|
+
agent.addAssistantResponse(terminalContent);
|
|
3032
3067
|
}
|
|
3033
3068
|
// Run afterInference hooks (no-op if no MCPL servers)
|
|
3034
3069
|
if (this.hookOrchestrator) {
|
|
@@ -3080,17 +3115,20 @@ export class AgentFramework {
|
|
|
3080
3115
|
cacheRead: du.cacheReadTokens,
|
|
3081
3116
|
}
|
|
3082
3117
|
: undefined;
|
|
3083
|
-
// Reset agent state
|
|
3084
|
-
// synchronous listeners
|
|
3085
|
-
//
|
|
3086
|
-
//
|
|
3087
|
-
//
|
|
3088
|
-
// trace boundary, failed its idle gate, and the promise hung
|
|
3089
|
-
// forever — every "zombie subagent" in production tracked back
|
|
3090
|
-
// here. Speech dispatch happens after but doesn't depend on the
|
|
3091
|
-
// status field.
|
|
3118
|
+
// Reset agent state before emitting inference:completed. Traces are
|
|
3119
|
+
// observability-only, but external synchronous listeners should
|
|
3120
|
+
// still see the terminal state at the terminal trace boundary.
|
|
3121
|
+
// Speech dispatch happens after but doesn't depend on the status
|
|
3122
|
+
// field.
|
|
3092
3123
|
agent.reset();
|
|
3093
3124
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
3125
|
+
this.settleAgent(agent.name, {
|
|
3126
|
+
stopReason: 'completed',
|
|
3127
|
+
speech: terminalContent
|
|
3128
|
+
.filter((block) => block.type === 'text')
|
|
3129
|
+
.map((block) => block.text)
|
|
3130
|
+
.join('\n'),
|
|
3131
|
+
});
|
|
3094
3132
|
this.emitTrace({
|
|
3095
3133
|
type: 'inference:completed',
|
|
3096
3134
|
agentName: agent.name,
|
|
@@ -3301,9 +3339,8 @@ export class AgentFramework {
|
|
|
3301
3339
|
}
|
|
3302
3340
|
}
|
|
3303
3341
|
}
|
|
3304
|
-
// NOTE: agent.reset() + onInferenceEnded() already ran above,
|
|
3305
|
-
// dispatchSpeech
|
|
3306
|
-
// must see status === 'idle'). Locus routing is speech dispatch and
|
|
3342
|
+
// NOTE: agent.reset() + onInferenceEnded() already ran above,
|
|
3343
|
+
// BEFORE dispatchSpeech. Locus routing is speech dispatch and
|
|
3307
3344
|
// doesn't depend on the status field, so it correctly runs after.
|
|
3308
3345
|
break;
|
|
3309
3346
|
}
|
|
@@ -3335,16 +3372,20 @@ export class AgentFramework {
|
|
|
3335
3372
|
await this.startAgentStream(agent, trigger, attempt + 1);
|
|
3336
3373
|
}
|
|
3337
3374
|
else {
|
|
3375
|
+
this.settleAgent(agent.name, {
|
|
3376
|
+
stopReason: 'exhausted',
|
|
3377
|
+
speech: '',
|
|
3378
|
+
error: err.message,
|
|
3379
|
+
});
|
|
3338
3380
|
this.emitTrace({
|
|
3339
3381
|
type: 'inference:exhausted',
|
|
3340
3382
|
agentName: agent.name,
|
|
3341
3383
|
error: err.message,
|
|
3342
3384
|
// Drives the poison-history breaker. `retryable` is kept for
|
|
3343
|
-
// observability, but the
|
|
3385
|
+
// observability, but the breakers gate on `errorType` — only an
|
|
3344
3386
|
// `invalid_request` (a 400-class rejection of the history
|
|
3345
3387
|
// itself) means retrying the same context can never succeed.
|
|
3346
|
-
|
|
3347
|
-
errorType: err instanceof MembraneError ? err.type : undefined,
|
|
3388
|
+
...this.classifyInferenceError(err),
|
|
3348
3389
|
});
|
|
3349
3390
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
3350
3391
|
if (action.emit) {
|
|
@@ -3354,12 +3395,28 @@ export class AgentFramework {
|
|
|
3354
3395
|
break;
|
|
3355
3396
|
}
|
|
3356
3397
|
case 'aborted': {
|
|
3398
|
+
// Framework-initiated, non-terminal cancels (endTurn tool result,
|
|
3399
|
+
// context-budget restart) also surface here as `aborted` — but
|
|
3400
|
+
// the turn either already settled (endTurn) or a replacement
|
|
3401
|
+
// stream is queued (budget restart). Neither is a failure: no
|
|
3402
|
+
// reset, no settle, no spurious inference:exhausted (which would
|
|
3403
|
+
// reject an ephemeral's promise mid-run and bump the failure
|
|
3404
|
+
// streak). Gate release + stream teardown happen in `finally`.
|
|
3405
|
+
if (this.frameworkCancelledStreams.delete(`${agent.name}:${myStreamId}`)) {
|
|
3406
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
3407
|
+
return;
|
|
3408
|
+
}
|
|
3357
3409
|
const reason = event.reason ?? 'unknown';
|
|
3358
3410
|
// Only reset if this is still the active stream (a budget restart
|
|
3359
3411
|
// may have already started a new stream, bumping streamId)
|
|
3360
3412
|
if (agent.streamId === myStreamId) {
|
|
3361
3413
|
const durationMs = Date.now() - startTime;
|
|
3362
3414
|
agent.reset();
|
|
3415
|
+
this.settleAgent(agent.name, {
|
|
3416
|
+
stopReason: 'exhausted',
|
|
3417
|
+
speech: '',
|
|
3418
|
+
error: `Stream aborted: ${reason}`,
|
|
3419
|
+
});
|
|
3363
3420
|
this.emitTrace({
|
|
3364
3421
|
type: 'inference:exhausted',
|
|
3365
3422
|
agentName: agent.name,
|
|
@@ -3429,10 +3486,16 @@ export class AgentFramework {
|
|
|
3429
3486
|
error: err.message,
|
|
3430
3487
|
stack: err.stack,
|
|
3431
3488
|
});
|
|
3489
|
+
this.settleAgent(agent.name, {
|
|
3490
|
+
stopReason: 'exhausted',
|
|
3491
|
+
speech: '',
|
|
3492
|
+
error: err.message,
|
|
3493
|
+
});
|
|
3432
3494
|
this.emitTrace({
|
|
3433
3495
|
type: 'inference:exhausted',
|
|
3434
3496
|
agentName: agent.name,
|
|
3435
3497
|
error: err.message,
|
|
3498
|
+
...this.classifyInferenceError(err),
|
|
3436
3499
|
});
|
|
3437
3500
|
// Postmortem 2026-05-28 P2 #7: catch-path failures (stream itself
|
|
3438
3501
|
// threw) were previously only visible via in-memory trace listeners.
|
|
@@ -3462,6 +3525,7 @@ export class AgentFramework {
|
|
|
3462
3525
|
// Stop the typing indicator on every exit path (complete, error,
|
|
3463
3526
|
// exhausted, abort) so it never sticks after the turn ends.
|
|
3464
3527
|
this.channelRegistry?.stopTyping();
|
|
3528
|
+
this.frameworkCancelledStreams.delete(`${agent.name}:${myStreamId}`);
|
|
3465
3529
|
this.activeStreams.delete(agent.name);
|
|
3466
3530
|
this.pendingAssistantBlocks.delete(agent.name);
|
|
3467
3531
|
// A conversation fork whose TTL closure turn just finished is done for
|
|
@@ -3979,6 +4043,27 @@ export class AgentFramework {
|
|
|
3979
4043
|
}
|
|
3980
4044
|
return streak;
|
|
3981
4045
|
}
|
|
4046
|
+
/**
|
|
4047
|
+
* Classify a terminal inference error for the `inference:exhausted` trace.
|
|
4048
|
+
* Single-sourced for every emit site: the downstream breakers (poison-history
|
|
4049
|
+
* quarantine, OverBudget drain kick) gate on `errorType`, so classification
|
|
4050
|
+
* drift between sites would silently disable a safety net.
|
|
4051
|
+
*
|
|
4052
|
+
* context-manager's OverBudgetError is recognized by `err.name`: CM does not
|
|
4053
|
+
* export the class from its package root, so a cross-package `instanceof` is
|
|
4054
|
+
* unavailable — but `name` is set in its constructor and survives the package
|
|
4055
|
+
* boundary. Deliberately NOT a message match: the message wording belongs to
|
|
4056
|
+
* CM and can be reworded without warning.
|
|
4057
|
+
*/
|
|
4058
|
+
classifyInferenceError(err) {
|
|
4059
|
+
if (err instanceof MembraneError) {
|
|
4060
|
+
return { retryable: err.retryable, errorType: err.type };
|
|
4061
|
+
}
|
|
4062
|
+
if (err.name === 'OverBudgetError') {
|
|
4063
|
+
return { errorType: 'over_budget' };
|
|
4064
|
+
}
|
|
4065
|
+
return {};
|
|
4066
|
+
}
|
|
3982
4067
|
noteInferenceExhausted(agentName, reason, retryable, errorType) {
|
|
3983
4068
|
const streak = (this.consecutiveInferenceFailures.get(agentName) ?? 0) + 1;
|
|
3984
4069
|
this.consecutiveInferenceFailures.set(agentName, streak);
|
|
@@ -4006,6 +4091,42 @@ export class AgentFramework {
|
|
|
4006
4091
|
console.error(`[inference-failed] could not record chronicle marker for ${agentName}:`, err);
|
|
4007
4092
|
}
|
|
4008
4093
|
}
|
|
4094
|
+
// (2b) OverBudget deadlock breaker. When compile fails with
|
|
4095
|
+
// OverBudgetError, the normal compression drain never runs: it is driven
|
|
4096
|
+
// by successful activity, which the over-budget state prevents - a closed
|
|
4097
|
+
// loop with no internal exit. Field data (2026-07-10, resident agent):
|
|
4098
|
+
// 36 minutes hard-down, zero self-rescue; only an operator raising the
|
|
4099
|
+
// budget externally could break the loop. Break it here instead: kick the
|
|
4100
|
+
// strategy drain directly so folding/merging frees space for the next
|
|
4101
|
+
// compile. Bounded ticks, best-effort, never throws, one kick per agent
|
|
4102
|
+
// at a time (see overBudgetDrainInFlight).
|
|
4103
|
+
//
|
|
4104
|
+
// Gates on the classified errorType (see classifyInferenceError); the
|
|
4105
|
+
// message match is only a fallback for paths that lost the Error object
|
|
4106
|
+
// (e.g. a reason string that crossed a serialization boundary). It matches
|
|
4107
|
+
// CM's current OverBudgetError wording and MAY rot if CM rewords it — the
|
|
4108
|
+
// errorType gate is the one that's load-bearing.
|
|
4109
|
+
const overBudget = errorType === 'over_budget' || /exceed hard budget/i.test(reason);
|
|
4110
|
+
if (agent && overBudget && !this.overBudgetDrainInFlight.has(agentName)) {
|
|
4111
|
+
this.overBudgetDrainInFlight.add(agentName);
|
|
4112
|
+
void (async () => {
|
|
4113
|
+
let ticks = 0;
|
|
4114
|
+
try {
|
|
4115
|
+
const cm = agent.getContextManager();
|
|
4116
|
+
while (ticks < 8) {
|
|
4117
|
+
await cm.tick();
|
|
4118
|
+
ticks++;
|
|
4119
|
+
}
|
|
4120
|
+
console.error(`[inference-failed] drain kicked for ${agentName} (OverBudget breaker, ${ticks} ticks)`);
|
|
4121
|
+
}
|
|
4122
|
+
catch (err) {
|
|
4123
|
+
console.error(`[inference-failed] drain kick failed for ${agentName} after ${ticks} ticks:`, err);
|
|
4124
|
+
}
|
|
4125
|
+
finally {
|
|
4126
|
+
this.overBudgetDrainInFlight.delete(agentName);
|
|
4127
|
+
}
|
|
4128
|
+
})();
|
|
4129
|
+
}
|
|
4009
4130
|
// (3) Hard-down escalation on repeated identical failure.
|
|
4010
4131
|
if (streak >= this.inferenceFailureEscalationThreshold) {
|
|
4011
4132
|
console.error(`[inference-hard-down] agent=${agentName} has FAILED ${streak} consecutive ` +
|