@animalabs/agent-framework 0.6.3 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/src/agent.d.ts +3 -1
- package/dist/src/agent.d.ts.map +1 -1
- package/dist/src/agent.js +27 -2
- package/dist/src/agent.js.map +1 -1
- package/dist/src/framework.d.ts +46 -1
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +280 -146
- package/dist/src/framework.js.map +1 -1
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/modules/discord/index.d.ts +1 -0
- package/dist/src/modules/discord/index.d.ts.map +1 -1
- package/dist/src/modules/discord/index.js +4 -1
- package/dist/src/modules/discord/index.js.map +1 -1
- package/dist/src/modules/discord/types.d.ts +2 -0
- package/dist/src/modules/discord/types.d.ts.map +1 -1
- package/dist/src/modules/health/index.d.ts +2 -0
- package/dist/src/modules/health/index.d.ts.map +1 -1
- package/dist/src/modules/health/index.js +15 -4
- package/dist/src/modules/health/index.js.map +1 -1
- package/dist/src/timezone.d.ts +14 -0
- package/dist/src/timezone.d.ts.map +1 -0
- package/dist/src/timezone.js +74 -0
- package/dist/src/timezone.js.map +1 -0
- package/dist/src/types/agent.d.ts +5 -2
- package/dist/src/types/agent.d.ts.map +1 -1
- package/dist/src/types/framework.d.ts +15 -0
- package/dist/src/types/framework.d.ts.map +1 -1
- package/dist/test/activation-request-sanitize.test.d.ts +2 -0
- package/dist/test/activation-request-sanitize.test.d.ts.map +1 -0
- package/dist/test/activation-request-sanitize.test.js +89 -0
- package/dist/test/activation-request-sanitize.test.js.map +1 -0
- package/dist/test/agent-cache-ttl.test.js +18 -2
- package/dist/test/agent-cache-ttl.test.js.map +1 -1
- package/dist/test/ephemeral-settle.test.d.ts +2 -0
- package/dist/test/ephemeral-settle.test.d.ts.map +1 -0
- package/dist/test/ephemeral-settle.test.js +351 -0
- package/dist/test/ephemeral-settle.test.js.map +1 -0
- package/dist/test/health-snapshot.test.d.ts +2 -0
- package/dist/test/health-snapshot.test.d.ts.map +1 -0
- package/dist/test/health-snapshot.test.js +74 -0
- package/dist/test/health-snapshot.test.js.map +1 -0
- package/dist/test/maintenance-timer.test.js +49 -0
- package/dist/test/maintenance-timer.test.js.map +1 -1
- package/dist/test/overbudget-drain-breaker.test.d.ts +2 -0
- package/dist/test/overbudget-drain-breaker.test.d.ts.map +1 -0
- package/dist/test/overbudget-drain-breaker.test.js +157 -0
- package/dist/test/overbudget-drain-breaker.test.js.map +1 -0
- package/dist/test/timezone.test.d.ts +2 -0
- package/dist/test/timezone.test.d.ts.map +1 -0
- package/dist/test/timezone.test.js +18 -0
- package/dist/test/timezone.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/agent.ts +30 -3
- package/src/framework.ts +313 -146
- package/src/index.ts +1 -0
- package/src/modules/discord/index.ts +4 -1
- package/src/modules/discord/types.ts +3 -0
- package/src/modules/health/index.ts +20 -4
- package/src/timezone.ts +90 -0
- package/src/types/agent.ts +6 -2
- package/src/types/framework.ts +17 -0
package/dist/src/framework.js
CHANGED
|
@@ -21,6 +21,7 @@ import { CheckpointManager } from './mcpl/checkpoint-manager.js';
|
|
|
21
21
|
import { isToolAllowed } from './mcpl/tool-policy.js';
|
|
22
22
|
import { EventGate } from './gate/event-gate.js';
|
|
23
23
|
import { UsageTracker } from './usage/usage-tracker.js';
|
|
24
|
+
import { formatZonedDateTime, resolveTimeZone } from './timezone.js';
|
|
24
25
|
const FRAMEWORK_STATE_ID = 'framework/state';
|
|
25
26
|
const CONVERSATION_ROUTER_STATE_ID = 'framework/conversation-router';
|
|
26
27
|
const INFERENCE_LOG_ID = 'framework/inference-log';
|
|
@@ -146,6 +147,17 @@ export class AgentFramework {
|
|
|
146
147
|
processLoggingBroadcast;
|
|
147
148
|
activeStreams = new Map();
|
|
148
149
|
pendingAssistantBlocks = new Map();
|
|
150
|
+
/** Streams the FRAMEWORK cancelled for non-terminal reasons, keyed
|
|
151
|
+
* `${agentName}:${streamId}`: an endTurn tool result or a context-budget
|
|
152
|
+
* restart. The membrane still delivers an `aborted` event for these, and
|
|
153
|
+
* without the marker the abort handler would treat it as a terminal
|
|
154
|
+
* failure — settling ephemerals with a rejection and emitting a spurious
|
|
155
|
+
* `inference:exhausted` (which also pollutes the failure streak). Kept
|
|
156
|
+
* separate from ephemeralRuns deliberately: endTurn/budget cancels happen
|
|
157
|
+
* for resident agents too, and the key is per-stream, not per-agent. */
|
|
158
|
+
frameworkCancelledStreams = new Map();
|
|
159
|
+
/** Active runEphemeralToCompletion runs, keyed by agent name. */
|
|
160
|
+
ephemeralRuns = new Map();
|
|
149
161
|
/** Per-agent count of consecutive exhausted inferences (reset on any success).
|
|
150
162
|
* Drives hard-down escalation — see noteInferenceExhausted. */
|
|
151
163
|
consecutiveInferenceFailures = new Map();
|
|
@@ -174,6 +186,13 @@ export class AgentFramework {
|
|
|
174
186
|
* Bounds the automatic quarantine loop the same way refusalRewinds bounds
|
|
175
187
|
* the refusal loop, so the breaker can never shed the whole history. */
|
|
176
188
|
exhaustionRewinds = new Map();
|
|
189
|
+
/** Agents with an OverBudget drain kick currently in flight. The breaker in
|
|
190
|
+
* noteInferenceExhausted fires on EVERY matching failure and the scenario it
|
|
191
|
+
* exists for is "every activation fails, repeatedly" — without this guard,
|
|
192
|
+
* overlapping kicks race the strategy's own pendingCompression gate and the
|
|
193
|
+
* tick counts in the success log stop meaning anything. One kick per agent
|
|
194
|
+
* at a time; cleared when the kick settles (success or failure). */
|
|
195
|
+
overBudgetDrainInFlight = new Set();
|
|
177
196
|
/** Per-agent current rewind episode: the single consolidated marker's id and
|
|
178
197
|
* how many turns have been shed so far. One marker per episode, updated in
|
|
179
198
|
* place (see updateRewindMarker); cleared when the episode ends. */
|
|
@@ -234,7 +253,9 @@ export class AgentFramework {
|
|
|
234
253
|
eventGate = null;
|
|
235
254
|
// Session-level token usage tracking (always-on)
|
|
236
255
|
usageTracker;
|
|
237
|
-
|
|
256
|
+
/** Presentation-only wall-clock zone; persistence remains UTC/epoch. */
|
|
257
|
+
timeZone;
|
|
258
|
+
constructor(store, ownsStore, membrane, inferencePolicy, errorPolicy, syncIntervalMs, maintenanceIntervalMs, processLoggingPersist, processLoggingBroadcast, timeZone) {
|
|
238
259
|
this.store = store;
|
|
239
260
|
this.ownsStore = ownsStore;
|
|
240
261
|
this.membrane = membrane;
|
|
@@ -244,6 +265,7 @@ export class AgentFramework {
|
|
|
244
265
|
this.maintenanceIntervalMs = maintenanceIntervalMs;
|
|
245
266
|
this.processLoggingPersist = processLoggingPersist;
|
|
246
267
|
this.processLoggingBroadcast = processLoggingBroadcast;
|
|
268
|
+
this.timeZone = timeZone;
|
|
247
269
|
this.queue = new ProcessQueueImpl();
|
|
248
270
|
this.usageTracker = new UsageTracker({
|
|
249
271
|
emitTrace: (e) => this.emitTrace({ ...e }),
|
|
@@ -323,7 +345,7 @@ export class AgentFramework {
|
|
|
323
345
|
// Already registered
|
|
324
346
|
}
|
|
325
347
|
}
|
|
326
|
-
const framework = new AgentFramework(store, ownsStore, config.membrane, config.inferencePolicy ?? new DefaultInferencePolicy(), config.errorPolicy ?? new DefaultErrorPolicy(), config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS, config.maintenanceIntervalMs ?? DEFAULT_MAINTENANCE_INTERVAL_MS, processLoggingPersist, processLoggingBroadcast);
|
|
348
|
+
const framework = new AgentFramework(store, ownsStore, config.membrane, config.inferencePolicy ?? new DefaultInferencePolicy(), config.errorPolicy ?? new DefaultErrorPolicy(), config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS, config.maintenanceIntervalMs ?? DEFAULT_MAINTENANCE_INTERVAL_MS, processLoggingPersist, processLoggingBroadcast, resolveTimeZone(config.timeZone));
|
|
327
349
|
// Restore persisted usage data (if any) from prior session
|
|
328
350
|
framework.restoreUsageState();
|
|
329
351
|
// Create agents
|
|
@@ -533,7 +555,9 @@ export class AgentFramework {
|
|
|
533
555
|
if (this.maintenancePass || !this.running)
|
|
534
556
|
return;
|
|
535
557
|
const pass = this.runQueuedMaintenance().catch((error) => {
|
|
536
|
-
|
|
558
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
559
|
+
console.error('[context-maintenance] pass failed:', reason);
|
|
560
|
+
this.opsAlert('context-maintenance-failed', 'framework', reason, { data: { scope: 'pass' } });
|
|
537
561
|
});
|
|
538
562
|
this.maintenancePass = pass;
|
|
539
563
|
void pass.finally(() => {
|
|
@@ -584,6 +608,13 @@ export class AgentFramework {
|
|
|
584
608
|
catch (error) {
|
|
585
609
|
record.error = error instanceof Error ? error.message : String(error);
|
|
586
610
|
console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
|
|
611
|
+
this.opsAlert('context-maintenance-failed', agent.name, record.error, {
|
|
612
|
+
data: {
|
|
613
|
+
scope: 'agent',
|
|
614
|
+
...(record.pendingBefore ? { pending: record.pendingBefore } : {}),
|
|
615
|
+
...(record.progressBefore ? { progress: record.progressBefore } : {}),
|
|
616
|
+
},
|
|
617
|
+
});
|
|
587
618
|
}
|
|
588
619
|
finally {
|
|
589
620
|
const pending = cm.getPendingWork()?.description;
|
|
@@ -930,6 +961,50 @@ export class AgentFramework {
|
|
|
930
961
|
};
|
|
931
962
|
return { agent, contextManager, cleanup };
|
|
932
963
|
}
|
|
964
|
+
createDeferred() {
|
|
965
|
+
let resolve;
|
|
966
|
+
let reject;
|
|
967
|
+
const promise = new Promise((res, rej) => {
|
|
968
|
+
resolve = res;
|
|
969
|
+
reject = rej;
|
|
970
|
+
});
|
|
971
|
+
return { promise, resolve, reject };
|
|
972
|
+
}
|
|
973
|
+
/** Liveness ping for an active ephemeral run (no-op otherwise). */
|
|
974
|
+
touchEphemeralRun(agentName, inferenceStarted = false) {
|
|
975
|
+
const run = this.ephemeralRuns.get(agentName);
|
|
976
|
+
if (!run)
|
|
977
|
+
return;
|
|
978
|
+
if (inferenceStarted) {
|
|
979
|
+
run.inferenceStarted = true;
|
|
980
|
+
}
|
|
981
|
+
run.lastActivity = Date.now();
|
|
982
|
+
}
|
|
983
|
+
recordEphemeralToolCalls(agentName, count) {
|
|
984
|
+
const run = this.ephemeralRuns.get(agentName);
|
|
985
|
+
if (!run)
|
|
986
|
+
return;
|
|
987
|
+
run.toolCallsCount += count;
|
|
988
|
+
run.lastActivity = Date.now();
|
|
989
|
+
}
|
|
990
|
+
/** Settle an active ephemeral run (no-op for resident agents). The tool-call
|
|
991
|
+
* count comes from the run's own counter — the single source of truth,
|
|
992
|
+
* which unlike a per-stream local survives budget restarts. */
|
|
993
|
+
settleAgent(agentName, outcome) {
|
|
994
|
+
const run = this.ephemeralRuns.get(agentName);
|
|
995
|
+
if (!run)
|
|
996
|
+
return;
|
|
997
|
+
run.lastActivity = Date.now();
|
|
998
|
+
if (outcome.stopReason === 'exhausted') {
|
|
999
|
+
run.settle.reject(new Error(outcome.error ?? 'Unknown error'));
|
|
1000
|
+
return;
|
|
1001
|
+
}
|
|
1002
|
+
run.settle.resolve({
|
|
1003
|
+
stopReason: outcome.stopReason,
|
|
1004
|
+
speech: outcome.speech,
|
|
1005
|
+
toolCallsCount: run.toolCallsCount,
|
|
1006
|
+
});
|
|
1007
|
+
}
|
|
933
1008
|
/**
|
|
934
1009
|
* Run an ephemeral agent to completion through the framework's event loop.
|
|
935
1010
|
*
|
|
@@ -940,139 +1015,73 @@ export class AgentFramework {
|
|
|
940
1015
|
* The caller provides a pre-created agent + contextManager (from createEphemeralAgent).
|
|
941
1016
|
* The task message should already be in the context manager.
|
|
942
1017
|
*/
|
|
943
|
-
async runEphemeralToCompletion(agent, contextManager) {
|
|
1018
|
+
async runEphemeralToCompletion(agent, contextManager, watchdogs) {
|
|
944
1019
|
// Register temporarily so the event loop can drive it
|
|
945
1020
|
this.agents.set(agent.name, agent);
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
clearTimeout(startupWatchdog);
|
|
965
|
-
clearInterval(completionWatchdog);
|
|
966
|
-
this.offTrace(traceListener);
|
|
967
|
-
this.agents.delete(agent.name);
|
|
968
|
-
this.evictTurnCheckpoints(agent.name);
|
|
969
|
-
};
|
|
970
|
-
// Watchdog: if inference never starts within 30s, the agent is a zombie.
|
|
971
|
-
// This catches cases where the event loop stalled, the agent was
|
|
972
|
-
// deregistered, or processInferenceRequests() skipped the request.
|
|
973
|
-
const startupWatchdog = setTimeout(() => {
|
|
974
|
-
if (!inferenceStarted && !settled) {
|
|
975
|
-
cleanup();
|
|
1021
|
+
const run = {
|
|
1022
|
+
settle: this.createDeferred(),
|
|
1023
|
+
inferenceStarted: false,
|
|
1024
|
+
lastActivity: Date.now(),
|
|
1025
|
+
toolCallsCount: 0,
|
|
1026
|
+
};
|
|
1027
|
+
this.ephemeralRuns.set(agent.name, run);
|
|
1028
|
+
const STARTUP_TIMEOUT_MS = watchdogs?.startupTimeoutMs ?? 30_000;
|
|
1029
|
+
// After inference has started, give it 15 minutes of activity-bounded
|
|
1030
|
+
// life. The stream driver refreshes the deadline as it makes progress;
|
|
1031
|
+
// sustained silence trips it.
|
|
1032
|
+
const COMPLETION_IDLE_TIMEOUT_MS = watchdogs?.idleTimeoutMs ?? 15 * 60_000;
|
|
1033
|
+
const IDLE_POLL_MS = watchdogs?.idlePollMs ?? 30_000;
|
|
1034
|
+
let startupWatchdog = null;
|
|
1035
|
+
let completionWatchdog = null;
|
|
1036
|
+
const startupTimeout = new Promise((_, reject) => {
|
|
1037
|
+
startupWatchdog = setTimeout(() => {
|
|
1038
|
+
if (!run.inferenceStarted) {
|
|
976
1039
|
reject(new Error(`Ephemeral agent "${agent.name}" failed to start inference within ` +
|
|
977
1040
|
`${STARTUP_TIMEOUT_MS}ms — zombie detected. The event loop may have ` +
|
|
978
1041
|
`stalled or the inference request was dropped.`));
|
|
979
1042
|
}
|
|
980
1043
|
}, STARTUP_TIMEOUT_MS);
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
// and keeps its concurrency slot pinned.
|
|
986
|
-
const completionWatchdog = setInterval(() => {
|
|
987
|
-
if (settled || !inferenceStarted)
|
|
1044
|
+
});
|
|
1045
|
+
const completionIdleTimeout = new Promise((_, reject) => {
|
|
1046
|
+
completionWatchdog = setInterval(() => {
|
|
1047
|
+
if (!run.inferenceStarted)
|
|
988
1048
|
return;
|
|
989
|
-
const idle = Date.now() - lastActivity;
|
|
1049
|
+
const idle = Date.now() - run.lastActivity;
|
|
990
1050
|
if (idle > COMPLETION_IDLE_TIMEOUT_MS) {
|
|
991
|
-
|
|
992
|
-
reject(new Error(`Ephemeral agent "${agent.name}" stalled: no trace activity for ` +
|
|
1051
|
+
reject(new Error(`Ephemeral agent "${agent.name}" stalled: no stream-driver activity for ` +
|
|
993
1052
|
`${Math.round(idle / 1000)}s after inference started (threshold ` +
|
|
994
1053
|
`${Math.round(COMPLETION_IDLE_TIMEOUT_MS / 1000)}s). Stream likely ` +
|
|
995
1054
|
`dropped or terminal event was lost.`));
|
|
996
1055
|
}
|
|
997
|
-
},
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
// Only track events for our ephemeral agent
|
|
1002
|
-
const agentName = 'agentName' in event ? event.agentName : null;
|
|
1003
|
-
if (agentName !== agent.name)
|
|
1004
|
-
return;
|
|
1005
|
-
// Any addressed event counts as liveness for the completion watchdog.
|
|
1006
|
-
lastActivity = Date.now();
|
|
1007
|
-
switch (event.type) {
|
|
1008
|
-
case 'inference:started':
|
|
1009
|
-
inferenceStarted = true;
|
|
1010
|
-
break;
|
|
1011
|
-
case 'inference:tokens': {
|
|
1012
|
-
inferenceStarted = true;
|
|
1013
|
-
const content = event.content;
|
|
1014
|
-
if (content)
|
|
1015
|
-
speech += content;
|
|
1016
|
-
break;
|
|
1017
|
-
}
|
|
1018
|
-
case 'inference:tool_calls_yielded': {
|
|
1019
|
-
const calls = event.calls;
|
|
1020
|
-
if (calls)
|
|
1021
|
-
toolCallsCount += calls.length;
|
|
1022
|
-
// Reset speech buffer — tool calls break the text
|
|
1023
|
-
speech = '';
|
|
1024
|
-
break;
|
|
1025
|
-
}
|
|
1026
|
-
case 'inference:stream_resumed':
|
|
1027
|
-
speech = '';
|
|
1028
|
-
break;
|
|
1029
|
-
case 'inference:completed': {
|
|
1030
|
-
// `inference:completed` is terminal — driveStream only emits it
|
|
1031
|
-
// from `case 'complete'`, which fires after the full tool
|
|
1032
|
-
// round-trip has finished and the model has stopped. Intermediate
|
|
1033
|
-
// tool-cycle transitions use `inference:stream_resumed`, not
|
|
1034
|
-
// `inference:completed`. So we resolve unconditionally here.
|
|
1035
|
-
//
|
|
1036
|
-
// Previously this branch gated on `agent.state.status === 'idle'`,
|
|
1037
|
-
// which was correct in intent but order-fragile: the framework
|
|
1038
|
-
// emitted `inference:completed` BEFORE calling `agent.reset()`,
|
|
1039
|
-
// so synchronous listeners observed status === 'streaming' at
|
|
1040
|
-
// the trace boundary. The gate failed silently and the promise
|
|
1041
|
-
// hung forever — every production zombie subagent originated
|
|
1042
|
-
// here. The reset+emit order has been fixed in driveStream's
|
|
1043
|
-
// `case 'complete'` handler; the gate is dropped here as
|
|
1044
|
-
// belt-and-suspenders.
|
|
1045
|
-
cleanup();
|
|
1046
|
-
resolve({ speech, toolCallsCount });
|
|
1047
|
-
break;
|
|
1048
|
-
}
|
|
1049
|
-
case 'inference:turn_ended': {
|
|
1050
|
-
// endTurn from a tool result — agent is done
|
|
1051
|
-
cleanup();
|
|
1052
|
-
resolve({ speech, toolCallsCount });
|
|
1053
|
-
break;
|
|
1054
|
-
}
|
|
1055
|
-
case 'inference:exhausted': {
|
|
1056
|
-
// Retries exhausted — reject only after the error policy gives up.
|
|
1057
|
-
// Earlier inference:failed events are left alone so the framework
|
|
1058
|
-
// retry path in driveStream/startAgentStream can restart the stream
|
|
1059
|
-
// while this listener stays alive.
|
|
1060
|
-
const error = event.error ?? 'Unknown error';
|
|
1061
|
-
cleanup();
|
|
1062
|
-
reject(new Error(error));
|
|
1063
|
-
break;
|
|
1064
|
-
}
|
|
1065
|
-
}
|
|
1066
|
-
};
|
|
1067
|
-
this.onTrace(traceListener);
|
|
1068
|
-
// Trigger inference
|
|
1056
|
+
}, IDLE_POLL_MS);
|
|
1057
|
+
});
|
|
1058
|
+
try {
|
|
1059
|
+
// Trigger inference after the settle promise is registered.
|
|
1069
1060
|
this.pendingRequests.push({
|
|
1070
1061
|
agentName: agent.name,
|
|
1071
1062
|
reason: 'ephemeral',
|
|
1072
1063
|
source: 'subagent',
|
|
1073
1064
|
timestamp: Date.now(),
|
|
1074
1065
|
});
|
|
1075
|
-
|
|
1066
|
+
const result = await Promise.race([
|
|
1067
|
+
run.settle.promise,
|
|
1068
|
+
startupTimeout,
|
|
1069
|
+
completionIdleTimeout,
|
|
1070
|
+
]);
|
|
1071
|
+
return { speech: result.speech, toolCallsCount: result.toolCallsCount };
|
|
1072
|
+
}
|
|
1073
|
+
finally {
|
|
1074
|
+
if (startupWatchdog)
|
|
1075
|
+
clearTimeout(startupWatchdog);
|
|
1076
|
+
if (completionWatchdog)
|
|
1077
|
+
clearInterval(completionWatchdog);
|
|
1078
|
+
this.ephemeralRuns.delete(agent.name);
|
|
1079
|
+
this.agents.delete(agent.name);
|
|
1080
|
+
// Spawn-and-dispose bookkeeping (main, d453165/fee96a7): without this,
|
|
1081
|
+
// ephemeral agents leave checkpoint-tree keys and diagnostics map
|
|
1082
|
+
// entries behind for the life of the store/session.
|
|
1083
|
+
this.evictTurnCheckpoints(agent.name);
|
|
1084
|
+
}
|
|
1076
1085
|
}
|
|
1077
1086
|
/**
|
|
1078
1087
|
* Get queue depth.
|
|
@@ -2192,6 +2201,9 @@ export class AgentFramework {
|
|
|
2192
2201
|
// messages between tool_use and tool_result, causing a 400 error.
|
|
2193
2202
|
if (event.type === 'tool-result') {
|
|
2194
2203
|
const agent = this.agents.get(event.agentName);
|
|
2204
|
+
if (agent) {
|
|
2205
|
+
this.touchEphemeralRun(event.agentName, true);
|
|
2206
|
+
}
|
|
2195
2207
|
if (!agent) {
|
|
2196
2208
|
console.warn(`[framework] Dropping tool-result for unknown agent '${event.agentName}' (callId=${event.callId}). ` +
|
|
2197
2209
|
`Agent may have been destroyed while tool was executing.`);
|
|
@@ -2256,13 +2268,29 @@ export class AgentFramework {
|
|
|
2256
2268
|
if (shouldEndTurn) {
|
|
2257
2269
|
// endTurn: messages already stored above, cancel stream, reset to idle.
|
|
2258
2270
|
if (currentState.stream) {
|
|
2259
|
-
agent.
|
|
2271
|
+
this.frameworkCancelledStreams.set(`${agent.name}:${agent.streamId}`, 'turn_ended');
|
|
2272
|
+
currentState.stream.cancel();
|
|
2260
2273
|
}
|
|
2261
2274
|
agent.reset();
|
|
2275
|
+
// The turn is over — release the gate here, not just in the abort
|
|
2276
|
+
// handler: the membrane's aborted event is asynchronous and the
|
|
2277
|
+
// agent must be wakeable the moment it goes idle. (Idempotent;
|
|
2278
|
+
// driveStream's finally is the backstop.)
|
|
2279
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
2280
|
+
this.settleAgent(agent.name, { stopReason: 'turn_ended', speech: '' });
|
|
2262
2281
|
this.emitTrace({ type: 'inference:turn_ended', agentName: agent.name });
|
|
2263
2282
|
}
|
|
2264
2283
|
else if (overBudget) {
|
|
2265
|
-
// Context budget exceeded: break the stream, let compile() compress
|
|
2284
|
+
// Context budget exceeded: break the stream, let compile() compress.
|
|
2285
|
+
// Mark the cancel as framework-initiated BEFORE cancelling: the
|
|
2286
|
+
// membrane delivers `aborted` before the restart bumps streamId,
|
|
2287
|
+
// and without the marker the abort handler treats the restart as
|
|
2288
|
+
// a terminal failure — rejecting an ephemeral's promise mid-run
|
|
2289
|
+
// and emitting a spurious inference:exhausted (same race shape as
|
|
2290
|
+
// endTurn above).
|
|
2291
|
+
if (currentState.stream) {
|
|
2292
|
+
this.frameworkCancelledStreams.set(`${agent.name}:${agent.streamId}`, 'budget_restart');
|
|
2293
|
+
}
|
|
2266
2294
|
agent.cancelStream();
|
|
2267
2295
|
this.emitTrace({
|
|
2268
2296
|
type: 'inference:stream_restarted',
|
|
@@ -2821,6 +2849,7 @@ export class AgentFramework {
|
|
|
2821
2849
|
else {
|
|
2822
2850
|
this.activeTriggerChannels.delete(agent.name);
|
|
2823
2851
|
}
|
|
2852
|
+
this.touchEphemeralRun(agent.name, true);
|
|
2824
2853
|
this.emitTrace({ type: 'inference:started', agentName: agent.name });
|
|
2825
2854
|
this.eventGate?.onInferenceStarted(agent.name);
|
|
2826
2855
|
this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), startedAt: Date.now() });
|
|
@@ -2876,17 +2905,23 @@ export class AgentFramework {
|
|
|
2876
2905
|
await this.startAgentStream(agent, trigger, attempt + 1);
|
|
2877
2906
|
}
|
|
2878
2907
|
else {
|
|
2908
|
+
this.settleAgent(agent.name, {
|
|
2909
|
+
stopReason: 'exhausted',
|
|
2910
|
+
speech: '',
|
|
2911
|
+
error: err.message,
|
|
2912
|
+
});
|
|
2879
2913
|
this.emitTrace({
|
|
2880
2914
|
type: 'inference:exhausted',
|
|
2881
2915
|
agentName: agent.name,
|
|
2882
2916
|
error: err.message,
|
|
2883
|
-
// Drives the poison-history breaker
|
|
2884
|
-
//
|
|
2885
|
-
//
|
|
2886
|
-
//
|
|
2887
|
-
//
|
|
2888
|
-
|
|
2889
|
-
|
|
2917
|
+
// Drives the poison-history breaker (`invalid_request`: a 400-class
|
|
2918
|
+
// rejection of the history itself — retrying the same context can
|
|
2919
|
+
// never succeed; auth/abort/context_length are also non-retryable
|
|
2920
|
+
// but must NOT cost history) and the OverBudget drain breaker
|
|
2921
|
+
// (`over_budget`: compile refused to fit the hard budget — this is
|
|
2922
|
+
// the site that sees it, since compile runs before the stream
|
|
2923
|
+
// exists). `retryable` is kept for observability.
|
|
2924
|
+
...this.classifyInferenceError(err),
|
|
2890
2925
|
});
|
|
2891
2926
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
2892
2927
|
if (action.emit) {
|
|
@@ -2909,6 +2944,7 @@ export class AgentFramework {
|
|
|
2909
2944
|
this.channelRegistry.startTyping(typingChannel);
|
|
2910
2945
|
try {
|
|
2911
2946
|
for await (const event of stream) {
|
|
2947
|
+
this.touchEphemeralRun(agent.name, true);
|
|
2912
2948
|
switch (event.type) {
|
|
2913
2949
|
case 'tokens':
|
|
2914
2950
|
this.emitTrace({
|
|
@@ -2932,6 +2968,7 @@ export class AgentFramework {
|
|
|
2932
2968
|
}
|
|
2933
2969
|
case 'tool-calls': {
|
|
2934
2970
|
hadToolCalls = true;
|
|
2971
|
+
this.recordEphemeralToolCalls(agent.name, event.calls.length);
|
|
2935
2972
|
this.emitTrace({
|
|
2936
2973
|
type: 'inference:tool_calls_yielded',
|
|
2937
2974
|
agentName: agent.name,
|
|
@@ -3017,14 +3054,16 @@ export class AgentFramework {
|
|
|
3017
3054
|
// thinking blocks from their tool_use turn (the API requires
|
|
3018
3055
|
// thinking to precede tool_use in the same assistant turn).
|
|
3019
3056
|
const lastToolIdx = response.content.reduce((last, b, i) => b.type === 'tool_use' || b.type === 'tool_result' ? i : last, -1);
|
|
3057
|
+
const terminalContent = lastToolIdx >= 0
|
|
3058
|
+
? response.content.slice(lastToolIdx + 1)
|
|
3059
|
+
: response.content;
|
|
3020
3060
|
if (lastToolIdx >= 0) {
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
agent.addAssistantResponse(trailingContent);
|
|
3061
|
+
if (terminalContent.length > 0) {
|
|
3062
|
+
agent.addAssistantResponse(terminalContent);
|
|
3024
3063
|
}
|
|
3025
3064
|
}
|
|
3026
3065
|
else {
|
|
3027
|
-
agent.addAssistantResponse(
|
|
3066
|
+
agent.addAssistantResponse(terminalContent);
|
|
3028
3067
|
}
|
|
3029
3068
|
// Run afterInference hooks (no-op if no MCPL servers)
|
|
3030
3069
|
if (this.hookOrchestrator) {
|
|
@@ -3076,17 +3115,20 @@ export class AgentFramework {
|
|
|
3076
3115
|
cacheRead: du.cacheReadTokens,
|
|
3077
3116
|
}
|
|
3078
3117
|
: undefined;
|
|
3079
|
-
// Reset agent state
|
|
3080
|
-
// synchronous listeners
|
|
3081
|
-
//
|
|
3082
|
-
//
|
|
3083
|
-
//
|
|
3084
|
-
// trace boundary, failed its idle gate, and the promise hung
|
|
3085
|
-
// forever — every "zombie subagent" in production tracked back
|
|
3086
|
-
// here. Speech dispatch happens after but doesn't depend on the
|
|
3087
|
-
// status field.
|
|
3118
|
+
// Reset agent state before emitting inference:completed. Traces are
|
|
3119
|
+
// observability-only, but external synchronous listeners should
|
|
3120
|
+
// still see the terminal state at the terminal trace boundary.
|
|
3121
|
+
// Speech dispatch happens after but doesn't depend on the status
|
|
3122
|
+
// field.
|
|
3088
3123
|
agent.reset();
|
|
3089
3124
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
3125
|
+
this.settleAgent(agent.name, {
|
|
3126
|
+
stopReason: 'completed',
|
|
3127
|
+
speech: terminalContent
|
|
3128
|
+
.filter((block) => block.type === 'text')
|
|
3129
|
+
.map((block) => block.text)
|
|
3130
|
+
.join('\n'),
|
|
3131
|
+
});
|
|
3090
3132
|
this.emitTrace({
|
|
3091
3133
|
type: 'inference:completed',
|
|
3092
3134
|
agentName: agent.name,
|
|
@@ -3297,9 +3339,8 @@ export class AgentFramework {
|
|
|
3297
3339
|
}
|
|
3298
3340
|
}
|
|
3299
3341
|
}
|
|
3300
|
-
// NOTE: agent.reset() + onInferenceEnded() already ran above,
|
|
3301
|
-
// dispatchSpeech
|
|
3302
|
-
// must see status === 'idle'). Locus routing is speech dispatch and
|
|
3342
|
+
// NOTE: agent.reset() + onInferenceEnded() already ran above,
|
|
3343
|
+
// BEFORE dispatchSpeech. Locus routing is speech dispatch and
|
|
3303
3344
|
// doesn't depend on the status field, so it correctly runs after.
|
|
3304
3345
|
break;
|
|
3305
3346
|
}
|
|
@@ -3331,16 +3372,20 @@ export class AgentFramework {
|
|
|
3331
3372
|
await this.startAgentStream(agent, trigger, attempt + 1);
|
|
3332
3373
|
}
|
|
3333
3374
|
else {
|
|
3375
|
+
this.settleAgent(agent.name, {
|
|
3376
|
+
stopReason: 'exhausted',
|
|
3377
|
+
speech: '',
|
|
3378
|
+
error: err.message,
|
|
3379
|
+
});
|
|
3334
3380
|
this.emitTrace({
|
|
3335
3381
|
type: 'inference:exhausted',
|
|
3336
3382
|
agentName: agent.name,
|
|
3337
3383
|
error: err.message,
|
|
3338
3384
|
// Drives the poison-history breaker. `retryable` is kept for
|
|
3339
|
-
// observability, but the
|
|
3385
|
+
// observability, but the breakers gate on `errorType` — only an
|
|
3340
3386
|
// `invalid_request` (a 400-class rejection of the history
|
|
3341
3387
|
// itself) means retrying the same context can never succeed.
|
|
3342
|
-
|
|
3343
|
-
errorType: err instanceof MembraneError ? err.type : undefined,
|
|
3388
|
+
...this.classifyInferenceError(err),
|
|
3344
3389
|
});
|
|
3345
3390
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
3346
3391
|
if (action.emit) {
|
|
@@ -3350,12 +3395,28 @@ export class AgentFramework {
|
|
|
3350
3395
|
break;
|
|
3351
3396
|
}
|
|
3352
3397
|
case 'aborted': {
|
|
3398
|
+
// Framework-initiated, non-terminal cancels (endTurn tool result,
|
|
3399
|
+
// context-budget restart) also surface here as `aborted` — but
|
|
3400
|
+
// the turn either already settled (endTurn) or a replacement
|
|
3401
|
+
// stream is queued (budget restart). Neither is a failure: no
|
|
3402
|
+
// reset, no settle, no spurious inference:exhausted (which would
|
|
3403
|
+
// reject an ephemeral's promise mid-run and bump the failure
|
|
3404
|
+
// streak). Gate release + stream teardown happen in `finally`.
|
|
3405
|
+
if (this.frameworkCancelledStreams.delete(`${agent.name}:${myStreamId}`)) {
|
|
3406
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
3407
|
+
return;
|
|
3408
|
+
}
|
|
3353
3409
|
const reason = event.reason ?? 'unknown';
|
|
3354
3410
|
// Only reset if this is still the active stream (a budget restart
|
|
3355
3411
|
// may have already started a new stream, bumping streamId)
|
|
3356
3412
|
if (agent.streamId === myStreamId) {
|
|
3357
3413
|
const durationMs = Date.now() - startTime;
|
|
3358
3414
|
agent.reset();
|
|
3415
|
+
this.settleAgent(agent.name, {
|
|
3416
|
+
stopReason: 'exhausted',
|
|
3417
|
+
speech: '',
|
|
3418
|
+
error: `Stream aborted: ${reason}`,
|
|
3419
|
+
});
|
|
3359
3420
|
this.emitTrace({
|
|
3360
3421
|
type: 'inference:exhausted',
|
|
3361
3422
|
agentName: agent.name,
|
|
@@ -3425,10 +3486,16 @@ export class AgentFramework {
|
|
|
3425
3486
|
error: err.message,
|
|
3426
3487
|
stack: err.stack,
|
|
3427
3488
|
});
|
|
3489
|
+
this.settleAgent(agent.name, {
|
|
3490
|
+
stopReason: 'exhausted',
|
|
3491
|
+
speech: '',
|
|
3492
|
+
error: err.message,
|
|
3493
|
+
});
|
|
3428
3494
|
this.emitTrace({
|
|
3429
3495
|
type: 'inference:exhausted',
|
|
3430
3496
|
agentName: agent.name,
|
|
3431
3497
|
error: err.message,
|
|
3498
|
+
...this.classifyInferenceError(err),
|
|
3432
3499
|
});
|
|
3433
3500
|
// Postmortem 2026-05-28 P2 #7: catch-path failures (stream itself
|
|
3434
3501
|
// threw) were previously only visible via in-memory trace listeners.
|
|
@@ -3458,6 +3525,7 @@ export class AgentFramework {
|
|
|
3458
3525
|
// Stop the typing indicator on every exit path (complete, error,
|
|
3459
3526
|
// exhausted, abort) so it never sticks after the turn ends.
|
|
3460
3527
|
this.channelRegistry?.stopTyping();
|
|
3528
|
+
this.frameworkCancelledStreams.delete(`${agent.name}:${myStreamId}`);
|
|
3461
3529
|
this.activeStreams.delete(agent.name);
|
|
3462
3530
|
this.pendingAssistantBlocks.delete(agent.name);
|
|
3463
3531
|
// A conversation fork whose TTL closure turn just finished is done for
|
|
@@ -3975,6 +4043,27 @@ export class AgentFramework {
|
|
|
3975
4043
|
}
|
|
3976
4044
|
return streak;
|
|
3977
4045
|
}
|
|
4046
|
+
/**
|
|
4047
|
+
* Classify a terminal inference error for the `inference:exhausted` trace.
|
|
4048
|
+
* Single-sourced for every emit site: the downstream breakers (poison-history
|
|
4049
|
+
* quarantine, OverBudget drain kick) gate on `errorType`, so classification
|
|
4050
|
+
* drift between sites would silently disable a safety net.
|
|
4051
|
+
*
|
|
4052
|
+
* context-manager's OverBudgetError is recognized by `err.name`: CM does not
|
|
4053
|
+
* export the class from its package root, so a cross-package `instanceof` is
|
|
4054
|
+
* unavailable — but `name` is set in its constructor and survives the package
|
|
4055
|
+
* boundary. Deliberately NOT a message match: the message wording belongs to
|
|
4056
|
+
* CM and can be reworded without warning.
|
|
4057
|
+
*/
|
|
4058
|
+
classifyInferenceError(err) {
|
|
4059
|
+
if (err instanceof MembraneError) {
|
|
4060
|
+
return { retryable: err.retryable, errorType: err.type };
|
|
4061
|
+
}
|
|
4062
|
+
if (err.name === 'OverBudgetError') {
|
|
4063
|
+
return { errorType: 'over_budget' };
|
|
4064
|
+
}
|
|
4065
|
+
return {};
|
|
4066
|
+
}
|
|
3978
4067
|
noteInferenceExhausted(agentName, reason, retryable, errorType) {
|
|
3979
4068
|
const streak = (this.consecutiveInferenceFailures.get(agentName) ?? 0) + 1;
|
|
3980
4069
|
this.consecutiveInferenceFailures.set(agentName, streak);
|
|
@@ -4002,6 +4091,42 @@ export class AgentFramework {
|
|
|
4002
4091
|
console.error(`[inference-failed] could not record chronicle marker for ${agentName}:`, err);
|
|
4003
4092
|
}
|
|
4004
4093
|
}
|
|
4094
|
+
// (2b) OverBudget deadlock breaker. When compile fails with
|
|
4095
|
+
// OverBudgetError, the normal compression drain never runs: it is driven
|
|
4096
|
+
// by successful activity, which the over-budget state prevents - a closed
|
|
4097
|
+
// loop with no internal exit. Field data (2026-07-10, resident agent):
|
|
4098
|
+
// 36 minutes hard-down, zero self-rescue; only an operator raising the
|
|
4099
|
+
// budget externally could break the loop. Break it here instead: kick the
|
|
4100
|
+
// strategy drain directly so folding/merging frees space for the next
|
|
4101
|
+
// compile. Bounded ticks, best-effort, never throws, one kick per agent
|
|
4102
|
+
// at a time (see overBudgetDrainInFlight).
|
|
4103
|
+
//
|
|
4104
|
+
// Gates on the classified errorType (see classifyInferenceError); the
|
|
4105
|
+
// message match is only a fallback for paths that lost the Error object
|
|
4106
|
+
// (e.g. a reason string that crossed a serialization boundary). It matches
|
|
4107
|
+
// CM's current OverBudgetError wording and MAY rot if CM rewords it — the
|
|
4108
|
+
// errorType gate is the one that's load-bearing.
|
|
4109
|
+
const overBudget = errorType === 'over_budget' || /exceed hard budget/i.test(reason);
|
|
4110
|
+
if (agent && overBudget && !this.overBudgetDrainInFlight.has(agentName)) {
|
|
4111
|
+
this.overBudgetDrainInFlight.add(agentName);
|
|
4112
|
+
void (async () => {
|
|
4113
|
+
let ticks = 0;
|
|
4114
|
+
try {
|
|
4115
|
+
const cm = agent.getContextManager();
|
|
4116
|
+
while (ticks < 8) {
|
|
4117
|
+
await cm.tick();
|
|
4118
|
+
ticks++;
|
|
4119
|
+
}
|
|
4120
|
+
console.error(`[inference-failed] drain kicked for ${agentName} (OverBudget breaker, ${ticks} ticks)`);
|
|
4121
|
+
}
|
|
4122
|
+
catch (err) {
|
|
4123
|
+
console.error(`[inference-failed] drain kick failed for ${agentName} after ${ticks} ticks:`, err);
|
|
4124
|
+
}
|
|
4125
|
+
finally {
|
|
4126
|
+
this.overBudgetDrainInFlight.delete(agentName);
|
|
4127
|
+
}
|
|
4128
|
+
})();
|
|
4129
|
+
}
|
|
4005
4130
|
// (3) Hard-down escalation on repeated identical failure.
|
|
4006
4131
|
if (streak >= this.inferenceFailureEscalationThreshold) {
|
|
4007
4132
|
console.error(`[inference-hard-down] agent=${agentName} has FAILED ${streak} consecutive ` +
|
|
@@ -4885,7 +5010,16 @@ export class AgentFramework {
|
|
|
4885
5010
|
}
|
|
4886
5011
|
console.error(`[sleep] agent=${agentName} seconds=${seconds} announce=${announce} until=${new Date(until).toISOString()}`);
|
|
4887
5012
|
// endTurn: the agent stops here and goes idle for the duration.
|
|
4888
|
-
finish({
|
|
5013
|
+
finish({
|
|
5014
|
+
success: true,
|
|
5015
|
+
data: {
|
|
5016
|
+
sleepingFor: human,
|
|
5017
|
+
until,
|
|
5018
|
+
untilLocal: formatZonedDateTime(until, this.timeZone),
|
|
5019
|
+
timeZone: this.timeZone,
|
|
5020
|
+
},
|
|
5021
|
+
endTurn: true,
|
|
5022
|
+
});
|
|
4889
5023
|
}
|
|
4890
5024
|
/** Aggregate the event-tag vocabulary: reserved chat:* core + each connected
|
|
4891
5025
|
* server's declared tag ontology + gate.js status. */
|