@animalabs/agent-framework 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/agent.d.ts +4 -0
- package/dist/src/agent.d.ts.map +1 -1
- package/dist/src/agent.js +15 -0
- package/dist/src/agent.js.map +1 -1
- package/dist/src/framework.d.ts +176 -0
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +911 -90
- package/dist/src/framework.js.map +1 -1
- package/dist/src/gate/event-gate.d.ts +20 -1
- package/dist/src/gate/event-gate.d.ts.map +1 -1
- package/dist/src/gate/event-gate.js +60 -4
- package/dist/src/gate/event-gate.js.map +1 -1
- package/dist/src/index.d.ts +3 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +4 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/mcpl/channel-registry.d.ts +17 -1
- package/dist/src/mcpl/channel-registry.d.ts.map +1 -1
- package/dist/src/mcpl/channel-registry.js +34 -3
- package/dist/src/mcpl/channel-registry.js.map +1 -1
- package/dist/src/mcpl/push-handler.d.ts.map +1 -1
- package/dist/src/mcpl/push-handler.js +4 -0
- package/dist/src/mcpl/push-handler.js.map +1 -1
- package/dist/src/mcpl/server-connection.d.ts +29 -0
- package/dist/src/mcpl/server-connection.d.ts.map +1 -1
- package/dist/src/mcpl/server-connection.js +102 -4
- package/dist/src/mcpl/server-connection.js.map +1 -1
- package/dist/src/mcpl/types.d.ts +17 -1
- package/dist/src/mcpl/types.d.ts.map +1 -1
- package/dist/src/mcpl/types.js.map +1 -1
- package/dist/src/module-registry.d.ts +15 -1
- package/dist/src/module-registry.d.ts.map +1 -1
- package/dist/src/module-registry.js +92 -4
- package/dist/src/module-registry.js.map +1 -1
- package/dist/src/types/agent.d.ts +37 -0
- package/dist/src/types/agent.d.ts.map +1 -1
- package/dist/src/types/module.d.ts +46 -0
- package/dist/src/types/module.d.ts.map +1 -1
- package/dist/src/types/trace.d.ts +42 -0
- package/dist/src/types/trace.d.ts.map +1 -1
- package/dist/test/agent-cache-ttl.test.d.ts +2 -0
- package/dist/test/agent-cache-ttl.test.d.ts.map +1 -0
- package/dist/test/agent-cache-ttl.test.js +25 -0
- package/dist/test/agent-cache-ttl.test.js.map +1 -0
- package/dist/test/conversation-fork-injection-scoping.test.d.ts +2 -0
- package/dist/test/conversation-fork-injection-scoping.test.d.ts.map +1 -0
- package/dist/test/conversation-fork-injection-scoping.test.js +184 -0
- package/dist/test/conversation-fork-injection-scoping.test.js.map +1 -0
- package/dist/test/gather-context-timeout.test.d.ts +2 -0
- package/dist/test/gather-context-timeout.test.d.ts.map +1 -0
- package/dist/test/gather-context-timeout.test.js +120 -0
- package/dist/test/gather-context-timeout.test.js.map +1 -0
- package/dist/test/inference-exhaustion-rewind.test.d.ts +2 -0
- package/dist/test/inference-exhaustion-rewind.test.d.ts.map +1 -0
- package/dist/test/inference-exhaustion-rewind.test.js +181 -0
- package/dist/test/inference-exhaustion-rewind.test.js.map +1 -0
- package/dist/test/inference-failure-observability.test.js +4 -0
- package/dist/test/inference-failure-observability.test.js.map +1 -1
- package/dist/test/mcpl-reconnect-events.test.d.ts +2 -0
- package/dist/test/mcpl-reconnect-events.test.d.ts.map +1 -0
- package/dist/test/mcpl-reconnect-events.test.js +119 -0
- package/dist/test/mcpl-reconnect-events.test.js.map +1 -0
- package/dist/test/mcpl-reconnect-reregistration.test.d.ts +2 -0
- package/dist/test/mcpl-reconnect-reregistration.test.d.ts.map +1 -0
- package/dist/test/mcpl-reconnect-reregistration.test.js +181 -0
- package/dist/test/mcpl-reconnect-reregistration.test.js.map +1 -0
- package/dist/test/mcpl-request-timeout.test.d.ts +2 -0
- package/dist/test/mcpl-request-timeout.test.d.ts.map +1 -0
- package/dist/test/mcpl-request-timeout.test.js +123 -0
- package/dist/test/mcpl-request-timeout.test.js.map +1 -0
- package/dist/test/mcpl-runtime-lifecycle.test.d.ts +7 -0
- package/dist/test/mcpl-runtime-lifecycle.test.d.ts.map +1 -0
- package/dist/test/mcpl-runtime-lifecycle.test.js +142 -0
- package/dist/test/mcpl-runtime-lifecycle.test.js.map +1 -0
- package/dist/test/refusal-rewind.test.d.ts +2 -0
- package/dist/test/refusal-rewind.test.d.ts.map +1 -0
- package/dist/test/refusal-rewind.test.js +108 -0
- package/dist/test/refusal-rewind.test.js.map +1 -0
- package/dist/test/state-scaling-e2e.test.d.ts +22 -0
- package/dist/test/state-scaling-e2e.test.d.ts.map +1 -0
- package/dist/test/state-scaling-e2e.test.js +200 -0
- package/dist/test/state-scaling-e2e.test.js.map +1 -0
- package/dist/test/state-scaling.test.d.ts +12 -0
- package/dist/test/state-scaling.test.d.ts.map +1 -0
- package/dist/test/state-scaling.test.js +243 -0
- package/dist/test/state-scaling.test.js.map +1 -0
- package/dist/test/tool-result-content.test.d.ts +2 -0
- package/dist/test/tool-result-content.test.d.ts.map +1 -0
- package/dist/test/tool-result-content.test.js +37 -0
- package/dist/test/tool-result-content.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +3 -2
- package/src/agent.ts +16 -0
- package/src/framework.ts +1069 -99
- package/src/gate/event-gate.ts +56 -4
- package/src/index.ts +10 -0
- package/src/mcpl/channel-registry.ts +34 -3
- package/src/mcpl/push-handler.ts +4 -0
- package/src/mcpl/server-connection.ts +110 -4
- package/src/mcpl/types.ts +19 -1
- package/src/module-registry.ts +113 -4
- package/src/types/agent.ts +38 -0
- package/src/types/module.ts +52 -0
- package/src/types/trace.ts +51 -0
package/dist/src/framework.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { join } from 'node:path';
|
|
2
|
+
import { appendFileSync, mkdirSync } from 'node:fs';
|
|
2
3
|
import { JsStore } from '@animalabs/chronicle';
|
|
3
4
|
import { MembraneError } from '@animalabs/membrane';
|
|
4
5
|
import { ContextManager, PassthroughStrategy } from '@animalabs/context-manager';
|
|
5
6
|
import { ProcessQueueImpl } from './queue.js';
|
|
6
7
|
import { Agent } from './agent.js';
|
|
7
|
-
import { ModuleRegistry } from './module-registry.js';
|
|
8
|
+
import { ModuleRegistry, isStateExistsError } from './module-registry.js';
|
|
8
9
|
import { McplServerRegistry } from './mcpl/server-registry.js';
|
|
9
10
|
import { FeatureSetManager } from './mcpl/feature-set-manager.js';
|
|
10
11
|
import { ScopeManager } from './mcpl/scope-manager.js';
|
|
@@ -24,7 +25,8 @@ const FRAMEWORK_STATE_ID = 'framework/state';
|
|
|
24
25
|
const CONVERSATION_ROUTER_STATE_ID = 'framework/conversation-router';
|
|
25
26
|
const INFERENCE_LOG_ID = 'framework/inference-log';
|
|
26
27
|
const PROCESS_LOG_ID = 'framework/process-log';
|
|
27
|
-
const TURN_CHECKPOINTS_ID = 'framework/turn-checkpoints';
|
|
28
|
+
const TURN_CHECKPOINTS_ID = 'framework/turn-checkpoints'; // legacy single-map layout, read-only fallback
|
|
29
|
+
const TURN_CHECKPOINTS_TREE_ID = 'framework/turn-checkpoints/tree';
|
|
28
30
|
/** Maximum number of turn checkpoints to keep per agent. */
|
|
29
31
|
const MAX_TURN_CHECKPOINTS = 20;
|
|
30
32
|
/**
|
|
@@ -95,6 +97,12 @@ function extractGateFields(event) {
|
|
|
95
97
|
: undefined;
|
|
96
98
|
return { content, metadata, mount, paths };
|
|
97
99
|
}
|
|
100
|
+
/** Cap an API error message for inline use in a rewind marker: keep enough to
|
|
101
|
+
* identify the failure class without pasting a wall of provider JSON into the
|
|
102
|
+
* agent's context. */
|
|
103
|
+
function truncateReason(reason, max = 160) {
|
|
104
|
+
return reason.length <= max ? reason : reason.slice(0, max) + '…';
|
|
105
|
+
}
|
|
98
106
|
export class AgentFramework {
|
|
99
107
|
store;
|
|
100
108
|
ownsStore;
|
|
@@ -120,6 +128,10 @@ export class AgentFramework {
|
|
|
120
128
|
traceListeners = [];
|
|
121
129
|
syncIntervalMs;
|
|
122
130
|
syncTimer = null;
|
|
131
|
+
/** Last time we console-warned about stale (busy-requeued) inference requests, per agent. */
|
|
132
|
+
staleWarnAt = new Map();
|
|
133
|
+
/** Per-agent last inference activity (epoch ms), for /healthz + doctor tooling. */
|
|
134
|
+
lastInferenceAt = new Map();
|
|
123
135
|
processLoggingPersist;
|
|
124
136
|
processLoggingBroadcast;
|
|
125
137
|
activeStreams = new Map();
|
|
@@ -130,6 +142,23 @@ export class AgentFramework {
|
|
|
130
142
|
/** N consecutive failed inferences ⇒ the agent is treated as hard-down and
|
|
131
143
|
* escalated loudly to stderr. */
|
|
132
144
|
inferenceFailureEscalationThreshold = 3;
|
|
145
|
+
/** Per-agent count of consecutive refusal-driven rewinds in the current turn
|
|
146
|
+
* chain (reset when a turn completes without a refusal). Bounds the auto
|
|
147
|
+
* rewind loop — see refusalHandling + rewindTriggeringTurn. */
|
|
148
|
+
refusalRewinds = new Map();
|
|
149
|
+
/** Per-agent count of poison-history rewinds performed by the hard-down
|
|
150
|
+
* breaker (noteInferenceExhausted). Reset on any successful inference.
|
|
151
|
+
* Bounds the automatic quarantine loop the same way refusalRewinds bounds
|
|
152
|
+
* the refusal loop, so the breaker can never shed the whole history. */
|
|
153
|
+
exhaustionRewinds = new Map();
|
|
154
|
+
/** Per-agent current rewind episode: the single consolidated marker's id and
|
|
155
|
+
* how many turns have been shed so far. One marker per episode, updated in
|
|
156
|
+
* place (see updateRewindMarker); cleared when the episode ends. */
|
|
157
|
+
rewindEpisode = new Map();
|
|
158
|
+
/** Active `/unstick` sessions: an admin-forced rewind-until-clean loop that
|
|
159
|
+
* runs even when the agent's autoRewind toggle is off. Tracks the remaining
|
|
160
|
+
* budget, what was shed (for the report), and where to post the outcome. */
|
|
161
|
+
forcedRewind = new Map();
|
|
133
162
|
/** Name of the primary (non-ephemeral) agent for routing framework messages. */
|
|
134
163
|
primaryAgentName = null;
|
|
135
164
|
// Messages deferred while an agent is waiting_for_tools (to preserve
|
|
@@ -171,6 +200,13 @@ export class AgentFramework {
|
|
|
171
200
|
mcplPrefixMap = new Map();
|
|
172
201
|
/** Maps serverId → McplServerConfig for prefix lookup. */
|
|
173
202
|
mcplServerConfigs = new Map();
|
|
203
|
+
/** Host capabilities advertised during the MCP handshake — stored so servers
|
|
204
|
+
* can be connected at runtime (connectMcplServer) after initialization. */
|
|
205
|
+
mcplHostCapabilities = null;
|
|
206
|
+
/** Inference routing policy from FrameworkConfig — stored so the MCPL
|
|
207
|
+
* subsystem can be lazily initialized by connectMcplServer when the
|
|
208
|
+
* framework started with zero configured servers. */
|
|
209
|
+
mcplInferenceRoutingConfig = null;
|
|
174
210
|
// EventGate (null when FrameworkConfig.gate is omitted)
|
|
175
211
|
eventGate = null;
|
|
176
212
|
// Session-level token usage tracking (always-on)
|
|
@@ -237,11 +273,14 @@ export class AgentFramework {
|
|
|
237
273
|
catch {
|
|
238
274
|
// Already registered
|
|
239
275
|
}
|
|
276
|
+
// The legacy single-map checkpoint state (TURN_CHECKPOINTS_ID) is no longer
|
|
277
|
+
// registered for new stores — it's read-only fallback data in old ones.
|
|
240
278
|
try {
|
|
241
|
-
store.registerState({ id:
|
|
279
|
+
store.registerState({ id: TURN_CHECKPOINTS_TREE_ID, strategy: 'tree' });
|
|
242
280
|
}
|
|
243
|
-
catch {
|
|
244
|
-
|
|
281
|
+
catch (error) {
|
|
282
|
+
if (!isStateExistsError(error))
|
|
283
|
+
throw error;
|
|
245
284
|
}
|
|
246
285
|
// Process logging config (default: disabled)
|
|
247
286
|
const processLoggingPersist = config.processLogging?.persist ?? false;
|
|
@@ -327,6 +366,9 @@ export class AgentFramework {
|
|
|
327
366
|
});
|
|
328
367
|
framework.livenessWatchdog.start();
|
|
329
368
|
}
|
|
369
|
+
// Stored for lazy MCPL initialization (connectMcplServer on a framework
|
|
370
|
+
// that started with zero configured servers).
|
|
371
|
+
framework.mcplInferenceRoutingConfig = config.inferenceRouting ?? null;
|
|
330
372
|
// Initialize MCPL subsystems if configured
|
|
331
373
|
if (config.mcplServers && config.mcplServers.length > 0) {
|
|
332
374
|
// Validate tool prefixes: no collisions with module names or between servers
|
|
@@ -345,6 +387,30 @@ export class AgentFramework {
|
|
|
345
387
|
}
|
|
346
388
|
await framework.initializeMcpl(config.mcplServers, config.inferenceRouting);
|
|
347
389
|
}
|
|
390
|
+
// Diagnostics: `kill -USR2 <pid>` dumps live wake/inference state to stderr
|
|
391
|
+
// (journal) without a restart — for catching the wake-wedge on the running
|
|
392
|
+
// process. Shows the gate's `inferring` set + buffered-event count (the
|
|
393
|
+
// wedge signature), active streams, and pending inference requests.
|
|
394
|
+
try {
|
|
395
|
+
process.on('SIGUSR2', () => {
|
|
396
|
+
try {
|
|
397
|
+
const gate = framework.eventGate?.inferenceDiagnostics() ?? null;
|
|
398
|
+
console.error('[diagnostics] ' + JSON.stringify({
|
|
399
|
+
at: new Date().toISOString(),
|
|
400
|
+
gate,
|
|
401
|
+
activeStreams: [...framework.activeStreams.keys()],
|
|
402
|
+
pendingRequests: framework.pendingRequests.length,
|
|
403
|
+
agents: [...framework.agents.keys()],
|
|
404
|
+
}));
|
|
405
|
+
}
|
|
406
|
+
catch (err) {
|
|
407
|
+
console.error('[diagnostics] dump failed:', err instanceof Error ? err.message : err);
|
|
408
|
+
}
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
catch {
|
|
412
|
+
// SIGUSR2 not available on this platform — non-fatal.
|
|
413
|
+
}
|
|
348
414
|
return framework;
|
|
349
415
|
}
|
|
350
416
|
/**
|
|
@@ -568,8 +634,9 @@ export class AgentFramework {
|
|
|
568
634
|
// Module gatherContext (fail-open, matches startAgentStream)
|
|
569
635
|
try {
|
|
570
636
|
const moduleInjections = await this.moduleRegistry.gatherContext(agentName);
|
|
571
|
-
|
|
572
|
-
|
|
637
|
+
const scopedModuleInjections = this.scopeInjectionsForAgent(agentName, moduleInjections);
|
|
638
|
+
if (scopedModuleInjections.length > 0) {
|
|
639
|
+
injections = scopedModuleInjections;
|
|
573
640
|
}
|
|
574
641
|
}
|
|
575
642
|
catch (error) {
|
|
@@ -580,7 +647,7 @@ export class AgentFramework {
|
|
|
580
647
|
if (this.hookOrchestrator) {
|
|
581
648
|
try {
|
|
582
649
|
const hookParams = this.buildBeforeInferenceParams(agent);
|
|
583
|
-
const hookInjections = await this.hookOrchestrator.beforeInference(hookParams);
|
|
650
|
+
const hookInjections = this.scopeInjectionsForAgent(agentName, await this.hookOrchestrator.beforeInference(hookParams));
|
|
584
651
|
if (hookInjections.length > 0) {
|
|
585
652
|
injections = injections ? [...injections, ...hookInjections] : hookInjections;
|
|
586
653
|
}
|
|
@@ -743,6 +810,7 @@ export class AgentFramework {
|
|
|
743
810
|
clearInterval(completionWatchdog);
|
|
744
811
|
this.offTrace(traceListener);
|
|
745
812
|
this.agents.delete(agent.name);
|
|
813
|
+
this.evictTurnCheckpoints(agent.name);
|
|
746
814
|
};
|
|
747
815
|
// Watchdog: if inference never starts within 30s, the agent is a zombie.
|
|
748
816
|
// This catches cases where the event loop stalled, the agent was
|
|
@@ -1080,6 +1148,181 @@ export class AgentFramework {
|
|
|
1080
1148
|
console.error('[inference-refusal] reaction failed:', err instanceof Error ? err.message : err);
|
|
1081
1149
|
}
|
|
1082
1150
|
}
|
|
1151
|
+
/**
|
|
1152
|
+
* Rewind the single turn that fed a refused generation: redact the newest
|
|
1153
|
+
* message the agent did NOT author (a tool result, or an incoming message)
|
|
1154
|
+
* and inject a metadata-only marker in its place. The marker carries a
|
|
1155
|
+
* *description* of what was withheld (kind + size), never the content — so it
|
|
1156
|
+
* cannot itself re-trip the classifier — while the raw record survives in the
|
|
1157
|
+
* chronicle log for forensics. Returns a record describing the removal, or
|
|
1158
|
+
* null if there is nothing eligible to rewind (e.g. only the agent's own
|
|
1159
|
+
* turns, or a prior system marker, remain).
|
|
1160
|
+
*
|
|
1161
|
+
* Shared by the auto path (on `stop_reason: refusal`) and the `/rewind`
|
|
1162
|
+
* host command.
|
|
1163
|
+
*/
|
|
1164
|
+
shedNewestTurn(agent) {
|
|
1165
|
+
const cm = agent.getContextManager();
|
|
1166
|
+
const all = cm.getAllMessages();
|
|
1167
|
+
const typeOf = (b) => b.type;
|
|
1168
|
+
const hasBlock = (m, t) => Array.isArray(m?.content) && m.content.some((b) => typeOf(b) === t);
|
|
1169
|
+
// Newest message that is not our own episode marker. We shed strictly
|
|
1170
|
+
// newest-first, in sequence — INCLUDING the agent's own turns (poison lives
|
|
1171
|
+
// in tool_use/narration turns too), because newest-first reaches whatever is
|
|
1172
|
+
// poisoning the context if allowed to run deep enough. The one invariant:
|
|
1173
|
+
// shed COMPLETE exchanges. Removing a `tool_result` also removes its paired
|
|
1174
|
+
// `tool_use` assistant turn, so we never leave an orphaned tool_use / signed
|
|
1175
|
+
// `thinking` block — which the API rejects with a 400 ("thinking blocks in
|
|
1176
|
+
// the latest assistant message cannot be modified"). We do NOT add a marker
|
|
1177
|
+
// here; the caller keeps one consolidated marker (updateRewindMarker).
|
|
1178
|
+
let idx = -1;
|
|
1179
|
+
for (let i = all.length - 1; i >= 0; i--) {
|
|
1180
|
+
const meta = (all[i].metadata ?? {});
|
|
1181
|
+
if (meta.system)
|
|
1182
|
+
continue;
|
|
1183
|
+
idx = i;
|
|
1184
|
+
break;
|
|
1185
|
+
}
|
|
1186
|
+
if (idx < 0)
|
|
1187
|
+
return null;
|
|
1188
|
+
const msg = all[idx];
|
|
1189
|
+
const md = (msg.metadata ?? {});
|
|
1190
|
+
const content = Array.isArray(msg.content) ? msg.content : [];
|
|
1191
|
+
const toolBlocks = content.filter((b) => typeOf(b) === 'tool_result');
|
|
1192
|
+
const images = content.filter((b) => typeOf(b) === 'image').length;
|
|
1193
|
+
const textLen = content
|
|
1194
|
+
.filter((b) => typeOf(b) === 'text')
|
|
1195
|
+
.reduce((n, b) => n + String(b.text ?? '').length, 0);
|
|
1196
|
+
// Determine the complete exchange to remove.
|
|
1197
|
+
const removedIds = [msg.id];
|
|
1198
|
+
let kind;
|
|
1199
|
+
let descriptor;
|
|
1200
|
+
if (toolBlocks.length > 0) {
|
|
1201
|
+
kind = 'tool';
|
|
1202
|
+
// Pair the tool_result with the tool_use assistant turn right before it.
|
|
1203
|
+
if (idx - 1 >= 0 && hasBlock(all[idx - 1], 'tool_use')) {
|
|
1204
|
+
removedIds.push(all[idx - 1].id);
|
|
1205
|
+
}
|
|
1206
|
+
const sz = toolBlocks.reduce((n, b) => n + String(b.content ?? '').length, 0);
|
|
1207
|
+
descriptor = `a tool exchange (~${Math.max(1, Math.round(sz / 1024))}KB` +
|
|
1208
|
+
`${images ? `, ${images} image(s)` : ''})`;
|
|
1209
|
+
}
|
|
1210
|
+
else if (md.messageId) {
|
|
1211
|
+
kind = 'human';
|
|
1212
|
+
descriptor = `an incoming message from ${msg.participant} ` +
|
|
1213
|
+
`(${textLen} chars${images ? `, ${images} image(s)` : ''})`;
|
|
1214
|
+
}
|
|
1215
|
+
else {
|
|
1216
|
+
kind = 'other';
|
|
1217
|
+
descriptor = `a ${msg.participant} turn ` +
|
|
1218
|
+
`(${textLen} chars${images ? `, ${images} image(s)` : ''})`;
|
|
1219
|
+
}
|
|
1220
|
+
const discordRef = md.messageId && md.channelId
|
|
1221
|
+
? { channelId: String(md.channelId), messageId: String(md.messageId) }
|
|
1222
|
+
: undefined;
|
|
1223
|
+
for (const id of removedIds)
|
|
1224
|
+
cm.removeMessage(id);
|
|
1225
|
+
return { kind, descriptor, removedIds, discordRef };
|
|
1226
|
+
}
|
|
1227
|
+
/**
|
|
1228
|
+
* Maintain exactly ONE consolidated marker for the current rewind episode,
|
|
1229
|
+
* updated in place as more turns are shed. Six rewinds ⇒ one message that
|
|
1230
|
+
* says "the 6 most recent turns were set aside", not six separate notes — so
|
|
1231
|
+
* the context converges (shed N, add 1) instead of growing, and the marker
|
|
1232
|
+
* sits at the tail giving the model something actionable to answer once the
|
|
1233
|
+
* refusal clears. Returns the running count.
|
|
1234
|
+
*/
|
|
1235
|
+
updateRewindMarker(agent, category, cause = 'refusal') {
|
|
1236
|
+
const cm = agent.getContextManager();
|
|
1237
|
+
const ep = this.rewindEpisode.get(agent.name);
|
|
1238
|
+
const count = (ep?.count ?? 0) + 1;
|
|
1239
|
+
const why = cause === 'refusal'
|
|
1240
|
+
? `the model refused on them (content filter: ${category}). Their content is ` +
|
|
1241
|
+
`not reproduced here (so this note can't re-trigger the filter); the ` +
|
|
1242
|
+
`originals remain in the raw record`
|
|
1243
|
+
: `the model API kept rejecting the conversation on them (${category}). ` +
|
|
1244
|
+
`Their content is not reproduced here (so this note can't re-trigger the ` +
|
|
1245
|
+
`rejection); the originals remain in the raw record`;
|
|
1246
|
+
const text = `[refusal-rewind] The ${count} most recent turn(s) were set aside because ` +
|
|
1247
|
+
`${why}. You are clear to continue — if you ` +
|
|
1248
|
+
`were mid-task, take a different approach; otherwise carry on with whatever ` +
|
|
1249
|
+
`is now in front of you, or briefly acknowledge the gap and ask what's next.`;
|
|
1250
|
+
const blocks = [{ type: 'text', text }];
|
|
1251
|
+
if (ep) {
|
|
1252
|
+
cm.editMessage(ep.markerId, blocks);
|
|
1253
|
+
this.rewindEpisode.set(agent.name, { markerId: ep.markerId, count, category });
|
|
1254
|
+
}
|
|
1255
|
+
else {
|
|
1256
|
+
const id = cm.addMessage('user', blocks, {
|
|
1257
|
+
system: true, kind: 'refusal-rewind', category, count, cause,
|
|
1258
|
+
});
|
|
1259
|
+
this.rewindEpisode.set(agent.name, { markerId: id, count, category });
|
|
1260
|
+
}
|
|
1261
|
+
return count;
|
|
1262
|
+
}
|
|
1263
|
+
/**
|
|
1264
|
+
* Announce a refusal-rewind on the conversational surface (Discord), used
|
|
1265
|
+
* when the withheld turn was a *human* message so it isn't dropped silently.
|
|
1266
|
+
* Best-effort; mirrors reactToRefusal's locus resolution.
|
|
1267
|
+
*/
|
|
1268
|
+
async announceRewind(agentName, rec, category) {
|
|
1269
|
+
try {
|
|
1270
|
+
const incoming = this.channelRegistry?.buildChannelContext()?.incoming;
|
|
1271
|
+
if (!incoming)
|
|
1272
|
+
return;
|
|
1273
|
+
const parts = incoming.channelId.split(':');
|
|
1274
|
+
if (parts[0] !== 'discord')
|
|
1275
|
+
return;
|
|
1276
|
+
const channelId = parts[parts.length - 1];
|
|
1277
|
+
const serverId = this.channelRegistry?.getChannelServerId(incoming.channelId);
|
|
1278
|
+
const server = serverId ? this.mcplServerRegistry?.getServer(serverId) : null;
|
|
1279
|
+
if (!server)
|
|
1280
|
+
return;
|
|
1281
|
+
await server.sendToolsCall('send_message', {
|
|
1282
|
+
channelId,
|
|
1283
|
+
content: `⚠️ I had to set aside ${rec.descriptor} — it tripped a content ` +
|
|
1284
|
+
`filter (${category}), so it's withheld from my context and I'm ` +
|
|
1285
|
+
`continuing without it. If it was important, please rephrase or re-send.`,
|
|
1286
|
+
});
|
|
1287
|
+
}
|
|
1288
|
+
catch (err) {
|
|
1289
|
+
console.error('[refusal-rewind] announce failed:', err instanceof Error ? err.message : err);
|
|
1290
|
+
}
|
|
1291
|
+
}
|
|
1292
|
+
/**
|
|
1293
|
+
* Conclude an active `/unstick` session: post the outcome (what was shed and
|
|
1294
|
+
* whether the model stopped refusing) to the channel the command came from,
|
|
1295
|
+
* then clear the session. Idempotent — a no-op if there's no session.
|
|
1296
|
+
*/
|
|
1297
|
+
finishUnstick(agentName, success, category) {
|
|
1298
|
+
const s = this.forcedRewind.get(agentName);
|
|
1299
|
+
if (!s)
|
|
1300
|
+
return;
|
|
1301
|
+
this.forcedRewind.delete(agentName);
|
|
1302
|
+
const n = s.removed.length;
|
|
1303
|
+
const list = n
|
|
1304
|
+
? '\n' + s.removed.map((r) => `• ${r.descriptor}`).join('\n')
|
|
1305
|
+
: '';
|
|
1306
|
+
const content = success
|
|
1307
|
+
? `🔧 Unstuck **${agentName}** — shed ${n} turn(s); the model responded.${list}`
|
|
1308
|
+
: `⚠️ Couldn't unstick **${agentName}** — still refusing after ${n} rewind(s)` +
|
|
1309
|
+
`${category ? ` (category=${category})` : ''}.${list}`;
|
|
1310
|
+
console.error(`[unstick] agent=${agentName} ${success ? 'succeeded' : 'gave up'} after ${n} rewind(s)`);
|
|
1311
|
+
if (s.channelId)
|
|
1312
|
+
void this.postToChannel(s.serverId, s.channelId, content);
|
|
1313
|
+
}
|
|
1314
|
+
/** Best-effort send_message to a raw channel via a named MCPL server. */
|
|
1315
|
+
async postToChannel(serverId, channelId, content) {
|
|
1316
|
+
try {
|
|
1317
|
+
const server = this.mcplServerRegistry?.getServer(serverId);
|
|
1318
|
+
if (!server)
|
|
1319
|
+
return;
|
|
1320
|
+
await server.sendToolsCall('send_message', { channelId, content });
|
|
1321
|
+
}
|
|
1322
|
+
catch (err) {
|
|
1323
|
+
console.error('[unstick] report post failed:', err instanceof Error ? err.message : err);
|
|
1324
|
+
}
|
|
1325
|
+
}
|
|
1083
1326
|
/**
|
|
1084
1327
|
* Handle a `host/command` request from an MCPL surface server (e.g. a
|
|
1085
1328
|
* Discord slash command). Currently supports:
|
|
@@ -1090,13 +1333,41 @@ export class AgentFramework {
|
|
|
1090
1333
|
* `previewActivation` render (no inference, no Chronicle writes).
|
|
1091
1334
|
*/
|
|
1092
1335
|
async handleHostCommand(serverId, params) {
|
|
1093
|
-
if (params.command !== 'undo' && params.command !== 'hide') {
|
|
1336
|
+
if (params.command !== 'undo' && params.command !== 'hide' && params.command !== 'unstick') {
|
|
1094
1337
|
return { ok: false, error: `Unknown host command: ${String(params.command)}` };
|
|
1095
1338
|
}
|
|
1096
1339
|
const agentName = params.agentName ?? [...this.agents.keys()][0];
|
|
1097
1340
|
if (!agentName || !this.agents.has(agentName)) {
|
|
1098
1341
|
return { ok: false, error: `Unknown agent: ${String(agentName)}` };
|
|
1099
1342
|
}
|
|
1343
|
+
// unstick: force the refusal-rewind loop on demand (even if the agent's
|
|
1344
|
+
// autoRewind toggle is off). Redacts the turn that fed the refusal and
|
|
1345
|
+
// re-runs, up to `cap` times, until the model stops refusing. Kicks the
|
|
1346
|
+
// loop and returns immediately; the outcome (what was shed + whether it
|
|
1347
|
+
// cleared) is posted to the channel when the chain resolves.
|
|
1348
|
+
if (params.command === 'unstick') {
|
|
1349
|
+
const agent = this.agents.get(agentName);
|
|
1350
|
+
if (agent.state.status !== 'idle') {
|
|
1351
|
+
return { ok: false, error: `Cannot unstick while agent is ${agent.state.status}` };
|
|
1352
|
+
}
|
|
1353
|
+
const cap = Math.max(1, Math.min(10, Math.floor(params.maxRewinds ?? agent.refusalHandling?.maxRewinds ?? 3)));
|
|
1354
|
+
this.forcedRewind.set(agentName, {
|
|
1355
|
+
remaining: cap,
|
|
1356
|
+
removed: [],
|
|
1357
|
+
serverId,
|
|
1358
|
+
channelId: params.channelId ?? '',
|
|
1359
|
+
});
|
|
1360
|
+
this.refusalRewinds.set(agentName, 0);
|
|
1361
|
+
this.pendingRequests.push({
|
|
1362
|
+
agentName,
|
|
1363
|
+
reason: 'unstick',
|
|
1364
|
+
source: 'framework',
|
|
1365
|
+
timestamp: Date.now(),
|
|
1366
|
+
});
|
|
1367
|
+
console.error(`[unstick] agent=${agentName} started cap=${cap} ` +
|
|
1368
|
+
`by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`);
|
|
1369
|
+
return { ok: true, started: true, cap };
|
|
1370
|
+
}
|
|
1100
1371
|
// hide: redact a single message (or an inclusive range) from the active
|
|
1101
1372
|
// branch, addressed by Discord message id. Unlike undo this is a
|
|
1102
1373
|
// removal-in-place (chronicle redact), not a branch rewind.
|
|
@@ -1615,18 +1886,86 @@ export class AgentFramework {
|
|
|
1615
1886
|
}
|
|
1616
1887
|
this.saveTurnCheckpoints(agentName, checkpoints);
|
|
1617
1888
|
}
|
|
1889
|
+
/**
|
|
1890
|
+
* Checkpoints live in one Tree state (TURN_CHECKPOINTS_TREE_ID) keyed by agent
|
|
1891
|
+
* name, each key pointing at a JSON blob of that agent's ≤MAX_TURN_CHECKPOINTS
|
|
1892
|
+
* list. This kills two unbounded-in-agents-ever dimensions at once:
|
|
1893
|
+
*
|
|
1894
|
+
* - the original single Record<agentName, list> map was rewritten whole on
|
|
1895
|
+
* every turn of every agent and never evicted a departed subagent's key;
|
|
1896
|
+
* - the intermediate design (one state slot per agent) fixed the writes but
|
|
1897
|
+
* leaked a permanent chronicle state *registration* per spawn — chronicle
|
|
1898
|
+
* has no deregistration, and the state index is rewritten and fsynced on
|
|
1899
|
+
* every sync tick, so per-tick cost grew with fleet history anyway.
|
|
1900
|
+
*
|
|
1901
|
+
* A tree gives per-key O(entry) writes, real key removal (treeRemove), and
|
|
1902
|
+
* exactly one registration for the life of the store. The legacy map state
|
|
1903
|
+
* (TURN_CHECKPOINTS_ID) is kept as a read-only fallback for stores written
|
|
1904
|
+
* before the split; eviction tombstones legacy keys so a reused agent name
|
|
1905
|
+
* can't inherit a dead agent's checkpoints through the fallback.
|
|
1906
|
+
*/
|
|
1907
|
+
legacyCheckpointKeys = null;
|
|
1908
|
+
hasLegacyCheckpoints(agentName) {
|
|
1909
|
+
if (!this.legacyCheckpointKeys) {
|
|
1910
|
+
const legacy = this.store.getStateJson(TURN_CHECKPOINTS_ID);
|
|
1911
|
+
this.legacyCheckpointKeys = new Set(legacy && typeof legacy === 'object' ? Object.keys(legacy) : []);
|
|
1912
|
+
}
|
|
1913
|
+
return this.legacyCheckpointKeys.has(agentName);
|
|
1914
|
+
}
|
|
1618
1915
|
getTurnCheckpoints(agentName) {
|
|
1619
|
-
const
|
|
1620
|
-
if (
|
|
1916
|
+
const entry = this.store.treeGet(TURN_CHECKPOINTS_TREE_ID, agentName);
|
|
1917
|
+
if (entry) {
|
|
1918
|
+
const blob = this.store.getBlob(entry.blobHash);
|
|
1919
|
+
if (!blob)
|
|
1920
|
+
return [];
|
|
1921
|
+
try {
|
|
1922
|
+
const parsed = JSON.parse(blob.toString());
|
|
1923
|
+
return Array.isArray(parsed) ? parsed : [];
|
|
1924
|
+
}
|
|
1925
|
+
catch {
|
|
1926
|
+
return [];
|
|
1927
|
+
}
|
|
1928
|
+
}
|
|
1929
|
+
if (!this.hasLegacyCheckpoints(agentName))
|
|
1621
1930
|
return [];
|
|
1622
|
-
const
|
|
1623
|
-
|
|
1931
|
+
const legacy = this.store.getStateJson(TURN_CHECKPOINTS_ID);
|
|
1932
|
+
const list = legacy?.[agentName];
|
|
1933
|
+
return Array.isArray(list) ? [...list] : [];
|
|
1624
1934
|
}
|
|
1625
1935
|
saveTurnCheckpoints(agentName, checkpoints) {
|
|
1626
|
-
const
|
|
1627
|
-
const
|
|
1628
|
-
|
|
1629
|
-
|
|
1936
|
+
const bytes = Buffer.from(JSON.stringify(checkpoints));
|
|
1937
|
+
const blobHash = this.store.storeBlob(bytes, 'application/json');
|
|
1938
|
+
this.store.treeSet(TURN_CHECKPOINTS_TREE_ID, agentName, {
|
|
1939
|
+
blobHash,
|
|
1940
|
+
size: bytes.length,
|
|
1941
|
+
mode: 0o644,
|
|
1942
|
+
});
|
|
1943
|
+
}
|
|
1944
|
+
/**
|
|
1945
|
+
* Drop a departed agent's checkpoint tree key and turn/redo bookkeeping.
|
|
1946
|
+
* Without this, spawn-and-dispose agents leave a key (and in-memory map
|
|
1947
|
+
* entries) behind for the life of the store/session.
|
|
1948
|
+
*/
|
|
1949
|
+
evictTurnCheckpoints(agentName) {
|
|
1950
|
+
this.turnCounters.delete(agentName);
|
|
1951
|
+
this.redoStacks.delete(agentName);
|
|
1952
|
+
// Diagnostics maps are also keyed by agent name and never evicted —
|
|
1953
|
+
// spawn-and-dispose fleets would grow them for the session's life.
|
|
1954
|
+
this.staleWarnAt.delete(agentName);
|
|
1955
|
+
this.lastInferenceAt.delete(agentName);
|
|
1956
|
+
const entry = this.store.treeGet(TURN_CHECKPOINTS_TREE_ID, agentName);
|
|
1957
|
+
if (this.hasLegacyCheckpoints(agentName)) {
|
|
1958
|
+
// A bare treeRemove would resurrect the legacy map's list through the
|
|
1959
|
+
// fallback next time this name is reused — shadow it with an empty
|
|
1960
|
+
// tombstone instead. size <= 2 ⇔ the blob is already "[]".
|
|
1961
|
+
if (!entry || entry.size > 2) {
|
|
1962
|
+
this.saveTurnCheckpoints(agentName, []);
|
|
1963
|
+
}
|
|
1964
|
+
return;
|
|
1965
|
+
}
|
|
1966
|
+
if (entry) {
|
|
1967
|
+
this.store.treeRemove(TURN_CHECKPOINTS_TREE_ID, agentName);
|
|
1968
|
+
}
|
|
1630
1969
|
}
|
|
1631
1970
|
/**
|
|
1632
1971
|
* Run until the queue is empty and all agents are idle.
|
|
@@ -1634,6 +1973,9 @@ export class AgentFramework {
|
|
|
1634
1973
|
*/
|
|
1635
1974
|
async runUntilIdle() {
|
|
1636
1975
|
while (!this.queue.isEmpty ||
|
|
1976
|
+
// Direct inference requests (e.g. runEphemeralToCompletion) bypass the
|
|
1977
|
+
// event queue — without this the loop can exit before they're drained.
|
|
1978
|
+
this.pendingRequests.length > 0 ||
|
|
1637
1979
|
this.activeStreams.size > 0 ||
|
|
1638
1980
|
Array.from(this.agents.values()).some((a) => a.state.status !== 'idle')) {
|
|
1639
1981
|
await this.processNextEvent();
|
|
@@ -2083,6 +2425,7 @@ export class AgentFramework {
|
|
|
2083
2425
|
this.agents.delete(agentName);
|
|
2084
2426
|
this.agentConfigs.delete(agentName);
|
|
2085
2427
|
this.conversationAgentHomes.delete(agentName);
|
|
2428
|
+
this.evictTurnCheckpoints(agentName);
|
|
2086
2429
|
this.emitTrace({
|
|
2087
2430
|
type: 'mcpl:conversation-disposed',
|
|
2088
2431
|
agentName,
|
|
@@ -2257,6 +2600,7 @@ export class AgentFramework {
|
|
|
2257
2600
|
requestCount: requests.length,
|
|
2258
2601
|
oldestRequestAge: now - oldest,
|
|
2259
2602
|
});
|
|
2603
|
+
console.error(`[inference-dropped] agent=${agentName} reason=agent_not_found requests=${requests.length}`);
|
|
2260
2604
|
continue;
|
|
2261
2605
|
}
|
|
2262
2606
|
// Skip if agent is busy (inferring, streaming, or waiting for tools)
|
|
@@ -2271,12 +2615,23 @@ export class AgentFramework {
|
|
|
2271
2615
|
requestCount: requests.length,
|
|
2272
2616
|
oldestRequestAge: now - oldest,
|
|
2273
2617
|
});
|
|
2618
|
+
// Loud (but throttled) note that requests are waiting on a busy agent.
|
|
2619
|
+
if ((this.staleWarnAt.get(agentName) ?? 0) < now - 60_000) {
|
|
2620
|
+
this.staleWarnAt.set(agentName, now);
|
|
2621
|
+
console.error(`[inference-stale] agent=${agentName} busy (${agent.state.status}) — ` +
|
|
2622
|
+
`${requests.length} request(s) waiting ${Math.round((now - oldest) / 1000)}s`);
|
|
2623
|
+
}
|
|
2274
2624
|
}
|
|
2275
2625
|
this.pendingRequests.push(...requests);
|
|
2276
2626
|
continue;
|
|
2277
2627
|
}
|
|
2278
2628
|
// Check policy
|
|
2279
2629
|
if (!this.inferencePolicy.shouldInfer(agentName, requests, state)) {
|
|
2630
|
+
// Loud drop: a queued request that dies here is otherwise invisible —
|
|
2631
|
+
// the 2026-07-09 mythos "not responding" diagnosis burned hours on
|
|
2632
|
+
// exactly this class of silent drop. One stderr line per drop.
|
|
2633
|
+
console.error(`[inference-dropped] agent=${agentName} reason=policy-skip ` +
|
|
2634
|
+
`requests=${requests.length} triggers=${requests.map((r) => r.reason).join(',')}`);
|
|
2280
2635
|
continue;
|
|
2281
2636
|
}
|
|
2282
2637
|
// Start streaming inference (non-blocking — driveStream runs in background)
|
|
@@ -2313,20 +2668,20 @@ export class AgentFramework {
|
|
|
2313
2668
|
}
|
|
2314
2669
|
this.emitTrace({ type: 'inference:started', agentName: agent.name });
|
|
2315
2670
|
this.eventGate?.onInferenceStarted(agent.name);
|
|
2671
|
+
this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), startedAt: Date.now() });
|
|
2316
2672
|
try {
|
|
2317
2673
|
const allTools = this.getAllTools();
|
|
2318
2674
|
const tools = allTools.filter((t) => agent.canUseTool(t.name));
|
|
2319
2675
|
// Gather context from modules (pull-based) and MCPL hooks (push-based)
|
|
2320
2676
|
// Both produce ContextInjection[] that get merged before inference.
|
|
2321
2677
|
let injections;
|
|
2322
|
-
// Module gatherContext (fail-open,
|
|
2323
|
-
//
|
|
2324
|
-
//
|
|
2325
|
-
//
|
|
2326
|
-
//
|
|
2327
|
-
// agent itself, or conversation forks will see cross-channel content.
|
|
2678
|
+
// Module gatherContext (fail-open, per-module timeout — the module's
|
|
2679
|
+
// contextTimeoutMs, else the registry default). Injections are
|
|
2680
|
+
// channel-scoped via scopeInjectionsForAgent, matching the MCPL hook
|
|
2681
|
+
// injections below, so conversation forks don't see cross-channel
|
|
2682
|
+
// content.
|
|
2328
2683
|
try {
|
|
2329
|
-
const moduleInjections = await this.moduleRegistry.gatherContext(agent.name);
|
|
2684
|
+
const moduleInjections = this.scopeInjectionsForAgent(agent.name, await this.moduleRegistry.gatherContext(agent.name));
|
|
2330
2685
|
if (moduleInjections.length > 0) {
|
|
2331
2686
|
injections = moduleInjections;
|
|
2332
2687
|
}
|
|
@@ -2370,6 +2725,13 @@ export class AgentFramework {
|
|
|
2370
2725
|
type: 'inference:exhausted',
|
|
2371
2726
|
agentName: agent.name,
|
|
2372
2727
|
error: err.message,
|
|
2728
|
+
// Drives the poison-history breaker. `retryable` is kept for
|
|
2729
|
+
// observability, but the breaker gates on `errorType` — only an
|
|
2730
|
+
// `invalid_request` (a 400-class rejection of the history itself)
|
|
2731
|
+
// means retrying the same context can never succeed. auth/abort/
|
|
2732
|
+
// context_length are also non-retryable but must NOT cost history.
|
|
2733
|
+
retryable: err instanceof MembraneError ? err.retryable : undefined,
|
|
2734
|
+
errorType: err instanceof MembraneError ? err.type : undefined,
|
|
2373
2735
|
});
|
|
2374
2736
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
2375
2737
|
if (action.emit) {
|
|
@@ -2608,7 +2970,73 @@ export class AgentFramework {
|
|
|
2608
2970
|
const category = stopDetails?.category ?? 'unknown';
|
|
2609
2971
|
console.error(`[inference-refusal] agent=${agent.name} category=${category}` +
|
|
2610
2972
|
(stopDetails?.explanation ? ` explanation=${stopDetails.explanation}` : ''));
|
|
2611
|
-
|
|
2973
|
+
// Rewind: excise the turn that fed the refusal, drop a
|
|
2974
|
+
// metadata-only marker, and retry — keeping the agent on its own
|
|
2975
|
+
// model instead of substituting a fallback. Driven either by an
|
|
2976
|
+
// admin `/unstick` (forced session) or the agent's autoRewind
|
|
2977
|
+
// config; both are bounded.
|
|
2978
|
+
let handledByRewind = false;
|
|
2979
|
+
const rh = agent.refusalHandling;
|
|
2980
|
+
const forced = this.forcedRewind.get(agent.name);
|
|
2981
|
+
// Shed exactly one more (newest, in-sequence) turn and keep the
|
|
2982
|
+
// single episode marker current. `budgetLeft` bounds the loop.
|
|
2983
|
+
const doRewind = (budgetLeft, onStep, onGiveUp) => {
|
|
2984
|
+
if (!budgetLeft) {
|
|
2985
|
+
onGiveUp();
|
|
2986
|
+
return;
|
|
2987
|
+
}
|
|
2988
|
+
const rec = this.shedNewestTurn(agent);
|
|
2989
|
+
if (!rec) {
|
|
2990
|
+
onGiveUp();
|
|
2991
|
+
return;
|
|
2992
|
+
}
|
|
2993
|
+
const count = this.updateRewindMarker(agent, category);
|
|
2994
|
+
onStep(count);
|
|
2995
|
+
if (forced)
|
|
2996
|
+
forced.removed.push(rec);
|
|
2997
|
+
this.pendingRequests.push({
|
|
2998
|
+
agentName: agent.name,
|
|
2999
|
+
reason: forced ? 'unstick-retry' : 'refusal-rewind-retry',
|
|
3000
|
+
source: 'framework',
|
|
3001
|
+
timestamp: Date.now(),
|
|
3002
|
+
});
|
|
3003
|
+
handledByRewind = true;
|
|
3004
|
+
};
|
|
3005
|
+
if (forced) {
|
|
3006
|
+
doRewind(forced.remaining > 0, (count) => {
|
|
3007
|
+
forced.remaining -= 1;
|
|
3008
|
+
console.error(`[unstick] agent=${agent.name} shed ${count} turn(s) ` +
|
|
3009
|
+
`(remaining ${forced.remaining})`);
|
|
3010
|
+
}, () => this.finishUnstick(agent.name, false, category));
|
|
3011
|
+
}
|
|
3012
|
+
else if (rh?.autoRewind) {
|
|
3013
|
+
const cap = Math.max(1, rh.maxRewinds ?? 3);
|
|
3014
|
+
const used = this.refusalRewinds.get(agent.name) ?? 0;
|
|
3015
|
+
doRewind(used < cap, (count) => {
|
|
3016
|
+
this.refusalRewinds.set(agent.name, used + 1);
|
|
3017
|
+
console.error(`[refusal-rewind] agent=${agent.name} shed ${count} turn(s) so far ` +
|
|
3018
|
+
`(cap ${cap})`);
|
|
3019
|
+
}, () => {
|
|
3020
|
+
console.error(`[refusal-rewind] agent=${agent.name} gave up: ` +
|
|
3021
|
+
`${used >= cap ? `cap ${cap} reached` : 'nothing left to shed'}`);
|
|
3022
|
+
this.refusalRewinds.set(agent.name, 0);
|
|
3023
|
+
this.rewindEpisode.delete(agent.name);
|
|
3024
|
+
});
|
|
3025
|
+
}
|
|
3026
|
+
if (!handledByRewind)
|
|
3027
|
+
void this.reactToRefusal(agent.name, category);
|
|
3028
|
+
}
|
|
3029
|
+
else {
|
|
3030
|
+
// A turn that completed WITHOUT a refusal ends the rewind episode:
|
|
3031
|
+
// the model responded. Leave the consolidated marker in place as
|
|
3032
|
+
// the durable record; just clear the per-episode counters.
|
|
3033
|
+
if (this.forcedRewind.has(agent.name)) {
|
|
3034
|
+
this.finishUnstick(agent.name, true);
|
|
3035
|
+
}
|
|
3036
|
+
if (this.refusalRewinds.get(agent.name)) {
|
|
3037
|
+
this.refusalRewinds.set(agent.name, 0);
|
|
3038
|
+
}
|
|
3039
|
+
this.rewindEpisode.delete(agent.name);
|
|
2612
3040
|
}
|
|
2613
3041
|
// Dispatch speech (and thoughts if any)
|
|
2614
3042
|
if (speechContent.length > 0 || thoughts.length > 0) {
|
|
@@ -2692,11 +3120,19 @@ export class AgentFramework {
|
|
|
2692
3120
|
`(${silenced ? 'silencing tool / explicit send' : 'no prose'})`);
|
|
2693
3121
|
}
|
|
2694
3122
|
else {
|
|
2695
|
-
|
|
3123
|
+
// Snapshot the locus ONCE. This loop dispatches AFTER the agent
|
|
3124
|
+
// is idle (see PR #32 note below), so a queued inbound from
|
|
3125
|
+
// another channel can start a NEW turn and overwrite the per-agent
|
|
3126
|
+
// triggering channel (`activeTriggerChannels`) BETWEEN segments —
|
|
3127
|
+
// splitting one turn's reply across channels (the residual
|
|
3128
|
+
// multi-segment race left by 83dedf8's per-agent locus). Pin every
|
|
3129
|
+
// segment of THIS turn to the channel resolved right now.
|
|
3130
|
+
const turnLocus = this.channelRegistry.resolveLocus(agent.name);
|
|
3131
|
+
console.error(`[routing] ${agent.name}: tool-call turn [${toolNames.join(', ')}] -> routing ${segments.length} prose segment(s) in order -> ${turnLocus ?? '(default)'}`);
|
|
2696
3132
|
// Deliver sequentially (await each) so the segments land in order.
|
|
2697
3133
|
for (const seg of segments) {
|
|
2698
3134
|
try {
|
|
2699
|
-
await this.channelRegistry.routeSpeech(agent.name, seg);
|
|
3135
|
+
await this.channelRegistry.routeSpeech(agent.name, seg, turnLocus);
|
|
2700
3136
|
}
|
|
2701
3137
|
catch (err) {
|
|
2702
3138
|
console.error('speech routing failed:', err);
|
|
@@ -2742,6 +3178,12 @@ export class AgentFramework {
|
|
|
2742
3178
|
type: 'inference:exhausted',
|
|
2743
3179
|
agentName: agent.name,
|
|
2744
3180
|
error: err.message,
|
|
3181
|
+
// Drives the poison-history breaker. `retryable` is kept for
|
|
3182
|
+
// observability, but the breaker gates on `errorType` — only an
|
|
3183
|
+
// `invalid_request` (a 400-class rejection of the history
|
|
3184
|
+
// itself) means retrying the same context can never succeed.
|
|
3185
|
+
retryable: err instanceof MembraneError ? err.retryable : undefined,
|
|
3186
|
+
errorType: err instanceof MembraneError ? err.type : undefined,
|
|
2745
3187
|
});
|
|
2746
3188
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
2747
3189
|
if (action.emit) {
|
|
@@ -2755,18 +3197,51 @@ export class AgentFramework {
|
|
|
2755
3197
|
// Only reset if this is still the active stream (a budget restart
|
|
2756
3198
|
// may have already started a new stream, bumping streamId)
|
|
2757
3199
|
if (agent.streamId === myStreamId) {
|
|
3200
|
+
const durationMs = Date.now() - startTime;
|
|
2758
3201
|
agent.reset();
|
|
2759
3202
|
this.emitTrace({
|
|
2760
3203
|
type: 'inference:exhausted',
|
|
2761
3204
|
agentName: agent.name,
|
|
2762
3205
|
error: `Stream aborted: ${reason}`,
|
|
2763
3206
|
});
|
|
3207
|
+
// Postmortem 2026-05-28 P2 #7: persist the abort to the
|
|
3208
|
+
// inference log so future investigations can attribute the
|
|
3209
|
+
// terminal cause without relying on live in-memory reducer
|
|
3210
|
+
// state. Without this, abort-terminated inferences are
|
|
3211
|
+
// invisible to forensic queries (only request-side telemetry
|
|
3212
|
+
// via llm-calls.jsonl shows them, and only by absence).
|
|
3213
|
+
this.logInference({
|
|
3214
|
+
timestamp: startTime,
|
|
3215
|
+
agentName: agent.name,
|
|
3216
|
+
requestId,
|
|
3217
|
+
success: false,
|
|
3218
|
+
error: `Stream aborted: ${reason}`,
|
|
3219
|
+
request: compiledRequest ?? { note: 'streaming request aborted' },
|
|
3220
|
+
durationMs,
|
|
3221
|
+
});
|
|
2764
3222
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
2765
3223
|
}
|
|
2766
3224
|
break;
|
|
2767
3225
|
}
|
|
2768
|
-
case 'usage':
|
|
3226
|
+
case 'usage': {
|
|
2769
3227
|
agent.lastStreamInputTokens = event.usage.inputTokens;
|
|
3228
|
+
// Closed-loop estimator calibration (2026-07-12). Sample the REAL
|
|
3229
|
+
// prefix size of THIS API call (fresh + cache write + cache read)
|
|
3230
|
+
// and hand it to the context strategy, which accepts exactly one
|
|
3231
|
+
// sample per compile (its arm-once gate) and rejects out-of-band
|
|
3232
|
+
// ratios. It must be sampled HERE, per call: `response.details.
|
|
3233
|
+
// usage` at turn completion is CUMULATIVE across the tool-use
|
|
3234
|
+
// loop (5 calls x ~160k reported as 884k), which is not a
|
|
3235
|
+
// window-shaped number and drove the multiplier to 2.37 before
|
|
3236
|
+
// the guards caught it.
|
|
3237
|
+
try {
|
|
3238
|
+
const realTotal = (event.usage.inputTokens ?? 0) +
|
|
3239
|
+
(event.usage.cacheCreationTokens ?? 0) +
|
|
3240
|
+
(event.usage.cacheReadTokens ?? 0);
|
|
3241
|
+
const strat = agent.getContextManager?.()?.getStrategy?.();
|
|
3242
|
+
strat?.reportRealInputTokens?.(realTotal);
|
|
3243
|
+
}
|
|
3244
|
+
catch { /* calibration is best-effort */ }
|
|
2770
3245
|
this.emitTrace({
|
|
2771
3246
|
type: 'inference:usage',
|
|
2772
3247
|
agentName: agent.name,
|
|
@@ -2778,6 +3253,7 @@ export class AgentFramework {
|
|
|
2778
3253
|
},
|
|
2779
3254
|
});
|
|
2780
3255
|
break;
|
|
3256
|
+
}
|
|
2781
3257
|
}
|
|
2782
3258
|
}
|
|
2783
3259
|
}
|
|
@@ -2785,6 +3261,7 @@ export class AgentFramework {
|
|
|
2785
3261
|
// Stream itself threw (unexpected) — no retry path here, so also emit
|
|
2786
3262
|
// inference:exhausted so ephemeral agent promises can settle.
|
|
2787
3263
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
3264
|
+
const durationMs = Date.now() - startTime;
|
|
2788
3265
|
this.emitTrace({
|
|
2789
3266
|
type: 'inference:failed',
|
|
2790
3267
|
agentName: agent.name,
|
|
@@ -2796,10 +3273,31 @@ export class AgentFramework {
|
|
|
2796
3273
|
agentName: agent.name,
|
|
2797
3274
|
error: err.message,
|
|
2798
3275
|
});
|
|
3276
|
+
// Postmortem 2026-05-28 P2 #7: catch-path failures (stream itself
|
|
3277
|
+
// threw) were previously only visible via in-memory trace listeners.
|
|
3278
|
+
// Persist to inference log for forensic attribution.
|
|
3279
|
+
this.logInference({
|
|
3280
|
+
timestamp: startTime,
|
|
3281
|
+
agentName: agent.name,
|
|
3282
|
+
requestId,
|
|
3283
|
+
success: false,
|
|
3284
|
+
error: `Stream threw: ${err.message}`,
|
|
3285
|
+
request: compiledRequest ?? { note: 'streaming request threw' },
|
|
3286
|
+
durationMs,
|
|
3287
|
+
});
|
|
2799
3288
|
agent.reset();
|
|
2800
3289
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
2801
3290
|
}
|
|
2802
3291
|
finally {
|
|
3292
|
+
// Clear the gate's inference flag on EVERY exit path (paired with the
|
|
3293
|
+
// onInferenceStarted in startAgentStream). The branch-level calls above
|
|
3294
|
+
// are kept but are best-effort; if any exit path bypassed them the agent
|
|
3295
|
+
// stayed stuck in the gate's `inferring` set, which permanently BUFFERS
|
|
3296
|
+
// all incoming events → the agent silently stops waking on messages
|
|
3297
|
+
// (typing still stops, compression still runs — matching the observed
|
|
3298
|
+
// wedge). onInferenceEnded is idempotent, so a redundant call is safe.
|
|
3299
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
3300
|
+
this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), endedAt: Date.now() });
|
|
2803
3301
|
// Stop the typing indicator on every exit path (complete, error,
|
|
2804
3302
|
// exhausted, abort) so it never sticks after the turn ends.
|
|
2805
3303
|
this.channelRegistry?.stopTyping();
|
|
@@ -2884,7 +3382,10 @@ export class AgentFramework {
|
|
|
2884
3382
|
if (blocks) {
|
|
2885
3383
|
return { toolUseId: callId, content: blocks, isError: false };
|
|
2886
3384
|
}
|
|
2887
|
-
|
|
3385
|
+
// JSON.stringify returns the VALUE undefined (not a string) for undefined
|
|
3386
|
+
// input — a module tool returning `{ success: true }` with no data would
|
|
3387
|
+
// otherwise make `content.length` below throw a TypeError mid-turn.
|
|
3388
|
+
let content = JSON.stringify(afResult.data) ?? '';
|
|
2888
3389
|
if (maxChars && content.length > maxChars) {
|
|
2889
3390
|
content = safeSlice(content, 0, maxChars)
|
|
2890
3391
|
+ '\n\n[truncated — original was ' + content.length + ' chars]';
|
|
@@ -2954,11 +3455,11 @@ export class AgentFramework {
|
|
|
2954
3455
|
entryToStore.response = { blobId };
|
|
2955
3456
|
}
|
|
2956
3457
|
}
|
|
2957
|
-
// Append
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
this.store.
|
|
3458
|
+
// Append one record per inference. A read-modify-write via setStateJson
|
|
3459
|
+
// would emit a whole-array Set — record size then grows with accumulated
|
|
3460
|
+
// history (O(n²) aggregate disk) and the append_log strategy registered
|
|
3461
|
+
// for this state never sees an append.
|
|
3462
|
+
this.store.appendToStateJson(INFERENCE_LOG_ID, entryToStore);
|
|
2962
3463
|
}
|
|
2963
3464
|
logProcessEvent(event, responses) {
|
|
2964
3465
|
const entry = {
|
|
@@ -2974,11 +3475,8 @@ export class AgentFramework {
|
|
|
2974
3475
|
const blobId = this.store.storeBlob(Buffer.from(responsesJson), 'application/json');
|
|
2975
3476
|
entryToStore.responses = { blobId };
|
|
2976
3477
|
}
|
|
2977
|
-
// Append
|
|
2978
|
-
|
|
2979
|
-
const entries = Array.isArray(data) ? data : [];
|
|
2980
|
-
entries.push(entryToStore);
|
|
2981
|
-
this.store.setStateJson(PROCESS_LOG_ID, entries);
|
|
3478
|
+
// Append one record per event — same rationale as logInference.
|
|
3479
|
+
this.store.appendToStateJson(PROCESS_LOG_ID, entryToStore);
|
|
2982
3480
|
}
|
|
2983
3481
|
/**
|
|
2984
3482
|
* Find the MCPL server for a tool call by checking against the prefix map.
|
|
@@ -3175,15 +3673,19 @@ export class AgentFramework {
|
|
|
3175
3673
|
// without this the only durable record of a failed inference is a field in
|
|
3176
3674
|
// llm-calls.jsonl — invisible to operator, agent, and monitoring.
|
|
3177
3675
|
if (event.type === 'inference:exhausted') {
|
|
3178
|
-
this.noteInferenceExhausted(event.agentName ?? 'unknown', event.error ?? 'unknown error');
|
|
3676
|
+
this.noteInferenceExhausted(event.agentName ?? 'unknown', event.error ?? 'unknown error', event.retryable, event.errorType);
|
|
3179
3677
|
}
|
|
3180
3678
|
else if (event.type === 'inference:completed') {
|
|
3181
3679
|
// A successful response — even mid-turn between tool calls — proves the
|
|
3182
|
-
// agent isn't hard-down; clear its consecutive-failure streak
|
|
3680
|
+
// agent isn't hard-down; clear its consecutive-failure streak (and the
|
|
3681
|
+
// poison-history breaker's rewind budget).
|
|
3183
3682
|
const name = event.agentName;
|
|
3184
3683
|
if (name && this.consecutiveInferenceFailures.get(name)) {
|
|
3185
3684
|
this.consecutiveInferenceFailures.set(name, 0);
|
|
3186
3685
|
}
|
|
3686
|
+
if (name && this.exhaustionRewinds.get(name)) {
|
|
3687
|
+
this.exhaustionRewinds.set(name, 0);
|
|
3688
|
+
}
|
|
3187
3689
|
}
|
|
3188
3690
|
const traceEvent = {
|
|
3189
3691
|
...event,
|
|
@@ -3212,12 +3714,49 @@ export class AgentFramework {
|
|
|
3212
3714
|
* inference, so this never causes a retry/wake loop.
|
|
3213
3715
|
* 3. escalation — after N consecutive failures the agent is hard-down;
|
|
3214
3716
|
* log that loudly (a repeated identical failure is the textbook signal).
|
|
3717
|
+
* 4. breaker — when the failures are an `invalid_request` (a 400-class
|
|
3718
|
+
* rejection of the history itself, e.g. corrupted tool_use/tool_result
|
|
3719
|
+
* pairing or an oversized attachment), retrying the same context can
|
|
3720
|
+
* never succeed: every new push event wakes the agent onto the same
|
|
3721
|
+
* poisoned history forever. At the hard-down threshold, automatically
|
|
3722
|
+
* quarantine: shed the newest complete exchange (the same forced-rewind
|
|
3723
|
+
* primitive `/unstick` uses — shedNewestTurn never orphans a
|
|
3724
|
+
* tool_use/thinking block) and retry, bounded by the same rewind cap so
|
|
3725
|
+
* the breaker can never eat the whole history.
|
|
3726
|
+
*/
|
|
3727
|
+
/**
|
|
3728
|
+
* Live health snapshot for /healthz and doctor tooling: gate state, queued
|
|
3729
|
+
* work, per-agent status + last inference activity. Cheap and read-only.
|
|
3215
3730
|
*/
|
|
3216
|
-
|
|
3731
|
+
healthSnapshot() {
|
|
3732
|
+
return {
|
|
3733
|
+
at: new Date().toISOString(),
|
|
3734
|
+
uptimeSec: Math.round(process.uptime()),
|
|
3735
|
+
gate: this.eventGate?.inferenceDiagnostics() ?? null,
|
|
3736
|
+
pendingRequests: this.pendingRequests.length,
|
|
3737
|
+
activeStreams: [...this.activeStreams.keys()],
|
|
3738
|
+
agents: [...this.agents.entries()].map(([name, agent]) => ({
|
|
3739
|
+
name,
|
|
3740
|
+
status: agent.state.status,
|
|
3741
|
+
consecutiveInferenceFailures: this.consecutiveInferenceFailures.get(name) ?? 0,
|
|
3742
|
+
lastInference: this.lastInferenceAt.get(name) ?? null,
|
|
3743
|
+
})),
|
|
3744
|
+
};
|
|
3745
|
+
}
|
|
3746
|
+
noteInferenceExhausted(agentName, reason, retryable, errorType) {
|
|
3217
3747
|
const streak = (this.consecutiveInferenceFailures.get(agentName) ?? 0) + 1;
|
|
3218
3748
|
this.consecutiveInferenceFailures.set(agentName, streak);
|
|
3749
|
+
this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), failedAt: Date.now(), lastError: reason.slice(0, 300) });
|
|
3219
3750
|
// (1) Durable stderr line — works in headless/daemon mode with no client.
|
|
3220
3751
|
console.error(`[inference-failed] agent=${agentName} consecutive=${streak}: ${reason}`);
|
|
3752
|
+
// (1b) Machine-greppable durable record, independent of journald/unit log
|
|
3753
|
+
// redirects: logs/failures.log under the host's working directory. This is
|
|
3754
|
+
// what connectome-doctor reads.
|
|
3755
|
+
try {
|
|
3756
|
+
mkdirSync('logs', { recursive: true });
|
|
3757
|
+
appendFileSync('logs/failures.log', JSON.stringify({ at: new Date().toISOString(), agent: agentName, consecutive: streak, reason }) + '\n');
|
|
3758
|
+
}
|
|
3759
|
+
catch { /* best-effort */ }
|
|
3221
3760
|
// (2) Agent-facing chronicle marker (no inference triggered → no loop).
|
|
3222
3761
|
const agent = this.agents.get(agentName);
|
|
3223
3762
|
if (agent && process.env.SUPPRESS_INFERENCE_FAILED_MARKER !== '1') {
|
|
@@ -3239,8 +3778,71 @@ export class AgentFramework {
|
|
|
3239
3778
|
if (streak >= this.inferenceFailureEscalationThreshold) {
|
|
3240
3779
|
console.error(`[inference-hard-down] agent=${agentName} has FAILED ${streak} consecutive ` +
|
|
3241
3780
|
`inferences — it cannot complete a turn. Last reason: ${reason}`);
|
|
3781
|
+
// Optional ops alert: set CONNECTOME_OPS_WEBHOOK to a Discord webhook URL.
|
|
3782
|
+
const hook = process.env.CONNECTOME_OPS_WEBHOOK;
|
|
3783
|
+
if (hook) {
|
|
3784
|
+
fetch(hook, {
|
|
3785
|
+
method: 'POST',
|
|
3786
|
+
headers: { 'content-type': 'application/json' },
|
|
3787
|
+
body: JSON.stringify({
|
|
3788
|
+
content: `\u{1F6A8} **${agentName}** hard-down: ${streak} consecutive inference failures.\nLast reason: ${reason.slice(0, 500)}`,
|
|
3789
|
+
}),
|
|
3790
|
+
}).catch((err) => console.error('[inference-hard-down] webhook post failed:', err instanceof Error ? err.message : err));
|
|
3791
|
+
}
|
|
3792
|
+
// (4) Poison-history breaker: ONLY for `invalid_request` — a 400-class
|
|
3793
|
+
// rejection of the history itself. Deliberately NOT keyed on
|
|
3794
|
+
// `retryable === false`, which membrane also returns for auth (expired
|
|
3795
|
+
// key), abort (deliberate cancel), context_length (compression's job,
|
|
3796
|
+
// and shedding newest is the wrong direction), safety and unsupported —
|
|
3797
|
+
// none of which mean the history is poisoned, and all of which would
|
|
3798
|
+
// otherwise shed good exchanges and stamp a false "the API kept
|
|
3799
|
+
// rejecting your history" marker. `retryable` is kept only for the trace.
|
|
3800
|
+
void retryable;
|
|
3801
|
+
if (errorType === 'invalid_request' && agent) {
|
|
3802
|
+
this.quarantinePoisonedHistory(agent, reason);
|
|
3803
|
+
}
|
|
3242
3804
|
}
|
|
3243
3805
|
}
|
|
3806
|
+
/**
|
|
3807
|
+
* Automatic poison-history quarantine (the actual "breaker"). Sheds the
|
|
3808
|
+
* newest complete exchange from the agent's history — reusing the same
|
|
3809
|
+
* primitives as the refusal auto-rewind / `/unstick` (shedNewestTurn +
|
|
3810
|
+
* the single consolidated episode marker) — and queues a retry so the
|
|
3811
|
+
* rewound history is verified immediately instead of waiting for the next
|
|
3812
|
+
* push event to wake the agent onto the same poisoned context.
|
|
3813
|
+
*
|
|
3814
|
+
* Bounded: at most `refusalHandling.maxRewinds` (default 3, hard cap 10)
|
|
3815
|
+
* sheds per failure episode; the budget resets on any successful inference.
|
|
3816
|
+
* At the cap (or with nothing left to shed) it stops — the agent stays up
|
|
3817
|
+
* and hard-down logging continues, but no further history is consumed.
|
|
3818
|
+
*/
|
|
3819
|
+
quarantinePoisonedHistory(agent, reason) {
|
|
3820
|
+
const cap = Math.max(1, Math.min(10, agent.refusalHandling?.maxRewinds ?? 3));
|
|
3821
|
+
const used = this.exhaustionRewinds.get(agent.name) ?? 0;
|
|
3822
|
+
if (used >= cap) {
|
|
3823
|
+
console.error(`[inference-rewind] agent=${agent.name} rewind cap ${cap} reached — ` +
|
|
3824
|
+
`not shedding further history. Manual repair (/unstick or /undo) needed.`);
|
|
3825
|
+
return;
|
|
3826
|
+
}
|
|
3827
|
+
const rec = this.shedNewestTurn(agent);
|
|
3828
|
+
if (!rec) {
|
|
3829
|
+
console.error(`[inference-rewind] agent=${agent.name} has nothing left to shed — ` +
|
|
3830
|
+
`history is only system markers. Manual repair needed.`);
|
|
3831
|
+
return;
|
|
3832
|
+
}
|
|
3833
|
+
this.exhaustionRewinds.set(agent.name, used + 1);
|
|
3834
|
+
const count = this.updateRewindMarker(agent, truncateReason(reason), 'inference-failure');
|
|
3835
|
+
console.error(`[inference-rewind] agent=${agent.name} auto-quarantined ${rec.descriptor} ` +
|
|
3836
|
+
`(${count} turn(s) shed this episode, cap ${cap}) after non-retryable ` +
|
|
3837
|
+
`inference failures — retrying on the rewound history.`);
|
|
3838
|
+
// Retry immediately on the repaired history (bounded by the cap above).
|
|
3839
|
+
this.pendingRequests.push({
|
|
3840
|
+
agentName: agent.name,
|
|
3841
|
+
reason: 'inference-failure-rewind-retry',
|
|
3842
|
+
source: 'framework',
|
|
3843
|
+
timestamp: Date.now(),
|
|
3844
|
+
});
|
|
3845
|
+
}
|
|
3244
3846
|
// ==========================================================================
|
|
3245
3847
|
// MCPL subsystem wiring
|
|
3246
3848
|
// ==========================================================================
|
|
@@ -3314,8 +3916,9 @@ export class AgentFramework {
|
|
|
3314
3916
|
}
|
|
3315
3917
|
},
|
|
3316
3918
|
});
|
|
3317
|
-
// Host capabilities advertised during the MCP handshake
|
|
3318
|
-
|
|
3919
|
+
// Host capabilities advertised during the MCP handshake — stored so
|
|
3920
|
+
// servers can also be connected later at runtime (connectMcplServer).
|
|
3921
|
+
this.mcplHostCapabilities = {
|
|
3319
3922
|
version: '0.4',
|
|
3320
3923
|
pushEvents: true,
|
|
3321
3924
|
contextHooks: {
|
|
@@ -3326,56 +3929,193 @@ export class AgentFramework {
|
|
|
3326
3929
|
};
|
|
3327
3930
|
for (const config of serverConfigs) {
|
|
3328
3931
|
try {
|
|
3329
|
-
|
|
3330
|
-
// registers channels — handleRegister fires during the handshake.
|
|
3331
|
-
if (config.channelSubscription !== undefined && this.channelRegistry) {
|
|
3332
|
-
this.channelRegistry.setSubscriptionPolicy(config.id, config.channelSubscription);
|
|
3333
|
-
}
|
|
3334
|
-
const connection = await this.mcplServerRegistry.addServer(config, hostCapabilities);
|
|
3335
|
-
// Wire event listeners then flush any events that arrived during the
|
|
3336
|
-
// handshake window (between setupMessageRouting and now).
|
|
3337
|
-
this.wireMcplEvents(connection);
|
|
3338
|
-
connection.ready();
|
|
3339
|
-
// Initialize feature sets if server advertises MCPL capabilities
|
|
3340
|
-
if (connection.capabilities) {
|
|
3341
|
-
const updateParams = this.featureSetManager.initializeServer(config.id, connection.capabilities, {
|
|
3342
|
-
enabledFeatureSets: config.enabledFeatureSets,
|
|
3343
|
-
disabledFeatureSets: config.disabledFeatureSets,
|
|
3344
|
-
});
|
|
3345
|
-
// Inform server which feature sets are enabled/disabled
|
|
3346
|
-
if (updateParams.enabled?.length || updateParams.disabled?.length) {
|
|
3347
|
-
connection.sendFeatureSetsUpdate(updateParams);
|
|
3348
|
-
}
|
|
3349
|
-
// Configure scope whitelist/blacklist patterns
|
|
3350
|
-
if (config.scopes) {
|
|
3351
|
-
this.scopeManager.configureAll(config.scopes);
|
|
3352
|
-
}
|
|
3353
|
-
// Register stateful feature sets with checkpoint manager (Step 8)
|
|
3354
|
-
if (this.checkpointManager) {
|
|
3355
|
-
const declared = this.featureSetManager.getDeclaredFeatureSets(config.id);
|
|
3356
|
-
if (declared) {
|
|
3357
|
-
for (const [fsName, fsDecl] of Object.entries(declared)) {
|
|
3358
|
-
if (fsDecl.rollback || fsDecl.hostState) {
|
|
3359
|
-
this.checkpointManager.registerFeatureSet(config.id, fsName, {
|
|
3360
|
-
hostState: fsDecl.hostState ?? false,
|
|
3361
|
-
rollback: fsDecl.rollback ?? false,
|
|
3362
|
-
});
|
|
3363
|
-
}
|
|
3364
|
-
}
|
|
3365
|
-
}
|
|
3366
|
-
}
|
|
3367
|
-
}
|
|
3368
|
-
this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` });
|
|
3932
|
+
await this.connectMcplServerInternal(config);
|
|
3369
3933
|
}
|
|
3370
3934
|
catch (error) {
|
|
3371
3935
|
// Fail-open: log and continue with remaining servers
|
|
3372
3936
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
3373
3937
|
console.error(`Failed to connect MCPL server "${config.id}":`, err.message);
|
|
3938
|
+
// Reaching this catch means no stub was created, so nothing will
|
|
3939
|
+
// retry — except in the edge case where addServer succeeded (with a
|
|
3940
|
+
// reconnecting stub) and a later setup step threw; config.reconnect
|
|
3941
|
+
// is the best cheap approximation of that.
|
|
3942
|
+
this.emitTrace({
|
|
3943
|
+
type: 'mcpl:server-connect-failed',
|
|
3944
|
+
serverId: config.id,
|
|
3945
|
+
error: err.message,
|
|
3946
|
+
attempt: 0,
|
|
3947
|
+
willRetry: config.reconnect === true,
|
|
3948
|
+
});
|
|
3374
3949
|
}
|
|
3375
3950
|
}
|
|
3376
3951
|
// Discover tools from all connected servers
|
|
3377
3952
|
await this.refreshMcplTools();
|
|
3378
3953
|
}
|
|
3954
|
+
/**
|
|
3955
|
+
* Connect a single MCPL server: register routing entries, open the
|
|
3956
|
+
* connection, wire events, and initialize feature sets / scopes /
|
|
3957
|
+
* checkpoints. Shared by startup (initializeMcpl) and the runtime
|
|
3958
|
+
* lifecycle API (connectMcplServer). Throws on connection failure.
|
|
3959
|
+
*/
|
|
3960
|
+
async connectMcplServerInternal(config) {
|
|
3961
|
+
if (!this.mcplServerRegistry || !this.mcplHostCapabilities) {
|
|
3962
|
+
throw new Error('MCPL subsystem is not initialized');
|
|
3963
|
+
}
|
|
3964
|
+
// Register prefix + config for tool dispatch routing (idempotent with
|
|
3965
|
+
// the pre-registration pass in initializeMcpl).
|
|
3966
|
+
const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
|
|
3967
|
+
this.mcplPrefixMap.set(prefix, config.id);
|
|
3968
|
+
this.mcplServerConfigs.set(config.id, config);
|
|
3969
|
+
// Record per-server channel subscription policy before the server
|
|
3970
|
+
// registers channels — handleRegister fires during the handshake.
|
|
3971
|
+
if (config.channelSubscription !== undefined && this.channelRegistry) {
|
|
3972
|
+
this.channelRegistry.setSubscriptionPolicy(config.id, config.channelSubscription);
|
|
3973
|
+
}
|
|
3974
|
+
const connection = await this.mcplServerRegistry.addServer(config, this.mcplHostCapabilities);
|
|
3975
|
+
// Wire event listeners then flush any events that arrived during the
|
|
3976
|
+
// handshake window (between setupMessageRouting and now).
|
|
3977
|
+
this.wireMcplEvents(connection);
|
|
3978
|
+
connection.ready();
|
|
3979
|
+
// Initialize feature sets if server advertises MCPL capabilities
|
|
3980
|
+
this.registerMcplServerFeatures(config, connection);
|
|
3981
|
+
this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` });
|
|
3982
|
+
}
|
|
3983
|
+
/**
|
|
3984
|
+
* (Re-)establish a server's host-side MCPL registration: feature-set state
|
|
3985
|
+
* (plus the featureSets/update sent to the server), scope patterns, and
|
|
3986
|
+
* stateful-feature-set checkpoint registration.
|
|
3987
|
+
*
|
|
3988
|
+
* Called on initial connect (connectMcplServerInternal) AND after every
|
|
3989
|
+
* auto-reconnect. The 'close' handler drops the feature-set registration, so
|
|
3990
|
+
* without the reconnect re-run FeatureSetManager.validateInbound would throw
|
|
3991
|
+
* "Unknown server" forever — every push event and inference request from the
|
|
3992
|
+
* revived server silently rejected until a full host restart.
|
|
3993
|
+
*
|
|
3994
|
+
* CheckpointManager.registerFeatureSet is idempotent (no-op for a key that
|
|
3995
|
+
* already has a tree), so checkpoint trees preserved across a transient
|
|
3996
|
+
* disconnect are resumed, not reset.
|
|
3997
|
+
*/
|
|
3998
|
+
registerMcplServerFeatures(config, connection) {
|
|
3999
|
+
if (!connection.capabilities)
|
|
4000
|
+
return;
|
|
4001
|
+
const updateParams = this.featureSetManager.initializeServer(config.id, connection.capabilities, {
|
|
4002
|
+
enabledFeatureSets: config.enabledFeatureSets,
|
|
4003
|
+
disabledFeatureSets: config.disabledFeatureSets,
|
|
4004
|
+
});
|
|
4005
|
+
// Inform server which feature sets are enabled/disabled
|
|
4006
|
+
if (updateParams.enabled?.length || updateParams.disabled?.length) {
|
|
4007
|
+
connection.sendFeatureSetsUpdate(updateParams);
|
|
4008
|
+
}
|
|
4009
|
+
// Configure scope whitelist/blacklist patterns
|
|
4010
|
+
if (config.scopes) {
|
|
4011
|
+
this.scopeManager.configureAll(config.scopes);
|
|
4012
|
+
}
|
|
4013
|
+
// Register stateful feature sets with checkpoint manager (Step 8)
|
|
4014
|
+
if (this.checkpointManager) {
|
|
4015
|
+
const declared = this.featureSetManager.getDeclaredFeatureSets(config.id);
|
|
4016
|
+
if (declared) {
|
|
4017
|
+
for (const [fsName, fsDecl] of Object.entries(declared)) {
|
|
4018
|
+
if (fsDecl.rollback || fsDecl.hostState) {
|
|
4019
|
+
this.checkpointManager.registerFeatureSet(config.id, fsName, {
|
|
4020
|
+
hostState: fsDecl.hostState ?? false,
|
|
4021
|
+
rollback: fsDecl.rollback ?? false,
|
|
4022
|
+
});
|
|
4023
|
+
}
|
|
4024
|
+
}
|
|
4025
|
+
}
|
|
4026
|
+
}
|
|
4027
|
+
}
|
|
4028
|
+
// ==========================================================================
|
|
4029
|
+
// Runtime MCPL server lifecycle (agent-facing hot deploy/restart/unload)
|
|
4030
|
+
// ==========================================================================
|
|
4031
|
+
/**
|
|
4032
|
+
* Connect a new MCPL server at runtime. Refreshes the tool list and
|
|
4033
|
+
* notifies the agent of newly available tools. Throws if the MCPL
|
|
4034
|
+
* subsystem is not initialized, the id is already connected, or the
|
|
4035
|
+
* connection/handshake fails.
|
|
4036
|
+
*/
|
|
4037
|
+
async connectMcplServer(config) {
|
|
4038
|
+
// Lazily bring up the MCPL subsystem — a framework that started with zero
|
|
4039
|
+
// configured servers can still deploy its first one at runtime.
|
|
4040
|
+
if (!this.mcplServerRegistry || !this.mcplHostCapabilities) {
|
|
4041
|
+
await this.initializeMcpl([], this.mcplInferenceRoutingConfig ?? undefined);
|
|
4042
|
+
}
|
|
4043
|
+
// Same collision rules create() enforces at startup: the tool prefix must
|
|
4044
|
+
// not shadow a module name or another server's prefix.
|
|
4045
|
+
const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
|
|
4046
|
+
if (this.moduleRegistry.getAllModules().some(m => m.name === prefix)) {
|
|
4047
|
+
throw new Error(`MCPL server "${config.id}" toolPrefix "${prefix}" collides with module "${prefix}"`);
|
|
4048
|
+
}
|
|
4049
|
+
const prefixOwner = this.mcplPrefixMap.get(prefix);
|
|
4050
|
+
if (prefixOwner && prefixOwner !== config.id) {
|
|
4051
|
+
throw new Error(`MCPL server "${config.id}" toolPrefix "${prefix}" collides with server "${prefixOwner}"`);
|
|
4052
|
+
}
|
|
4053
|
+
const oldToolNames = new Set(this.mcplTools.map(t => t.name));
|
|
4054
|
+
await this.connectMcplServerInternal(config);
|
|
4055
|
+
await this.refreshMcplTools();
|
|
4056
|
+
this.emitMcplToolDiff(oldToolNames, config.id);
|
|
4057
|
+
}
|
|
4058
|
+
/**
|
|
4059
|
+
* Disconnect an MCPL server at runtime: close the connection, destroy its
|
|
4060
|
+
* feature-set and checkpoint state (permanent removal — unlike a transient
|
|
4061
|
+
* transport close, which preserves checkpoints for the reconnect), remove
|
|
4062
|
+
* its channels from the registry, drop routing entries, and refresh tools.
|
|
4063
|
+
* No-op-ish if the server is not connected (still clears routing state).
|
|
4064
|
+
*/
|
|
4065
|
+
async disconnectMcplServer(id) {
|
|
4066
|
+
if (!this.mcplServerRegistry) {
|
|
4067
|
+
throw new Error('MCPL subsystem is not initialized');
|
|
4068
|
+
}
|
|
4069
|
+
const config = this.mcplServerConfigs.get(id);
|
|
4070
|
+
const oldToolNames = new Set(this.mcplTools.map(t => t.name));
|
|
4071
|
+
await this.mcplServerRegistry.removeServer(id);
|
|
4072
|
+
this.channelRegistry?.removeServer(id);
|
|
4073
|
+
// Permanent removal: also destroy feature-set and checkpoint state
|
|
4074
|
+
// explicitly. The 'close' handler usually does this, but a connection that
|
|
4075
|
+
// already transiently closed (reconnect pending) emits no second 'close'
|
|
4076
|
+
// from close(), and the transient path deliberately preserves checkpoints.
|
|
4077
|
+
this.featureSetManager?.removeServer(id);
|
|
4078
|
+
this.checkpointManager?.removeServer(id);
|
|
4079
|
+
const prefix = config?.toolPrefix ?? `mcpl--${id}`;
|
|
4080
|
+
this.mcplPrefixMap.delete(prefix);
|
|
4081
|
+
this.mcplServerConfigs.delete(id);
|
|
4082
|
+
await this.refreshMcplTools();
|
|
4083
|
+
this.emitMcplToolDiff(oldToolNames, id);
|
|
4084
|
+
}
|
|
4085
|
+
/**
|
|
4086
|
+
* Restart an MCPL server at runtime: disconnect, then reconnect with the
|
|
4087
|
+
* same (or an updated) config. This actually respawns a stdio child —
|
|
4088
|
+
* unlike `reconnect: true`, which only retries after transport-level
|
|
4089
|
+
* failures. Throws if the server has no stored config and none is given.
|
|
4090
|
+
*/
|
|
4091
|
+
async restartMcplServer(id, newConfig) {
|
|
4092
|
+
const config = newConfig ?? this.mcplServerConfigs.get(id);
|
|
4093
|
+
if (!config) {
|
|
4094
|
+
throw new Error(`MCPL server "${id}" is not configured`);
|
|
4095
|
+
}
|
|
4096
|
+
await this.disconnectMcplServer(id);
|
|
4097
|
+
await this.connectMcplServer(config);
|
|
4098
|
+
}
|
|
4099
|
+
/**
|
|
4100
|
+
* List configured MCPL servers with live connection status and the number
|
|
4101
|
+
* of tools each currently contributes.
|
|
4102
|
+
*/
|
|
4103
|
+
listMcplServers() {
|
|
4104
|
+
const result = [];
|
|
4105
|
+
for (const [id, config] of this.mcplServerConfigs) {
|
|
4106
|
+
const prefix = config.toolPrefix ?? `mcpl--${id}`;
|
|
4107
|
+
const connection = this.mcplServerRegistry?.getServer(id) ?? null;
|
|
4108
|
+
result.push({
|
|
4109
|
+
id,
|
|
4110
|
+
connected: connection?.isConnected ?? false,
|
|
4111
|
+
toolPrefix: prefix,
|
|
4112
|
+
toolCount: this.mcplTools.filter(t => t.name.startsWith(`${prefix}--`)).length,
|
|
4113
|
+
command: config.command,
|
|
4114
|
+
url: config.url,
|
|
4115
|
+
});
|
|
4116
|
+
}
|
|
4117
|
+
return result;
|
|
4118
|
+
}
|
|
3379
4119
|
/**
|
|
3380
4120
|
* Wire event listeners on an MCPL server connection.
|
|
3381
4121
|
* Push events and inference requests are deferred to Steps 6/7.
|
|
@@ -3446,14 +4186,95 @@ export class AgentFramework {
|
|
|
3446
4186
|
connection.on('tools-list-changed', () => {
|
|
3447
4187
|
this.handleToolsListChanged(connection.id);
|
|
3448
4188
|
});
|
|
3449
|
-
//
|
|
3450
|
-
|
|
4189
|
+
// Re-establish full server registration on reconnect. The 'close' handler
|
|
4190
|
+
// removed the feature-set state, so pushes / inference requests from the
|
|
4191
|
+
// revived server would otherwise be rejected with "Unknown server" until a
|
|
4192
|
+
// host restart. Re-runs the same init as the initial connect path (the
|
|
4193
|
+
// fresh handshake refreshed connection.capabilities); checkpoint trees
|
|
4194
|
+
// preserved across the transient close are resumed by idempotent
|
|
4195
|
+
// registration. Then refresh tools (server may have different tools).
|
|
4196
|
+
connection.on('reconnect', (info) => {
|
|
4197
|
+
try {
|
|
4198
|
+
const config = this.mcplServerConfigs.get(connection.id);
|
|
4199
|
+
if (config) {
|
|
4200
|
+
this.registerMcplServerFeatures(config, connection);
|
|
4201
|
+
}
|
|
4202
|
+
}
|
|
4203
|
+
catch (error) {
|
|
4204
|
+
console.error(`MCPL server "${connection.id}" re-registration after reconnect failed:`, error instanceof Error ? error.message : error);
|
|
4205
|
+
}
|
|
3451
4206
|
this.handleToolsListChanged(connection.id);
|
|
4207
|
+
this.emitTrace({
|
|
4208
|
+
type: 'mcpl:server-reconnected',
|
|
4209
|
+
serverId: connection.id,
|
|
4210
|
+
attempts: info?.attempts ?? 0,
|
|
4211
|
+
});
|
|
4212
|
+
// Mirror the module:removed emitted on 'close' so module-lifecycle
|
|
4213
|
+
// consumers see the server come back, not just vanish.
|
|
4214
|
+
this.emitTrace({ type: 'module:added', moduleName: `mcpl:${connection.id}` });
|
|
3452
4215
|
});
|
|
3453
|
-
//
|
|
3454
|
-
|
|
4216
|
+
// Surface connect/reconnect failures. Before these traces existed the
|
|
4217
|
+
// only receipt was a console.error on the host's own stderr — invisible
|
|
4218
|
+
// unless someone ssh'd in and read the process log.
|
|
4219
|
+
connection.on('connect-failed', (params) => {
|
|
4220
|
+
this.emitTrace({
|
|
4221
|
+
type: 'mcpl:server-connect-failed',
|
|
4222
|
+
serverId: connection.id,
|
|
4223
|
+
error: params.error,
|
|
4224
|
+
attempt: params.attempt,
|
|
4225
|
+
willRetry: connection.willReconnect,
|
|
4226
|
+
});
|
|
4227
|
+
});
|
|
4228
|
+
connection.on('reconnect-failed', (params) => {
|
|
4229
|
+
this.emitTrace({
|
|
4230
|
+
type: 'mcpl:server-connect-failed',
|
|
4231
|
+
serverId: connection.id,
|
|
4232
|
+
error: params.error,
|
|
4233
|
+
attempt: params.attempt,
|
|
4234
|
+
willRetry: connection.willReconnect,
|
|
4235
|
+
});
|
|
4236
|
+
});
|
|
4237
|
+
// A late response to a tools/call that already timed out, carrying
|
|
4238
|
+
// stateful data the server advanced to. The host can't re-inject it (the
|
|
4239
|
+
// dispatch context is gone), but surfacing it makes the checkpoint-tree
|
|
4240
|
+
// divergence greppable instead of a silent drift into stale state.
|
|
4241
|
+
connection.on('orphaned-response', (info) => {
|
|
4242
|
+
this.emitTrace({
|
|
4243
|
+
type: 'mcpl:orphaned-response',
|
|
4244
|
+
serverId: connection.id,
|
|
4245
|
+
responseId: info.id,
|
|
4246
|
+
hadState: info.hadState,
|
|
4247
|
+
hadCheckpoint: info.hadCheckpoint,
|
|
4248
|
+
});
|
|
4249
|
+
});
|
|
4250
|
+
// Connection-level errors (e.g. child process 'error' after spawn).
|
|
4251
|
+
// Attaching this listener also keeps an unhandled EventEmitter 'error'
|
|
4252
|
+
// from crashing the host process.
|
|
4253
|
+
connection.on('error', (err) => {
|
|
4254
|
+
this.emitTrace({
|
|
4255
|
+
type: 'mcpl:server-error',
|
|
4256
|
+
serverId: connection.id,
|
|
4257
|
+
error: err.message,
|
|
4258
|
+
});
|
|
4259
|
+
});
|
|
4260
|
+
// Cleanup on disconnect. Feature-set state is always dropped (the server
|
|
4261
|
+
// can't be validated while down; reconnect re-registers it). Checkpoint
|
|
4262
|
+
// trees, however, are durable state: CheckpointManager.removeServer deletes
|
|
4263
|
+
// them AND persists the deletion to Chronicle. The close handler must
|
|
4264
|
+
// therefore NEVER destroy checkpoints — `willReconnect` is false on a clean
|
|
4265
|
+
// stop() too (reconnectEnabled is cleared before 'close' fires), so gating
|
|
4266
|
+
// on it would erase every checkpoint tree on an ordinary host restart while
|
|
4267
|
+
// a SIGKILL preserves them. Permanent removal is owned solely by
|
|
4268
|
+
// disconnectMcplServer, which deletes the trees explicitly.
|
|
4269
|
+
connection.on('close', (code, signal) => {
|
|
3455
4270
|
this.featureSetManager?.removeServer(connection.id);
|
|
3456
|
-
this.
|
|
4271
|
+
this.emitTrace({
|
|
4272
|
+
type: 'mcpl:server-closed',
|
|
4273
|
+
serverId: connection.id,
|
|
4274
|
+
code: code ?? null,
|
|
4275
|
+
signal: signal ?? null,
|
|
4276
|
+
willReconnect: connection.willReconnect,
|
|
4277
|
+
});
|
|
3457
4278
|
this.emitTrace({ type: 'module:removed', moduleName: `mcpl:${connection.id}` });
|
|
3458
4279
|
});
|
|
3459
4280
|
}
|
|
@@ -3822,7 +4643,7 @@ export class AgentFramework {
|
|
|
3822
4643
|
finish({ success: false, error: 'sleep: `seconds` must be a positive number', isError: true });
|
|
3823
4644
|
return;
|
|
3824
4645
|
}
|
|
3825
|
-
const { until } = gate.setSleep(seconds, input.message);
|
|
4646
|
+
const { until } = gate.setSleep(seconds, input.message, agentName);
|
|
3826
4647
|
const announce = input.announce !== false; // default true
|
|
3827
4648
|
const human = seconds >= 3600 ? `${(seconds / 3600).toFixed(1)}h`
|
|
3828
4649
|
: seconds >= 60 ? `${Math.round(seconds / 60)}m`
|