@animalabs/agent-framework 0.6.0 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/src/agent.d.ts +4 -0
  2. package/dist/src/agent.d.ts.map +1 -1
  3. package/dist/src/agent.js +15 -0
  4. package/dist/src/agent.js.map +1 -1
  5. package/dist/src/framework.d.ts +176 -0
  6. package/dist/src/framework.d.ts.map +1 -1
  7. package/dist/src/framework.js +911 -90
  8. package/dist/src/framework.js.map +1 -1
  9. package/dist/src/gate/event-gate.d.ts +20 -1
  10. package/dist/src/gate/event-gate.d.ts.map +1 -1
  11. package/dist/src/gate/event-gate.js +60 -4
  12. package/dist/src/gate/event-gate.js.map +1 -1
  13. package/dist/src/index.d.ts +3 -0
  14. package/dist/src/index.d.ts.map +1 -1
  15. package/dist/src/index.js +4 -0
  16. package/dist/src/index.js.map +1 -1
  17. package/dist/src/mcpl/channel-registry.d.ts +17 -1
  18. package/dist/src/mcpl/channel-registry.d.ts.map +1 -1
  19. package/dist/src/mcpl/channel-registry.js +34 -3
  20. package/dist/src/mcpl/channel-registry.js.map +1 -1
  21. package/dist/src/mcpl/push-handler.d.ts.map +1 -1
  22. package/dist/src/mcpl/push-handler.js +4 -0
  23. package/dist/src/mcpl/push-handler.js.map +1 -1
  24. package/dist/src/mcpl/server-connection.d.ts +29 -0
  25. package/dist/src/mcpl/server-connection.d.ts.map +1 -1
  26. package/dist/src/mcpl/server-connection.js +102 -4
  27. package/dist/src/mcpl/server-connection.js.map +1 -1
  28. package/dist/src/mcpl/types.d.ts +17 -1
  29. package/dist/src/mcpl/types.d.ts.map +1 -1
  30. package/dist/src/mcpl/types.js.map +1 -1
  31. package/dist/src/module-registry.d.ts +15 -1
  32. package/dist/src/module-registry.d.ts.map +1 -1
  33. package/dist/src/module-registry.js +92 -4
  34. package/dist/src/module-registry.js.map +1 -1
  35. package/dist/src/types/agent.d.ts +37 -0
  36. package/dist/src/types/agent.d.ts.map +1 -1
  37. package/dist/src/types/module.d.ts +46 -0
  38. package/dist/src/types/module.d.ts.map +1 -1
  39. package/dist/src/types/trace.d.ts +42 -0
  40. package/dist/src/types/trace.d.ts.map +1 -1
  41. package/dist/test/agent-cache-ttl.test.d.ts +2 -0
  42. package/dist/test/agent-cache-ttl.test.d.ts.map +1 -0
  43. package/dist/test/agent-cache-ttl.test.js +25 -0
  44. package/dist/test/agent-cache-ttl.test.js.map +1 -0
  45. package/dist/test/conversation-fork-injection-scoping.test.d.ts +2 -0
  46. package/dist/test/conversation-fork-injection-scoping.test.d.ts.map +1 -0
  47. package/dist/test/conversation-fork-injection-scoping.test.js +184 -0
  48. package/dist/test/conversation-fork-injection-scoping.test.js.map +1 -0
  49. package/dist/test/gather-context-timeout.test.d.ts +2 -0
  50. package/dist/test/gather-context-timeout.test.d.ts.map +1 -0
  51. package/dist/test/gather-context-timeout.test.js +120 -0
  52. package/dist/test/gather-context-timeout.test.js.map +1 -0
  53. package/dist/test/inference-exhaustion-rewind.test.d.ts +2 -0
  54. package/dist/test/inference-exhaustion-rewind.test.d.ts.map +1 -0
  55. package/dist/test/inference-exhaustion-rewind.test.js +181 -0
  56. package/dist/test/inference-exhaustion-rewind.test.js.map +1 -0
  57. package/dist/test/inference-failure-observability.test.js +4 -0
  58. package/dist/test/inference-failure-observability.test.js.map +1 -1
  59. package/dist/test/mcpl-reconnect-events.test.d.ts +2 -0
  60. package/dist/test/mcpl-reconnect-events.test.d.ts.map +1 -0
  61. package/dist/test/mcpl-reconnect-events.test.js +119 -0
  62. package/dist/test/mcpl-reconnect-events.test.js.map +1 -0
  63. package/dist/test/mcpl-reconnect-reregistration.test.d.ts +2 -0
  64. package/dist/test/mcpl-reconnect-reregistration.test.d.ts.map +1 -0
  65. package/dist/test/mcpl-reconnect-reregistration.test.js +181 -0
  66. package/dist/test/mcpl-reconnect-reregistration.test.js.map +1 -0
  67. package/dist/test/mcpl-request-timeout.test.d.ts +2 -0
  68. package/dist/test/mcpl-request-timeout.test.d.ts.map +1 -0
  69. package/dist/test/mcpl-request-timeout.test.js +123 -0
  70. package/dist/test/mcpl-request-timeout.test.js.map +1 -0
  71. package/dist/test/mcpl-runtime-lifecycle.test.d.ts +7 -0
  72. package/dist/test/mcpl-runtime-lifecycle.test.d.ts.map +1 -0
  73. package/dist/test/mcpl-runtime-lifecycle.test.js +142 -0
  74. package/dist/test/mcpl-runtime-lifecycle.test.js.map +1 -0
  75. package/dist/test/refusal-rewind.test.d.ts +2 -0
  76. package/dist/test/refusal-rewind.test.d.ts.map +1 -0
  77. package/dist/test/refusal-rewind.test.js +108 -0
  78. package/dist/test/refusal-rewind.test.js.map +1 -0
  79. package/dist/test/state-scaling-e2e.test.d.ts +22 -0
  80. package/dist/test/state-scaling-e2e.test.d.ts.map +1 -0
  81. package/dist/test/state-scaling-e2e.test.js +200 -0
  82. package/dist/test/state-scaling-e2e.test.js.map +1 -0
  83. package/dist/test/state-scaling.test.d.ts +12 -0
  84. package/dist/test/state-scaling.test.d.ts.map +1 -0
  85. package/dist/test/state-scaling.test.js +243 -0
  86. package/dist/test/state-scaling.test.js.map +1 -0
  87. package/dist/test/tool-result-content.test.d.ts +2 -0
  88. package/dist/test/tool-result-content.test.d.ts.map +1 -0
  89. package/dist/test/tool-result-content.test.js +37 -0
  90. package/dist/test/tool-result-content.test.js.map +1 -0
  91. package/dist/tsconfig.tsbuildinfo +1 -1
  92. package/package.json +3 -2
  93. package/src/agent.ts +16 -0
  94. package/src/framework.ts +1069 -99
  95. package/src/gate/event-gate.ts +56 -4
  96. package/src/index.ts +10 -0
  97. package/src/mcpl/channel-registry.ts +34 -3
  98. package/src/mcpl/push-handler.ts +4 -0
  99. package/src/mcpl/server-connection.ts +110 -4
  100. package/src/mcpl/types.ts +19 -1
  101. package/src/module-registry.ts +113 -4
  102. package/src/types/agent.ts +38 -0
  103. package/src/types/module.ts +52 -0
  104. package/src/types/trace.ts +51 -0
@@ -1,10 +1,11 @@
1
1
  import { join } from 'node:path';
2
+ import { appendFileSync, mkdirSync } from 'node:fs';
2
3
  import { JsStore } from '@animalabs/chronicle';
3
4
  import { MembraneError } from '@animalabs/membrane';
4
5
  import { ContextManager, PassthroughStrategy } from '@animalabs/context-manager';
5
6
  import { ProcessQueueImpl } from './queue.js';
6
7
  import { Agent } from './agent.js';
7
- import { ModuleRegistry } from './module-registry.js';
8
+ import { ModuleRegistry, isStateExistsError } from './module-registry.js';
8
9
  import { McplServerRegistry } from './mcpl/server-registry.js';
9
10
  import { FeatureSetManager } from './mcpl/feature-set-manager.js';
10
11
  import { ScopeManager } from './mcpl/scope-manager.js';
@@ -24,7 +25,8 @@ const FRAMEWORK_STATE_ID = 'framework/state';
24
25
  const CONVERSATION_ROUTER_STATE_ID = 'framework/conversation-router';
25
26
  const INFERENCE_LOG_ID = 'framework/inference-log';
26
27
  const PROCESS_LOG_ID = 'framework/process-log';
27
- const TURN_CHECKPOINTS_ID = 'framework/turn-checkpoints';
28
+ const TURN_CHECKPOINTS_ID = 'framework/turn-checkpoints'; // legacy single-map layout, read-only fallback
29
+ const TURN_CHECKPOINTS_TREE_ID = 'framework/turn-checkpoints/tree';
28
30
  /** Maximum number of turn checkpoints to keep per agent. */
29
31
  const MAX_TURN_CHECKPOINTS = 20;
30
32
  /**
@@ -95,6 +97,12 @@ function extractGateFields(event) {
95
97
  : undefined;
96
98
  return { content, metadata, mount, paths };
97
99
  }
100
+ /** Cap an API error message for inline use in a rewind marker: keep enough to
101
+ * identify the failure class without pasting a wall of provider JSON into the
102
+ * agent's context. */
103
+ function truncateReason(reason, max = 160) {
104
+ return reason.length <= max ? reason : reason.slice(0, max) + '…';
105
+ }
98
106
  export class AgentFramework {
99
107
  store;
100
108
  ownsStore;
@@ -120,6 +128,10 @@ export class AgentFramework {
120
128
  traceListeners = [];
121
129
  syncIntervalMs;
122
130
  syncTimer = null;
131
+ /** Last time we console-warned about stale (busy-requeued) inference requests, per agent. */
132
+ staleWarnAt = new Map();
133
+ /** Per-agent last inference activity (epoch ms), for /healthz + doctor tooling. */
134
+ lastInferenceAt = new Map();
123
135
  processLoggingPersist;
124
136
  processLoggingBroadcast;
125
137
  activeStreams = new Map();
@@ -130,6 +142,23 @@ export class AgentFramework {
130
142
  /** N consecutive failed inferences ⇒ the agent is treated as hard-down and
131
143
  * escalated loudly to stderr. */
132
144
  inferenceFailureEscalationThreshold = 3;
145
+ /** Per-agent count of consecutive refusal-driven rewinds in the current turn
146
+ * chain (reset when a turn completes without a refusal). Bounds the auto
147
+ * rewind loop — see refusalHandling + rewindTriggeringTurn. */
148
+ refusalRewinds = new Map();
149
+ /** Per-agent count of poison-history rewinds performed by the hard-down
150
+ * breaker (noteInferenceExhausted). Reset on any successful inference.
151
+ * Bounds the automatic quarantine loop the same way refusalRewinds bounds
152
+ * the refusal loop, so the breaker can never shed the whole history. */
153
+ exhaustionRewinds = new Map();
154
+ /** Per-agent current rewind episode: the single consolidated marker's id and
155
+ * how many turns have been shed so far. One marker per episode, updated in
156
+ * place (see updateRewindMarker); cleared when the episode ends. */
157
+ rewindEpisode = new Map();
158
+ /** Active `/unstick` sessions: an admin-forced rewind-until-clean loop that
159
+ * runs even when the agent's autoRewind toggle is off. Tracks the remaining
160
+ * budget, what was shed (for the report), and where to post the outcome. */
161
+ forcedRewind = new Map();
133
162
  /** Name of the primary (non-ephemeral) agent for routing framework messages. */
134
163
  primaryAgentName = null;
135
164
  // Messages deferred while an agent is waiting_for_tools (to preserve
@@ -171,6 +200,13 @@ export class AgentFramework {
171
200
  mcplPrefixMap = new Map();
172
201
  /** Maps serverId → McplServerConfig for prefix lookup. */
173
202
  mcplServerConfigs = new Map();
203
+ /** Host capabilities advertised during the MCP handshake — stored so servers
204
+ * can be connected at runtime (connectMcplServer) after initialization. */
205
+ mcplHostCapabilities = null;
206
+ /** Inference routing policy from FrameworkConfig — stored so the MCPL
207
+ * subsystem can be lazily initialized by connectMcplServer when the
208
+ * framework started with zero configured servers. */
209
+ mcplInferenceRoutingConfig = null;
174
210
  // EventGate (null when FrameworkConfig.gate is omitted)
175
211
  eventGate = null;
176
212
  // Session-level token usage tracking (always-on)
@@ -237,11 +273,14 @@ export class AgentFramework {
237
273
  catch {
238
274
  // Already registered
239
275
  }
276
+ // The legacy single-map checkpoint state (TURN_CHECKPOINTS_ID) is no longer
277
+ // registered for new stores — it's read-only fallback data in old ones.
240
278
  try {
241
- store.registerState({ id: TURN_CHECKPOINTS_ID, strategy: 'snapshot' });
279
+ store.registerState({ id: TURN_CHECKPOINTS_TREE_ID, strategy: 'tree' });
242
280
  }
243
- catch {
244
- // Already registered
281
+ catch (error) {
282
+ if (!isStateExistsError(error))
283
+ throw error;
245
284
  }
246
285
  // Process logging config (default: disabled)
247
286
  const processLoggingPersist = config.processLogging?.persist ?? false;
@@ -327,6 +366,9 @@ export class AgentFramework {
327
366
  });
328
367
  framework.livenessWatchdog.start();
329
368
  }
369
+ // Stored for lazy MCPL initialization (connectMcplServer on a framework
370
+ // that started with zero configured servers).
371
+ framework.mcplInferenceRoutingConfig = config.inferenceRouting ?? null;
330
372
  // Initialize MCPL subsystems if configured
331
373
  if (config.mcplServers && config.mcplServers.length > 0) {
332
374
  // Validate tool prefixes: no collisions with module names or between servers
@@ -345,6 +387,30 @@ export class AgentFramework {
345
387
  }
346
388
  await framework.initializeMcpl(config.mcplServers, config.inferenceRouting);
347
389
  }
390
+ // Diagnostics: `kill -USR2 <pid>` dumps live wake/inference state to stderr
391
+ // (journal) without a restart — for catching the wake-wedge on the running
392
+ // process. Shows the gate's `inferring` set + buffered-event count (the
393
+ // wedge signature), active streams, and pending inference requests.
394
+ try {
395
+ process.on('SIGUSR2', () => {
396
+ try {
397
+ const gate = framework.eventGate?.inferenceDiagnostics() ?? null;
398
+ console.error('[diagnostics] ' + JSON.stringify({
399
+ at: new Date().toISOString(),
400
+ gate,
401
+ activeStreams: [...framework.activeStreams.keys()],
402
+ pendingRequests: framework.pendingRequests.length,
403
+ agents: [...framework.agents.keys()],
404
+ }));
405
+ }
406
+ catch (err) {
407
+ console.error('[diagnostics] dump failed:', err instanceof Error ? err.message : err);
408
+ }
409
+ });
410
+ }
411
+ catch {
412
+ // SIGUSR2 not available on this platform — non-fatal.
413
+ }
348
414
  return framework;
349
415
  }
350
416
  /**
@@ -568,8 +634,9 @@ export class AgentFramework {
568
634
  // Module gatherContext (fail-open, matches startAgentStream)
569
635
  try {
570
636
  const moduleInjections = await this.moduleRegistry.gatherContext(agentName);
571
- if (moduleInjections.length > 0) {
572
- injections = moduleInjections;
637
+ const scopedModuleInjections = this.scopeInjectionsForAgent(agentName, moduleInjections);
638
+ if (scopedModuleInjections.length > 0) {
639
+ injections = scopedModuleInjections;
573
640
  }
574
641
  }
575
642
  catch (error) {
@@ -580,7 +647,7 @@ export class AgentFramework {
580
647
  if (this.hookOrchestrator) {
581
648
  try {
582
649
  const hookParams = this.buildBeforeInferenceParams(agent);
583
- const hookInjections = await this.hookOrchestrator.beforeInference(hookParams);
650
+ const hookInjections = this.scopeInjectionsForAgent(agentName, await this.hookOrchestrator.beforeInference(hookParams));
584
651
  if (hookInjections.length > 0) {
585
652
  injections = injections ? [...injections, ...hookInjections] : hookInjections;
586
653
  }
@@ -743,6 +810,7 @@ export class AgentFramework {
743
810
  clearInterval(completionWatchdog);
744
811
  this.offTrace(traceListener);
745
812
  this.agents.delete(agent.name);
813
+ this.evictTurnCheckpoints(agent.name);
746
814
  };
747
815
  // Watchdog: if inference never starts within 30s, the agent is a zombie.
748
816
  // This catches cases where the event loop stalled, the agent was
@@ -1080,6 +1148,181 @@ export class AgentFramework {
1080
1148
  console.error('[inference-refusal] reaction failed:', err instanceof Error ? err.message : err);
1081
1149
  }
1082
1150
  }
1151
+ /**
1152
+ * Rewind the single turn that fed a refused generation: redact the newest
1153
+ * message the agent did NOT author (a tool result, or an incoming message)
1154
+ * and inject a metadata-only marker in its place. The marker carries a
1155
+ * *description* of what was withheld (kind + size), never the content — so it
1156
+ * cannot itself re-trip the classifier — while the raw record survives in the
1157
+ * chronicle log for forensics. Returns a record describing the removal, or
1158
+ * null if there is nothing eligible to rewind (e.g. only the agent's own
1159
+ * turns, or a prior system marker, remain).
1160
+ *
1161
+ * Shared by the auto path (on `stop_reason: refusal`) and the `/rewind`
1162
+ * host command.
1163
+ */
1164
+ shedNewestTurn(agent) {
1165
+ const cm = agent.getContextManager();
1166
+ const all = cm.getAllMessages();
1167
+ const typeOf = (b) => b.type;
1168
+ const hasBlock = (m, t) => Array.isArray(m?.content) && m.content.some((b) => typeOf(b) === t);
1169
+ // Newest message that is not our own episode marker. We shed strictly
1170
+ // newest-first, in sequence — INCLUDING the agent's own turns (poison lives
1171
+ // in tool_use/narration turns too), because newest-first reaches whatever is
1172
+ // poisoning the context if allowed to run deep enough. The one invariant:
1173
+ // shed COMPLETE exchanges. Removing a `tool_result` also removes its paired
1174
+ // `tool_use` assistant turn, so we never leave an orphaned tool_use / signed
1175
+ // `thinking` block — which the API rejects with a 400 ("thinking blocks in
1176
+ // the latest assistant message cannot be modified"). We do NOT add a marker
1177
+ // here; the caller keeps one consolidated marker (updateRewindMarker).
1178
+ let idx = -1;
1179
+ for (let i = all.length - 1; i >= 0; i--) {
1180
+ const meta = (all[i].metadata ?? {});
1181
+ if (meta.system)
1182
+ continue;
1183
+ idx = i;
1184
+ break;
1185
+ }
1186
+ if (idx < 0)
1187
+ return null;
1188
+ const msg = all[idx];
1189
+ const md = (msg.metadata ?? {});
1190
+ const content = Array.isArray(msg.content) ? msg.content : [];
1191
+ const toolBlocks = content.filter((b) => typeOf(b) === 'tool_result');
1192
+ const images = content.filter((b) => typeOf(b) === 'image').length;
1193
+ const textLen = content
1194
+ .filter((b) => typeOf(b) === 'text')
1195
+ .reduce((n, b) => n + String(b.text ?? '').length, 0);
1196
+ // Determine the complete exchange to remove.
1197
+ const removedIds = [msg.id];
1198
+ let kind;
1199
+ let descriptor;
1200
+ if (toolBlocks.length > 0) {
1201
+ kind = 'tool';
1202
+ // Pair the tool_result with the tool_use assistant turn right before it.
1203
+ if (idx - 1 >= 0 && hasBlock(all[idx - 1], 'tool_use')) {
1204
+ removedIds.push(all[idx - 1].id);
1205
+ }
1206
+ const sz = toolBlocks.reduce((n, b) => n + String(b.content ?? '').length, 0);
1207
+ descriptor = `a tool exchange (~${Math.max(1, Math.round(sz / 1024))}KB` +
1208
+ `${images ? `, ${images} image(s)` : ''})`;
1209
+ }
1210
+ else if (md.messageId) {
1211
+ kind = 'human';
1212
+ descriptor = `an incoming message from ${msg.participant} ` +
1213
+ `(${textLen} chars${images ? `, ${images} image(s)` : ''})`;
1214
+ }
1215
+ else {
1216
+ kind = 'other';
1217
+ descriptor = `a ${msg.participant} turn ` +
1218
+ `(${textLen} chars${images ? `, ${images} image(s)` : ''})`;
1219
+ }
1220
+ const discordRef = md.messageId && md.channelId
1221
+ ? { channelId: String(md.channelId), messageId: String(md.messageId) }
1222
+ : undefined;
1223
+ for (const id of removedIds)
1224
+ cm.removeMessage(id);
1225
+ return { kind, descriptor, removedIds, discordRef };
1226
+ }
1227
+ /**
1228
+ * Maintain exactly ONE consolidated marker for the current rewind episode,
1229
+ * updated in place as more turns are shed. Six rewinds ⇒ one message that
1230
+ * says "the 6 most recent turns were set aside", not six separate notes — so
1231
+ * the context converges (shed N, add 1) instead of growing, and the marker
1232
+ * sits at the tail giving the model something actionable to answer once the
1233
+ * refusal clears. Returns the running count.
1234
+ */
1235
+ updateRewindMarker(agent, category, cause = 'refusal') {
1236
+ const cm = agent.getContextManager();
1237
+ const ep = this.rewindEpisode.get(agent.name);
1238
+ const count = (ep?.count ?? 0) + 1;
1239
+ const why = cause === 'refusal'
1240
+ ? `the model refused on them (content filter: ${category}). Their content is ` +
1241
+ `not reproduced here (so this note can't re-trigger the filter); the ` +
1242
+ `originals remain in the raw record`
1243
+ : `the model API kept rejecting the conversation on them (${category}). ` +
1244
+ `Their content is not reproduced here (so this note can't re-trigger the ` +
1245
+ `rejection); the originals remain in the raw record`;
1246
+ const text = `[refusal-rewind] The ${count} most recent turn(s) were set aside because ` +
1247
+ `${why}. You are clear to continue — if you ` +
1248
+ `were mid-task, take a different approach; otherwise carry on with whatever ` +
1249
+ `is now in front of you, or briefly acknowledge the gap and ask what's next.`;
1250
+ const blocks = [{ type: 'text', text }];
1251
+ if (ep) {
1252
+ cm.editMessage(ep.markerId, blocks);
1253
+ this.rewindEpisode.set(agent.name, { markerId: ep.markerId, count, category });
1254
+ }
1255
+ else {
1256
+ const id = cm.addMessage('user', blocks, {
1257
+ system: true, kind: 'refusal-rewind', category, count, cause,
1258
+ });
1259
+ this.rewindEpisode.set(agent.name, { markerId: id, count, category });
1260
+ }
1261
+ return count;
1262
+ }
1263
+ /**
1264
+ * Announce a refusal-rewind on the conversational surface (Discord), used
1265
+ * when the withheld turn was a *human* message so it isn't dropped silently.
1266
+ * Best-effort; mirrors reactToRefusal's locus resolution.
1267
+ */
1268
+ async announceRewind(agentName, rec, category) {
1269
+ try {
1270
+ const incoming = this.channelRegistry?.buildChannelContext()?.incoming;
1271
+ if (!incoming)
1272
+ return;
1273
+ const parts = incoming.channelId.split(':');
1274
+ if (parts[0] !== 'discord')
1275
+ return;
1276
+ const channelId = parts[parts.length - 1];
1277
+ const serverId = this.channelRegistry?.getChannelServerId(incoming.channelId);
1278
+ const server = serverId ? this.mcplServerRegistry?.getServer(serverId) : null;
1279
+ if (!server)
1280
+ return;
1281
+ await server.sendToolsCall('send_message', {
1282
+ channelId,
1283
+ content: `⚠️ I had to set aside ${rec.descriptor} — it tripped a content ` +
1284
+ `filter (${category}), so it's withheld from my context and I'm ` +
1285
+ `continuing without it. If it was important, please rephrase or re-send.`,
1286
+ });
1287
+ }
1288
+ catch (err) {
1289
+ console.error('[refusal-rewind] announce failed:', err instanceof Error ? err.message : err);
1290
+ }
1291
+ }
1292
+ /**
1293
+ * Conclude an active `/unstick` session: post the outcome (what was shed and
1294
+ * whether the model stopped refusing) to the channel the command came from,
1295
+ * then clear the session. Idempotent — a no-op if there's no session.
1296
+ */
1297
+ finishUnstick(agentName, success, category) {
1298
+ const s = this.forcedRewind.get(agentName);
1299
+ if (!s)
1300
+ return;
1301
+ this.forcedRewind.delete(agentName);
1302
+ const n = s.removed.length;
1303
+ const list = n
1304
+ ? '\n' + s.removed.map((r) => `• ${r.descriptor}`).join('\n')
1305
+ : '';
1306
+ const content = success
1307
+ ? `🔧 Unstuck **${agentName}** — shed ${n} turn(s); the model responded.${list}`
1308
+ : `⚠️ Couldn't unstick **${agentName}** — still refusing after ${n} rewind(s)` +
1309
+ `${category ? ` (category=${category})` : ''}.${list}`;
1310
+ console.error(`[unstick] agent=${agentName} ${success ? 'succeeded' : 'gave up'} after ${n} rewind(s)`);
1311
+ if (s.channelId)
1312
+ void this.postToChannel(s.serverId, s.channelId, content);
1313
+ }
1314
+ /** Best-effort send_message to a raw channel via a named MCPL server. */
1315
+ async postToChannel(serverId, channelId, content) {
1316
+ try {
1317
+ const server = this.mcplServerRegistry?.getServer(serverId);
1318
+ if (!server)
1319
+ return;
1320
+ await server.sendToolsCall('send_message', { channelId, content });
1321
+ }
1322
+ catch (err) {
1323
+ console.error('[unstick] report post failed:', err instanceof Error ? err.message : err);
1324
+ }
1325
+ }
1083
1326
  /**
1084
1327
  * Handle a `host/command` request from an MCPL surface server (e.g. a
1085
1328
  * Discord slash command). Currently supports:
@@ -1090,13 +1333,41 @@ export class AgentFramework {
1090
1333
  * `previewActivation` render (no inference, no Chronicle writes).
1091
1334
  */
1092
1335
  async handleHostCommand(serverId, params) {
1093
- if (params.command !== 'undo' && params.command !== 'hide') {
1336
+ if (params.command !== 'undo' && params.command !== 'hide' && params.command !== 'unstick') {
1094
1337
  return { ok: false, error: `Unknown host command: ${String(params.command)}` };
1095
1338
  }
1096
1339
  const agentName = params.agentName ?? [...this.agents.keys()][0];
1097
1340
  if (!agentName || !this.agents.has(agentName)) {
1098
1341
  return { ok: false, error: `Unknown agent: ${String(agentName)}` };
1099
1342
  }
1343
+ // unstick: force the refusal-rewind loop on demand (even if the agent's
1344
+ // autoRewind toggle is off). Redacts the turn that fed the refusal and
1345
+ // re-runs, up to `cap` times, until the model stops refusing. Kicks the
1346
+ // loop and returns immediately; the outcome (what was shed + whether it
1347
+ // cleared) is posted to the channel when the chain resolves.
1348
+ if (params.command === 'unstick') {
1349
+ const agent = this.agents.get(agentName);
1350
+ if (agent.state.status !== 'idle') {
1351
+ return { ok: false, error: `Cannot unstick while agent is ${agent.state.status}` };
1352
+ }
1353
+ const cap = Math.max(1, Math.min(10, Math.floor(params.maxRewinds ?? agent.refusalHandling?.maxRewinds ?? 3)));
1354
+ this.forcedRewind.set(agentName, {
1355
+ remaining: cap,
1356
+ removed: [],
1357
+ serverId,
1358
+ channelId: params.channelId ?? '',
1359
+ });
1360
+ this.refusalRewinds.set(agentName, 0);
1361
+ this.pendingRequests.push({
1362
+ agentName,
1363
+ reason: 'unstick',
1364
+ source: 'framework',
1365
+ timestamp: Date.now(),
1366
+ });
1367
+ console.error(`[unstick] agent=${agentName} started cap=${cap} ` +
1368
+ `by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`);
1369
+ return { ok: true, started: true, cap };
1370
+ }
1100
1371
  // hide: redact a single message (or an inclusive range) from the active
1101
1372
  // branch, addressed by Discord message id. Unlike undo this is a
1102
1373
  // removal-in-place (chronicle redact), not a branch rewind.
@@ -1615,18 +1886,86 @@ export class AgentFramework {
1615
1886
  }
1616
1887
  this.saveTurnCheckpoints(agentName, checkpoints);
1617
1888
  }
1889
+ /**
1890
+ * Checkpoints live in one Tree state (TURN_CHECKPOINTS_TREE_ID) keyed by agent
1891
+ * name, each key pointing at a JSON blob of that agent's ≤MAX_TURN_CHECKPOINTS
1892
+ * list. This kills two unbounded-in-agents-ever dimensions at once:
1893
+ *
1894
+ * - the original single Record<agentName, list> map was rewritten whole on
1895
+ * every turn of every agent and never evicted a departed subagent's key;
1896
+ * - the intermediate design (one state slot per agent) fixed the writes but
1897
+ * leaked a permanent chronicle state *registration* per spawn — chronicle
1898
+ * has no deregistration, and the state index is rewritten and fsynced on
1899
+ * every sync tick, so per-tick cost grew with fleet history anyway.
1900
+ *
1901
+ * A tree gives per-key O(entry) writes, real key removal (treeRemove), and
1902
+ * exactly one registration for the life of the store. The legacy map state
1903
+ * (TURN_CHECKPOINTS_ID) is kept as a read-only fallback for stores written
1904
+ * before the split; eviction tombstones legacy keys so a reused agent name
1905
+ * can't inherit a dead agent's checkpoints through the fallback.
1906
+ */
1907
+ legacyCheckpointKeys = null;
1908
+ hasLegacyCheckpoints(agentName) {
1909
+ if (!this.legacyCheckpointKeys) {
1910
+ const legacy = this.store.getStateJson(TURN_CHECKPOINTS_ID);
1911
+ this.legacyCheckpointKeys = new Set(legacy && typeof legacy === 'object' ? Object.keys(legacy) : []);
1912
+ }
1913
+ return this.legacyCheckpointKeys.has(agentName);
1914
+ }
1618
1915
  getTurnCheckpoints(agentName) {
1619
- const data = this.store.getStateJson(TURN_CHECKPOINTS_ID);
1620
- if (!data || typeof data !== 'object')
1916
+ const entry = this.store.treeGet(TURN_CHECKPOINTS_TREE_ID, agentName);
1917
+ if (entry) {
1918
+ const blob = this.store.getBlob(entry.blobHash);
1919
+ if (!blob)
1920
+ return [];
1921
+ try {
1922
+ const parsed = JSON.parse(blob.toString());
1923
+ return Array.isArray(parsed) ? parsed : [];
1924
+ }
1925
+ catch {
1926
+ return [];
1927
+ }
1928
+ }
1929
+ if (!this.hasLegacyCheckpoints(agentName))
1621
1930
  return [];
1622
- const allCheckpoints = data;
1623
- return Array.isArray(allCheckpoints[agentName]) ? [...allCheckpoints[agentName]] : [];
1931
+ const legacy = this.store.getStateJson(TURN_CHECKPOINTS_ID);
1932
+ const list = legacy?.[agentName];
1933
+ return Array.isArray(list) ? [...list] : [];
1624
1934
  }
1625
1935
  saveTurnCheckpoints(agentName, checkpoints) {
1626
- const data = this.store.getStateJson(TURN_CHECKPOINTS_ID);
1627
- const allCheckpoints = (data && typeof data === 'object' ? data : {});
1628
- allCheckpoints[agentName] = checkpoints;
1629
- this.store.setStateJson(TURN_CHECKPOINTS_ID, allCheckpoints);
1936
+ const bytes = Buffer.from(JSON.stringify(checkpoints));
1937
+ const blobHash = this.store.storeBlob(bytes, 'application/json');
1938
+ this.store.treeSet(TURN_CHECKPOINTS_TREE_ID, agentName, {
1939
+ blobHash,
1940
+ size: bytes.length,
1941
+ mode: 0o644,
1942
+ });
1943
+ }
1944
+ /**
1945
+ * Drop a departed agent's checkpoint tree key and turn/redo bookkeeping.
1946
+ * Without this, spawn-and-dispose agents leave a key (and in-memory map
1947
+ * entries) behind for the life of the store/session.
1948
+ */
1949
+ evictTurnCheckpoints(agentName) {
1950
+ this.turnCounters.delete(agentName);
1951
+ this.redoStacks.delete(agentName);
1952
+ // Diagnostics maps are also keyed by agent name and never evicted —
1953
+ // spawn-and-dispose fleets would grow them for the session's life.
1954
+ this.staleWarnAt.delete(agentName);
1955
+ this.lastInferenceAt.delete(agentName);
1956
+ const entry = this.store.treeGet(TURN_CHECKPOINTS_TREE_ID, agentName);
1957
+ if (this.hasLegacyCheckpoints(agentName)) {
1958
+ // A bare treeRemove would resurrect the legacy map's list through the
1959
+ // fallback next time this name is reused — shadow it with an empty
1960
+ // tombstone instead. size <= 2 ⇔ the blob is already "[]".
1961
+ if (!entry || entry.size > 2) {
1962
+ this.saveTurnCheckpoints(agentName, []);
1963
+ }
1964
+ return;
1965
+ }
1966
+ if (entry) {
1967
+ this.store.treeRemove(TURN_CHECKPOINTS_TREE_ID, agentName);
1968
+ }
1630
1969
  }
1631
1970
  /**
1632
1971
  * Run until the queue is empty and all agents are idle.
@@ -1634,6 +1973,9 @@ export class AgentFramework {
1634
1973
  */
1635
1974
  async runUntilIdle() {
1636
1975
  while (!this.queue.isEmpty ||
1976
+ // Direct inference requests (e.g. runEphemeralToCompletion) bypass the
1977
+ // event queue — without this the loop can exit before they're drained.
1978
+ this.pendingRequests.length > 0 ||
1637
1979
  this.activeStreams.size > 0 ||
1638
1980
  Array.from(this.agents.values()).some((a) => a.state.status !== 'idle')) {
1639
1981
  await this.processNextEvent();
@@ -2083,6 +2425,7 @@ export class AgentFramework {
2083
2425
  this.agents.delete(agentName);
2084
2426
  this.agentConfigs.delete(agentName);
2085
2427
  this.conversationAgentHomes.delete(agentName);
2428
+ this.evictTurnCheckpoints(agentName);
2086
2429
  this.emitTrace({
2087
2430
  type: 'mcpl:conversation-disposed',
2088
2431
  agentName,
@@ -2257,6 +2600,7 @@ export class AgentFramework {
2257
2600
  requestCount: requests.length,
2258
2601
  oldestRequestAge: now - oldest,
2259
2602
  });
2603
+ console.error(`[inference-dropped] agent=${agentName} reason=agent_not_found requests=${requests.length}`);
2260
2604
  continue;
2261
2605
  }
2262
2606
  // Skip if agent is busy (inferring, streaming, or waiting for tools)
@@ -2271,12 +2615,23 @@ export class AgentFramework {
2271
2615
  requestCount: requests.length,
2272
2616
  oldestRequestAge: now - oldest,
2273
2617
  });
2618
+ // Loud (but throttled) note that requests are waiting on a busy agent.
2619
+ if ((this.staleWarnAt.get(agentName) ?? 0) < now - 60_000) {
2620
+ this.staleWarnAt.set(agentName, now);
2621
+ console.error(`[inference-stale] agent=${agentName} busy (${agent.state.status}) — ` +
2622
+ `${requests.length} request(s) waiting ${Math.round((now - oldest) / 1000)}s`);
2623
+ }
2274
2624
  }
2275
2625
  this.pendingRequests.push(...requests);
2276
2626
  continue;
2277
2627
  }
2278
2628
  // Check policy
2279
2629
  if (!this.inferencePolicy.shouldInfer(agentName, requests, state)) {
2630
+ // Loud drop: a queued request that dies here is otherwise invisible —
2631
+ // the 2026-07-09 mythos "not responding" diagnosis burned hours on
2632
+ // exactly this class of silent drop. One stderr line per drop.
2633
+ console.error(`[inference-dropped] agent=${agentName} reason=policy-skip ` +
2634
+ `requests=${requests.length} triggers=${requests.map((r) => r.reason).join(',')}`);
2280
2635
  continue;
2281
2636
  }
2282
2637
  // Start streaming inference (non-blocking — driveStream runs in background)
@@ -2313,20 +2668,20 @@ export class AgentFramework {
2313
2668
  }
2314
2669
  this.emitTrace({ type: 'inference:started', agentName: agent.name });
2315
2670
  this.eventGate?.onInferenceStarted(agent.name);
2671
+ this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), startedAt: Date.now() });
2316
2672
  try {
2317
2673
  const allTools = this.getAllTools();
2318
2674
  const tools = allTools.filter((t) => agent.canUseTool(t.name));
2319
2675
  // Gather context from modules (pull-based) and MCPL hooks (push-based)
2320
2676
  // Both produce ContextInjection[] that get merged before inference.
2321
2677
  let injections;
2322
- // Module gatherContext (fail-open, 5s timeout per module)
2323
- // NOTE: module injections are NOT channel-scoped — only the MCPL hook
2324
- // injections below pass through scopeInjectionsForAgent (channel
2325
- // context arrives via beforeInference hooks by adapter convention). A
2326
- // module that starts emitting per-channel context must scope it per
2327
- // agent itself, or conversation forks will see cross-channel content.
2678
+ // Module gatherContext (fail-open, per-module timeout — the module's
2679
+ // contextTimeoutMs, else the registry default). Injections are
2680
+ // channel-scoped via scopeInjectionsForAgent, matching the MCPL hook
2681
+ // injections below, so conversation forks don't see cross-channel
2682
+ // content.
2328
2683
  try {
2329
- const moduleInjections = await this.moduleRegistry.gatherContext(agent.name);
2684
+ const moduleInjections = this.scopeInjectionsForAgent(agent.name, await this.moduleRegistry.gatherContext(agent.name));
2330
2685
  if (moduleInjections.length > 0) {
2331
2686
  injections = moduleInjections;
2332
2687
  }
@@ -2370,6 +2725,13 @@ export class AgentFramework {
2370
2725
  type: 'inference:exhausted',
2371
2726
  agentName: agent.name,
2372
2727
  error: err.message,
2728
+ // Drives the poison-history breaker. `retryable` is kept for
2729
+ // observability, but the breaker gates on `errorType` — only an
2730
+ // `invalid_request` (a 400-class rejection of the history itself)
2731
+ // means retrying the same context can never succeed. auth/abort/
2732
+ // context_length are also non-retryable but must NOT cost history.
2733
+ retryable: err instanceof MembraneError ? err.retryable : undefined,
2734
+ errorType: err instanceof MembraneError ? err.type : undefined,
2373
2735
  });
2374
2736
  this.eventGate?.onInferenceEnded(agent.name);
2375
2737
  if (action.emit) {
@@ -2608,7 +2970,73 @@ export class AgentFramework {
2608
2970
  const category = stopDetails?.category ?? 'unknown';
2609
2971
  console.error(`[inference-refusal] agent=${agent.name} category=${category}` +
2610
2972
  (stopDetails?.explanation ? ` explanation=${stopDetails.explanation}` : ''));
2611
- void this.reactToRefusal(agent.name, category);
2973
+ // Rewind: excise the turn that fed the refusal, drop a
2974
+ // metadata-only marker, and retry — keeping the agent on its own
2975
+ // model instead of substituting a fallback. Driven either by an
2976
+ // admin `/unstick` (forced session) or the agent's autoRewind
2977
+ // config; both are bounded.
2978
+ let handledByRewind = false;
2979
+ const rh = agent.refusalHandling;
2980
+ const forced = this.forcedRewind.get(agent.name);
2981
+ // Shed exactly one more (newest, in-sequence) turn and keep the
2982
+ // single episode marker current. `budgetLeft` bounds the loop.
2983
+ const doRewind = (budgetLeft, onStep, onGiveUp) => {
2984
+ if (!budgetLeft) {
2985
+ onGiveUp();
2986
+ return;
2987
+ }
2988
+ const rec = this.shedNewestTurn(agent);
2989
+ if (!rec) {
2990
+ onGiveUp();
2991
+ return;
2992
+ }
2993
+ const count = this.updateRewindMarker(agent, category);
2994
+ onStep(count);
2995
+ if (forced)
2996
+ forced.removed.push(rec);
2997
+ this.pendingRequests.push({
2998
+ agentName: agent.name,
2999
+ reason: forced ? 'unstick-retry' : 'refusal-rewind-retry',
3000
+ source: 'framework',
3001
+ timestamp: Date.now(),
3002
+ });
3003
+ handledByRewind = true;
3004
+ };
3005
+ if (forced) {
3006
+ doRewind(forced.remaining > 0, (count) => {
3007
+ forced.remaining -= 1;
3008
+ console.error(`[unstick] agent=${agent.name} shed ${count} turn(s) ` +
3009
+ `(remaining ${forced.remaining})`);
3010
+ }, () => this.finishUnstick(agent.name, false, category));
3011
+ }
3012
+ else if (rh?.autoRewind) {
3013
+ const cap = Math.max(1, rh.maxRewinds ?? 3);
3014
+ const used = this.refusalRewinds.get(agent.name) ?? 0;
3015
+ doRewind(used < cap, (count) => {
3016
+ this.refusalRewinds.set(agent.name, used + 1);
3017
+ console.error(`[refusal-rewind] agent=${agent.name} shed ${count} turn(s) so far ` +
3018
+ `(cap ${cap})`);
3019
+ }, () => {
3020
+ console.error(`[refusal-rewind] agent=${agent.name} gave up: ` +
3021
+ `${used >= cap ? `cap ${cap} reached` : 'nothing left to shed'}`);
3022
+ this.refusalRewinds.set(agent.name, 0);
3023
+ this.rewindEpisode.delete(agent.name);
3024
+ });
3025
+ }
3026
+ if (!handledByRewind)
3027
+ void this.reactToRefusal(agent.name, category);
3028
+ }
3029
+ else {
3030
+ // A turn that completed WITHOUT a refusal ends the rewind episode:
3031
+ // the model responded. Leave the consolidated marker in place as
3032
+ // the durable record; just clear the per-episode counters.
3033
+ if (this.forcedRewind.has(agent.name)) {
3034
+ this.finishUnstick(agent.name, true);
3035
+ }
3036
+ if (this.refusalRewinds.get(agent.name)) {
3037
+ this.refusalRewinds.set(agent.name, 0);
3038
+ }
3039
+ this.rewindEpisode.delete(agent.name);
2612
3040
  }
2613
3041
  // Dispatch speech (and thoughts if any)
2614
3042
  if (speechContent.length > 0 || thoughts.length > 0) {
@@ -2692,11 +3120,19 @@ export class AgentFramework {
2692
3120
  `(${silenced ? 'silencing tool / explicit send' : 'no prose'})`);
2693
3121
  }
2694
3122
  else {
2695
- console.error(`[routing] ${agent.name}: tool-call turn [${toolNames.join(', ')}] -> routing ${segments.length} prose segment(s) in order`);
3123
+ // Snapshot the locus ONCE. This loop dispatches AFTER the agent
3124
+ // is idle (see PR #32 note below), so a queued inbound from
3125
+ // another channel can start a NEW turn and overwrite the per-agent
3126
+ // triggering channel (`activeTriggerChannels`) BETWEEN segments —
3127
+ // splitting one turn's reply across channels (the residual
3128
+ // multi-segment race left by 83dedf8's per-agent locus). Pin every
3129
+ // segment of THIS turn to the channel resolved right now.
3130
+ const turnLocus = this.channelRegistry.resolveLocus(agent.name);
3131
+ console.error(`[routing] ${agent.name}: tool-call turn [${toolNames.join(', ')}] -> routing ${segments.length} prose segment(s) in order -> ${turnLocus ?? '(default)'}`);
2696
3132
  // Deliver sequentially (await each) so the segments land in order.
2697
3133
  for (const seg of segments) {
2698
3134
  try {
2699
- await this.channelRegistry.routeSpeech(agent.name, seg);
3135
+ await this.channelRegistry.routeSpeech(agent.name, seg, turnLocus);
2700
3136
  }
2701
3137
  catch (err) {
2702
3138
  console.error('speech routing failed:', err);
@@ -2742,6 +3178,12 @@ export class AgentFramework {
2742
3178
  type: 'inference:exhausted',
2743
3179
  agentName: agent.name,
2744
3180
  error: err.message,
3181
+ // Drives the poison-history breaker. `retryable` is kept for
3182
+ // observability, but the breaker gates on `errorType` — only an
3183
+ // `invalid_request` (a 400-class rejection of the history
3184
+ // itself) means retrying the same context can never succeed.
3185
+ retryable: err instanceof MembraneError ? err.retryable : undefined,
3186
+ errorType: err instanceof MembraneError ? err.type : undefined,
2745
3187
  });
2746
3188
  this.eventGate?.onInferenceEnded(agent.name);
2747
3189
  if (action.emit) {
@@ -2755,18 +3197,51 @@ export class AgentFramework {
2755
3197
  // Only reset if this is still the active stream (a budget restart
2756
3198
  // may have already started a new stream, bumping streamId)
2757
3199
  if (agent.streamId === myStreamId) {
3200
+ const durationMs = Date.now() - startTime;
2758
3201
  agent.reset();
2759
3202
  this.emitTrace({
2760
3203
  type: 'inference:exhausted',
2761
3204
  agentName: agent.name,
2762
3205
  error: `Stream aborted: ${reason}`,
2763
3206
  });
3207
+ // Postmortem 2026-05-28 P2 #7: persist the abort to the
3208
+ // inference log so future investigations can attribute the
3209
+ // terminal cause without relying on live in-memory reducer
3210
+ // state. Without this, abort-terminated inferences are
3211
+ // invisible to forensic queries (only request-side telemetry
3212
+ // via llm-calls.jsonl shows them, and only by absence).
3213
+ this.logInference({
3214
+ timestamp: startTime,
3215
+ agentName: agent.name,
3216
+ requestId,
3217
+ success: false,
3218
+ error: `Stream aborted: ${reason}`,
3219
+ request: compiledRequest ?? { note: 'streaming request aborted' },
3220
+ durationMs,
3221
+ });
2764
3222
  this.eventGate?.onInferenceEnded(agent.name);
2765
3223
  }
2766
3224
  break;
2767
3225
  }
2768
- case 'usage':
3226
+ case 'usage': {
2769
3227
  agent.lastStreamInputTokens = event.usage.inputTokens;
3228
+ // Closed-loop estimator calibration (2026-07-12). Sample the REAL
3229
+ // prefix size of THIS API call (fresh + cache write + cache read)
3230
+ // and hand it to the context strategy, which accepts exactly one
3231
+ // sample per compile (its arm-once gate) and rejects out-of-band
3232
+ // ratios. It must be sampled HERE, per call: `response.details.
3233
+ // usage` at turn completion is CUMULATIVE across the tool-use
3234
+ // loop (5 calls x ~160k reported as 884k), which is not a
3235
+ // window-shaped number and drove the multiplier to 2.37 before
3236
+ // the guards caught it.
3237
+ try {
3238
+ const realTotal = (event.usage.inputTokens ?? 0) +
3239
+ (event.usage.cacheCreationTokens ?? 0) +
3240
+ (event.usage.cacheReadTokens ?? 0);
3241
+ const strat = agent.getContextManager?.()?.getStrategy?.();
3242
+ strat?.reportRealInputTokens?.(realTotal);
3243
+ }
3244
+ catch { /* calibration is best-effort */ }
2770
3245
  this.emitTrace({
2771
3246
  type: 'inference:usage',
2772
3247
  agentName: agent.name,
@@ -2778,6 +3253,7 @@ export class AgentFramework {
2778
3253
  },
2779
3254
  });
2780
3255
  break;
3256
+ }
2781
3257
  }
2782
3258
  }
2783
3259
  }
@@ -2785,6 +3261,7 @@ export class AgentFramework {
2785
3261
  // Stream itself threw (unexpected) — no retry path here, so also emit
2786
3262
  // inference:exhausted so ephemeral agent promises can settle.
2787
3263
  const err = error instanceof Error ? error : new Error(String(error));
3264
+ const durationMs = Date.now() - startTime;
2788
3265
  this.emitTrace({
2789
3266
  type: 'inference:failed',
2790
3267
  agentName: agent.name,
@@ -2796,10 +3273,31 @@ export class AgentFramework {
2796
3273
  agentName: agent.name,
2797
3274
  error: err.message,
2798
3275
  });
3276
+ // Postmortem 2026-05-28 P2 #7: catch-path failures (stream itself
3277
+ // threw) were previously only visible via in-memory trace listeners.
3278
+ // Persist to inference log for forensic attribution.
3279
+ this.logInference({
3280
+ timestamp: startTime,
3281
+ agentName: agent.name,
3282
+ requestId,
3283
+ success: false,
3284
+ error: `Stream threw: ${err.message}`,
3285
+ request: compiledRequest ?? { note: 'streaming request threw' },
3286
+ durationMs,
3287
+ });
2799
3288
  agent.reset();
2800
3289
  this.eventGate?.onInferenceEnded(agent.name);
2801
3290
  }
2802
3291
  finally {
3292
+ // Clear the gate's inference flag on EVERY exit path (paired with the
3293
+ // onInferenceStarted in startAgentStream). The branch-level calls above
3294
+ // are kept but are best-effort; if any exit path bypassed them the agent
3295
+ // stayed stuck in the gate's `inferring` set, which permanently BUFFERS
3296
+ // all incoming events → the agent silently stops waking on messages
3297
+ // (typing still stops, compression still runs — matching the observed
3298
+ // wedge). onInferenceEnded is idempotent, so a redundant call is safe.
3299
+ this.eventGate?.onInferenceEnded(agent.name);
3300
+ this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), endedAt: Date.now() });
2803
3301
  // Stop the typing indicator on every exit path (complete, error,
2804
3302
  // exhausted, abort) so it never sticks after the turn ends.
2805
3303
  this.channelRegistry?.stopTyping();
@@ -2884,7 +3382,10 @@ export class AgentFramework {
2884
3382
  if (blocks) {
2885
3383
  return { toolUseId: callId, content: blocks, isError: false };
2886
3384
  }
2887
- let content = JSON.stringify(afResult.data);
3385
+ // JSON.stringify returns the VALUE undefined (not a string) for undefined
3386
+ // input — a module tool returning `{ success: true }` with no data would
3387
+ // otherwise make `content.length` below throw a TypeError mid-turn.
3388
+ let content = JSON.stringify(afResult.data) ?? '';
2888
3389
  if (maxChars && content.length > maxChars) {
2889
3390
  content = safeSlice(content, 0, maxChars)
2890
3391
  + '\n\n[truncated — original was ' + content.length + ' chars]';
@@ -2954,11 +3455,11 @@ export class AgentFramework {
2954
3455
  entryToStore.response = { blobId };
2955
3456
  }
2956
3457
  }
2957
- // Append to the inference log state
2958
- const data = this.store.getStateJson(INFERENCE_LOG_ID);
2959
- const entries = Array.isArray(data) ? data : [];
2960
- entries.push(entryToStore);
2961
- this.store.setStateJson(INFERENCE_LOG_ID, entries);
3458
+ // Append one record per inference. A read-modify-write via setStateJson
3459
+ // would emit a whole-array Set — record size then grows with accumulated
3460
+ // history (O(n²) aggregate disk) and the append_log strategy registered
3461
+ // for this state never sees an append.
3462
+ this.store.appendToStateJson(INFERENCE_LOG_ID, entryToStore);
2962
3463
  }
2963
3464
  logProcessEvent(event, responses) {
2964
3465
  const entry = {
@@ -2974,11 +3475,8 @@ export class AgentFramework {
2974
3475
  const blobId = this.store.storeBlob(Buffer.from(responsesJson), 'application/json');
2975
3476
  entryToStore.responses = { blobId };
2976
3477
  }
2977
- // Append to the process log state
2978
- const data = this.store.getStateJson(PROCESS_LOG_ID);
2979
- const entries = Array.isArray(data) ? data : [];
2980
- entries.push(entryToStore);
2981
- this.store.setStateJson(PROCESS_LOG_ID, entries);
3478
+ // Append one record per event — same rationale as logInference.
3479
+ this.store.appendToStateJson(PROCESS_LOG_ID, entryToStore);
2982
3480
  }
2983
3481
  /**
2984
3482
  * Find the MCPL server for a tool call by checking against the prefix map.
@@ -3175,15 +3673,19 @@ export class AgentFramework {
3175
3673
  // without this the only durable record of a failed inference is a field in
3176
3674
  // llm-calls.jsonl — invisible to operator, agent, and monitoring.
3177
3675
  if (event.type === 'inference:exhausted') {
3178
- this.noteInferenceExhausted(event.agentName ?? 'unknown', event.error ?? 'unknown error');
3676
+ this.noteInferenceExhausted(event.agentName ?? 'unknown', event.error ?? 'unknown error', event.retryable, event.errorType);
3179
3677
  }
3180
3678
  else if (event.type === 'inference:completed') {
3181
3679
  // A successful response — even mid-turn between tool calls — proves the
3182
- // agent isn't hard-down; clear its consecutive-failure streak.
3680
+ // agent isn't hard-down; clear its consecutive-failure streak (and the
3681
+ // poison-history breaker's rewind budget).
3183
3682
  const name = event.agentName;
3184
3683
  if (name && this.consecutiveInferenceFailures.get(name)) {
3185
3684
  this.consecutiveInferenceFailures.set(name, 0);
3186
3685
  }
3686
+ if (name && this.exhaustionRewinds.get(name)) {
3687
+ this.exhaustionRewinds.set(name, 0);
3688
+ }
3187
3689
  }
3188
3690
  const traceEvent = {
3189
3691
  ...event,
@@ -3212,12 +3714,49 @@ export class AgentFramework {
3212
3714
  * inference, so this never causes a retry/wake loop.
3213
3715
  * 3. escalation — after N consecutive failures the agent is hard-down;
3214
3716
  * log that loudly (a repeated identical failure is the textbook signal).
3717
+ * 4. breaker — when the failures are an `invalid_request` (a 400-class
3718
+ * rejection of the history itself, e.g. corrupted tool_use/tool_result
3719
+ * pairing or an oversized attachment), retrying the same context can
3720
+ * never succeed: every new push event wakes the agent onto the same
3721
+ * poisoned history forever. At the hard-down threshold, automatically
3722
+ * quarantine: shed the newest complete exchange (the same forced-rewind
3723
+ * primitive `/unstick` uses — shedNewestTurn never orphans a
3724
+ * tool_use/thinking block) and retry, bounded by the same rewind cap so
3725
+ * the breaker can never eat the whole history.
3726
+ */
3727
+ /**
3728
+ * Live health snapshot for /healthz and doctor tooling: gate state, queued
3729
+ * work, per-agent status + last inference activity. Cheap and read-only.
3215
3730
  */
3216
- noteInferenceExhausted(agentName, reason) {
3731
+ healthSnapshot() {
3732
+ return {
3733
+ at: new Date().toISOString(),
3734
+ uptimeSec: Math.round(process.uptime()),
3735
+ gate: this.eventGate?.inferenceDiagnostics() ?? null,
3736
+ pendingRequests: this.pendingRequests.length,
3737
+ activeStreams: [...this.activeStreams.keys()],
3738
+ agents: [...this.agents.entries()].map(([name, agent]) => ({
3739
+ name,
3740
+ status: agent.state.status,
3741
+ consecutiveInferenceFailures: this.consecutiveInferenceFailures.get(name) ?? 0,
3742
+ lastInference: this.lastInferenceAt.get(name) ?? null,
3743
+ })),
3744
+ };
3745
+ }
3746
+ noteInferenceExhausted(agentName, reason, retryable, errorType) {
3217
3747
  const streak = (this.consecutiveInferenceFailures.get(agentName) ?? 0) + 1;
3218
3748
  this.consecutiveInferenceFailures.set(agentName, streak);
3749
+ this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), failedAt: Date.now(), lastError: reason.slice(0, 300) });
3219
3750
  // (1) Durable stderr line — works in headless/daemon mode with no client.
3220
3751
  console.error(`[inference-failed] agent=${agentName} consecutive=${streak}: ${reason}`);
3752
+ // (1b) Machine-greppable durable record, independent of journald/unit log
3753
+ // redirects: logs/failures.log under the host's working directory. This is
3754
+ // what connectome-doctor reads.
3755
+ try {
3756
+ mkdirSync('logs', { recursive: true });
3757
+ appendFileSync('logs/failures.log', JSON.stringify({ at: new Date().toISOString(), agent: agentName, consecutive: streak, reason }) + '\n');
3758
+ }
3759
+ catch { /* best-effort */ }
3221
3760
  // (2) Agent-facing chronicle marker (no inference triggered → no loop).
3222
3761
  const agent = this.agents.get(agentName);
3223
3762
  if (agent && process.env.SUPPRESS_INFERENCE_FAILED_MARKER !== '1') {
@@ -3239,8 +3778,71 @@ export class AgentFramework {
3239
3778
  if (streak >= this.inferenceFailureEscalationThreshold) {
3240
3779
  console.error(`[inference-hard-down] agent=${agentName} has FAILED ${streak} consecutive ` +
3241
3780
  `inferences — it cannot complete a turn. Last reason: ${reason}`);
3781
+ // Optional ops alert: set CONNECTOME_OPS_WEBHOOK to a Discord webhook URL.
3782
+ const hook = process.env.CONNECTOME_OPS_WEBHOOK;
3783
+ if (hook) {
3784
+ fetch(hook, {
3785
+ method: 'POST',
3786
+ headers: { 'content-type': 'application/json' },
3787
+ body: JSON.stringify({
3788
+ content: `\u{1F6A8} **${agentName}** hard-down: ${streak} consecutive inference failures.\nLast reason: ${reason.slice(0, 500)}`,
3789
+ }),
3790
+ }).catch((err) => console.error('[inference-hard-down] webhook post failed:', err instanceof Error ? err.message : err));
3791
+ }
3792
+ // (4) Poison-history breaker: ONLY for `invalid_request` — a 400-class
3793
+ // rejection of the history itself. Deliberately NOT keyed on
3794
+ // `retryable === false`, which membrane also returns for auth (expired
3795
+ // key), abort (deliberate cancel), context_length (compression's job,
3796
+ // and shedding newest is the wrong direction), safety and unsupported —
3797
+ // none of which mean the history is poisoned, and all of which would
3798
+ // otherwise shed good exchanges and stamp a false "the API kept
3799
+ // rejecting your history" marker. `retryable` is kept only for the trace.
3800
+ void retryable;
3801
+ if (errorType === 'invalid_request' && agent) {
3802
+ this.quarantinePoisonedHistory(agent, reason);
3803
+ }
3242
3804
  }
3243
3805
  }
3806
+ /**
3807
+ * Automatic poison-history quarantine (the actual "breaker"). Sheds the
3808
+ * newest complete exchange from the agent's history — reusing the same
3809
+ * primitives as the refusal auto-rewind / `/unstick` (shedNewestTurn +
3810
+ * the single consolidated episode marker) — and queues a retry so the
3811
+ * rewound history is verified immediately instead of waiting for the next
3812
+ * push event to wake the agent onto the same poisoned context.
3813
+ *
3814
+ * Bounded: at most `refusalHandling.maxRewinds` (default 3, hard cap 10)
3815
+ * sheds per failure episode; the budget resets on any successful inference.
3816
+ * At the cap (or with nothing left to shed) it stops — the agent stays up
3817
+ * and hard-down logging continues, but no further history is consumed.
3818
+ */
3819
+ quarantinePoisonedHistory(agent, reason) {
3820
+ const cap = Math.max(1, Math.min(10, agent.refusalHandling?.maxRewinds ?? 3));
3821
+ const used = this.exhaustionRewinds.get(agent.name) ?? 0;
3822
+ if (used >= cap) {
3823
+ console.error(`[inference-rewind] agent=${agent.name} rewind cap ${cap} reached — ` +
3824
+ `not shedding further history. Manual repair (/unstick or /undo) needed.`);
3825
+ return;
3826
+ }
3827
+ const rec = this.shedNewestTurn(agent);
3828
+ if (!rec) {
3829
+ console.error(`[inference-rewind] agent=${agent.name} has nothing left to shed — ` +
3830
+ `history is only system markers. Manual repair needed.`);
3831
+ return;
3832
+ }
3833
+ this.exhaustionRewinds.set(agent.name, used + 1);
3834
+ const count = this.updateRewindMarker(agent, truncateReason(reason), 'inference-failure');
3835
+ console.error(`[inference-rewind] agent=${agent.name} auto-quarantined ${rec.descriptor} ` +
3836
+ `(${count} turn(s) shed this episode, cap ${cap}) after non-retryable ` +
3837
+ `inference failures — retrying on the rewound history.`);
3838
+ // Retry immediately on the repaired history (bounded by the cap above).
3839
+ this.pendingRequests.push({
3840
+ agentName: agent.name,
3841
+ reason: 'inference-failure-rewind-retry',
3842
+ source: 'framework',
3843
+ timestamp: Date.now(),
3844
+ });
3845
+ }
3244
3846
  // ==========================================================================
3245
3847
  // MCPL subsystem wiring
3246
3848
  // ==========================================================================
@@ -3314,8 +3916,9 @@ export class AgentFramework {
3314
3916
  }
3315
3917
  },
3316
3918
  });
3317
- // Host capabilities advertised during the MCP handshake
3318
- const hostCapabilities = {
3919
+ // Host capabilities advertised during the MCP handshake — stored so
3920
+ // servers can also be connected later at runtime (connectMcplServer).
3921
+ this.mcplHostCapabilities = {
3319
3922
  version: '0.4',
3320
3923
  pushEvents: true,
3321
3924
  contextHooks: {
@@ -3326,56 +3929,193 @@ export class AgentFramework {
3326
3929
  };
3327
3930
  for (const config of serverConfigs) {
3328
3931
  try {
3329
- // Record per-server channel subscription policy before the server
3330
- // registers channels — handleRegister fires during the handshake.
3331
- if (config.channelSubscription !== undefined && this.channelRegistry) {
3332
- this.channelRegistry.setSubscriptionPolicy(config.id, config.channelSubscription);
3333
- }
3334
- const connection = await this.mcplServerRegistry.addServer(config, hostCapabilities);
3335
- // Wire event listeners then flush any events that arrived during the
3336
- // handshake window (between setupMessageRouting and now).
3337
- this.wireMcplEvents(connection);
3338
- connection.ready();
3339
- // Initialize feature sets if server advertises MCPL capabilities
3340
- if (connection.capabilities) {
3341
- const updateParams = this.featureSetManager.initializeServer(config.id, connection.capabilities, {
3342
- enabledFeatureSets: config.enabledFeatureSets,
3343
- disabledFeatureSets: config.disabledFeatureSets,
3344
- });
3345
- // Inform server which feature sets are enabled/disabled
3346
- if (updateParams.enabled?.length || updateParams.disabled?.length) {
3347
- connection.sendFeatureSetsUpdate(updateParams);
3348
- }
3349
- // Configure scope whitelist/blacklist patterns
3350
- if (config.scopes) {
3351
- this.scopeManager.configureAll(config.scopes);
3352
- }
3353
- // Register stateful feature sets with checkpoint manager (Step 8)
3354
- if (this.checkpointManager) {
3355
- const declared = this.featureSetManager.getDeclaredFeatureSets(config.id);
3356
- if (declared) {
3357
- for (const [fsName, fsDecl] of Object.entries(declared)) {
3358
- if (fsDecl.rollback || fsDecl.hostState) {
3359
- this.checkpointManager.registerFeatureSet(config.id, fsName, {
3360
- hostState: fsDecl.hostState ?? false,
3361
- rollback: fsDecl.rollback ?? false,
3362
- });
3363
- }
3364
- }
3365
- }
3366
- }
3367
- }
3368
- this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` });
3932
+ await this.connectMcplServerInternal(config);
3369
3933
  }
3370
3934
  catch (error) {
3371
3935
  // Fail-open: log and continue with remaining servers
3372
3936
  const err = error instanceof Error ? error : new Error(String(error));
3373
3937
  console.error(`Failed to connect MCPL server "${config.id}":`, err.message);
3938
+ // Reaching this catch means no stub was created, so nothing will
3939
+ // retry — except in the edge case where addServer succeeded (with a
3940
+ // reconnecting stub) and a later setup step threw; config.reconnect
3941
+ // is the best cheap approximation of that.
3942
+ this.emitTrace({
3943
+ type: 'mcpl:server-connect-failed',
3944
+ serverId: config.id,
3945
+ error: err.message,
3946
+ attempt: 0,
3947
+ willRetry: config.reconnect === true,
3948
+ });
3374
3949
  }
3375
3950
  }
3376
3951
  // Discover tools from all connected servers
3377
3952
  await this.refreshMcplTools();
3378
3953
  }
3954
+ /**
3955
+ * Connect a single MCPL server: register routing entries, open the
3956
+ * connection, wire events, and initialize feature sets / scopes /
3957
+ * checkpoints. Shared by startup (initializeMcpl) and the runtime
3958
+ * lifecycle API (connectMcplServer). Throws on connection failure.
3959
+ */
3960
+ async connectMcplServerInternal(config) {
3961
+ if (!this.mcplServerRegistry || !this.mcplHostCapabilities) {
3962
+ throw new Error('MCPL subsystem is not initialized');
3963
+ }
3964
+ // Register prefix + config for tool dispatch routing (idempotent with
3965
+ // the pre-registration pass in initializeMcpl).
3966
+ const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
3967
+ this.mcplPrefixMap.set(prefix, config.id);
3968
+ this.mcplServerConfigs.set(config.id, config);
3969
+ // Record per-server channel subscription policy before the server
3970
+ // registers channels — handleRegister fires during the handshake.
3971
+ if (config.channelSubscription !== undefined && this.channelRegistry) {
3972
+ this.channelRegistry.setSubscriptionPolicy(config.id, config.channelSubscription);
3973
+ }
3974
+ const connection = await this.mcplServerRegistry.addServer(config, this.mcplHostCapabilities);
3975
+ // Wire event listeners then flush any events that arrived during the
3976
+ // handshake window (between setupMessageRouting and now).
3977
+ this.wireMcplEvents(connection);
3978
+ connection.ready();
3979
+ // Initialize feature sets if server advertises MCPL capabilities
3980
+ this.registerMcplServerFeatures(config, connection);
3981
+ this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` });
3982
+ }
3983
+ /**
3984
+ * (Re-)establish a server's host-side MCPL registration: feature-set state
3985
+ * (plus the featureSets/update sent to the server), scope patterns, and
3986
+ * stateful-feature-set checkpoint registration.
3987
+ *
3988
+ * Called on initial connect (connectMcplServerInternal) AND after every
3989
+ * auto-reconnect. The 'close' handler drops the feature-set registration, so
3990
+ * without the reconnect re-run FeatureSetManager.validateInbound would throw
3991
+ * "Unknown server" forever — every push event and inference request from the
3992
+ * revived server silently rejected until a full host restart.
3993
+ *
3994
+ * CheckpointManager.registerFeatureSet is idempotent (no-op for a key that
3995
+ * already has a tree), so checkpoint trees preserved across a transient
3996
+ * disconnect are resumed, not reset.
3997
+ */
3998
+ registerMcplServerFeatures(config, connection) {
3999
+ if (!connection.capabilities)
4000
+ return;
4001
+ const updateParams = this.featureSetManager.initializeServer(config.id, connection.capabilities, {
4002
+ enabledFeatureSets: config.enabledFeatureSets,
4003
+ disabledFeatureSets: config.disabledFeatureSets,
4004
+ });
4005
+ // Inform server which feature sets are enabled/disabled
4006
+ if (updateParams.enabled?.length || updateParams.disabled?.length) {
4007
+ connection.sendFeatureSetsUpdate(updateParams);
4008
+ }
4009
+ // Configure scope whitelist/blacklist patterns
4010
+ if (config.scopes) {
4011
+ this.scopeManager.configureAll(config.scopes);
4012
+ }
4013
+ // Register stateful feature sets with checkpoint manager (Step 8)
4014
+ if (this.checkpointManager) {
4015
+ const declared = this.featureSetManager.getDeclaredFeatureSets(config.id);
4016
+ if (declared) {
4017
+ for (const [fsName, fsDecl] of Object.entries(declared)) {
4018
+ if (fsDecl.rollback || fsDecl.hostState) {
4019
+ this.checkpointManager.registerFeatureSet(config.id, fsName, {
4020
+ hostState: fsDecl.hostState ?? false,
4021
+ rollback: fsDecl.rollback ?? false,
4022
+ });
4023
+ }
4024
+ }
4025
+ }
4026
+ }
4027
+ }
4028
+ // ==========================================================================
4029
+ // Runtime MCPL server lifecycle (agent-facing hot deploy/restart/unload)
4030
+ // ==========================================================================
4031
+ /**
4032
+ * Connect a new MCPL server at runtime. Refreshes the tool list and
4033
+ * notifies the agent of newly available tools. Throws if the MCPL
4034
+ * subsystem is not initialized, the id is already connected, or the
4035
+ * connection/handshake fails.
4036
+ */
4037
+ async connectMcplServer(config) {
4038
+ // Lazily bring up the MCPL subsystem — a framework that started with zero
4039
+ // configured servers can still deploy its first one at runtime.
4040
+ if (!this.mcplServerRegistry || !this.mcplHostCapabilities) {
4041
+ await this.initializeMcpl([], this.mcplInferenceRoutingConfig ?? undefined);
4042
+ }
4043
+ // Same collision rules create() enforces at startup: the tool prefix must
4044
+ // not shadow a module name or another server's prefix.
4045
+ const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
4046
+ if (this.moduleRegistry.getAllModules().some(m => m.name === prefix)) {
4047
+ throw new Error(`MCPL server "${config.id}" toolPrefix "${prefix}" collides with module "${prefix}"`);
4048
+ }
4049
+ const prefixOwner = this.mcplPrefixMap.get(prefix);
4050
+ if (prefixOwner && prefixOwner !== config.id) {
4051
+ throw new Error(`MCPL server "${config.id}" toolPrefix "${prefix}" collides with server "${prefixOwner}"`);
4052
+ }
4053
+ const oldToolNames = new Set(this.mcplTools.map(t => t.name));
4054
+ await this.connectMcplServerInternal(config);
4055
+ await this.refreshMcplTools();
4056
+ this.emitMcplToolDiff(oldToolNames, config.id);
4057
+ }
4058
+ /**
4059
+ * Disconnect an MCPL server at runtime: close the connection, destroy its
4060
+ * feature-set and checkpoint state (permanent removal — unlike a transient
4061
+ * transport close, which preserves checkpoints for the reconnect), remove
4062
+ * its channels from the registry, drop routing entries, and refresh tools.
4063
+ * No-op-ish if the server is not connected (still clears routing state).
4064
+ */
4065
+ async disconnectMcplServer(id) {
4066
+ if (!this.mcplServerRegistry) {
4067
+ throw new Error('MCPL subsystem is not initialized');
4068
+ }
4069
+ const config = this.mcplServerConfigs.get(id);
4070
+ const oldToolNames = new Set(this.mcplTools.map(t => t.name));
4071
+ await this.mcplServerRegistry.removeServer(id);
4072
+ this.channelRegistry?.removeServer(id);
4073
+ // Permanent removal: also destroy feature-set and checkpoint state
4074
+ // explicitly. The 'close' handler usually does this, but a connection that
4075
+ // already transiently closed (reconnect pending) emits no second 'close'
4076
+ // from close(), and the transient path deliberately preserves checkpoints.
4077
+ this.featureSetManager?.removeServer(id);
4078
+ this.checkpointManager?.removeServer(id);
4079
+ const prefix = config?.toolPrefix ?? `mcpl--${id}`;
4080
+ this.mcplPrefixMap.delete(prefix);
4081
+ this.mcplServerConfigs.delete(id);
4082
+ await this.refreshMcplTools();
4083
+ this.emitMcplToolDiff(oldToolNames, id);
4084
+ }
4085
+ /**
4086
+ * Restart an MCPL server at runtime: disconnect, then reconnect with the
4087
+ * same (or an updated) config. This actually respawns a stdio child —
4088
+ * unlike `reconnect: true`, which only retries after transport-level
4089
+ * failures. Throws if the server has no stored config and none is given.
4090
+ */
4091
+ async restartMcplServer(id, newConfig) {
4092
+ const config = newConfig ?? this.mcplServerConfigs.get(id);
4093
+ if (!config) {
4094
+ throw new Error(`MCPL server "${id}" is not configured`);
4095
+ }
4096
+ await this.disconnectMcplServer(id);
4097
+ await this.connectMcplServer(config);
4098
+ }
4099
+ /**
4100
+ * List configured MCPL servers with live connection status and the number
4101
+ * of tools each currently contributes.
4102
+ */
4103
+ listMcplServers() {
4104
+ const result = [];
4105
+ for (const [id, config] of this.mcplServerConfigs) {
4106
+ const prefix = config.toolPrefix ?? `mcpl--${id}`;
4107
+ const connection = this.mcplServerRegistry?.getServer(id) ?? null;
4108
+ result.push({
4109
+ id,
4110
+ connected: connection?.isConnected ?? false,
4111
+ toolPrefix: prefix,
4112
+ toolCount: this.mcplTools.filter(t => t.name.startsWith(`${prefix}--`)).length,
4113
+ command: config.command,
4114
+ url: config.url,
4115
+ });
4116
+ }
4117
+ return result;
4118
+ }
3379
4119
  /**
3380
4120
  * Wire event listeners on an MCPL server connection.
3381
4121
  * Push events and inference requests are deferred to Steps 6/7.
@@ -3446,14 +4186,95 @@ export class AgentFramework {
3446
4186
  connection.on('tools-list-changed', () => {
3447
4187
  this.handleToolsListChanged(connection.id);
3448
4188
  });
3449
- // Also refresh tools on reconnect (server may have different tools)
3450
- connection.on('reconnect', () => {
4189
+ // Re-establish full server registration on reconnect. The 'close' handler
4190
+ // removed the feature-set state, so pushes / inference requests from the
4191
+ // revived server would otherwise be rejected with "Unknown server" until a
4192
+ // host restart. Re-runs the same init as the initial connect path (the
4193
+ // fresh handshake refreshed connection.capabilities); checkpoint trees
4194
+ // preserved across the transient close are resumed by idempotent
4195
+ // registration. Then refresh tools (server may have different tools).
4196
+ connection.on('reconnect', (info) => {
4197
+ try {
4198
+ const config = this.mcplServerConfigs.get(connection.id);
4199
+ if (config) {
4200
+ this.registerMcplServerFeatures(config, connection);
4201
+ }
4202
+ }
4203
+ catch (error) {
4204
+ console.error(`MCPL server "${connection.id}" re-registration after reconnect failed:`, error instanceof Error ? error.message : error);
4205
+ }
3451
4206
  this.handleToolsListChanged(connection.id);
4207
+ this.emitTrace({
4208
+ type: 'mcpl:server-reconnected',
4209
+ serverId: connection.id,
4210
+ attempts: info?.attempts ?? 0,
4211
+ });
4212
+ // Mirror the module:removed emitted on 'close' so module-lifecycle
4213
+ // consumers see the server come back, not just vanish.
4214
+ this.emitTrace({ type: 'module:added', moduleName: `mcpl:${connection.id}` });
3452
4215
  });
3453
- // Cleanup on disconnect
3454
- connection.on('close', () => {
4216
+ // Surface connect/reconnect failures. Before these traces existed the
4217
+ // only receipt was a console.error on the host's own stderr — invisible
4218
+ // unless someone ssh'd in and read the process log.
4219
+ connection.on('connect-failed', (params) => {
4220
+ this.emitTrace({
4221
+ type: 'mcpl:server-connect-failed',
4222
+ serverId: connection.id,
4223
+ error: params.error,
4224
+ attempt: params.attempt,
4225
+ willRetry: connection.willReconnect,
4226
+ });
4227
+ });
4228
+ connection.on('reconnect-failed', (params) => {
4229
+ this.emitTrace({
4230
+ type: 'mcpl:server-connect-failed',
4231
+ serverId: connection.id,
4232
+ error: params.error,
4233
+ attempt: params.attempt,
4234
+ willRetry: connection.willReconnect,
4235
+ });
4236
+ });
4237
+ // A late response to a tools/call that already timed out, carrying
4238
+ // stateful data the server advanced to. The host can't re-inject it (the
4239
+ // dispatch context is gone), but surfacing it makes the checkpoint-tree
4240
+ // divergence greppable instead of a silent drift into stale state.
4241
+ connection.on('orphaned-response', (info) => {
4242
+ this.emitTrace({
4243
+ type: 'mcpl:orphaned-response',
4244
+ serverId: connection.id,
4245
+ responseId: info.id,
4246
+ hadState: info.hadState,
4247
+ hadCheckpoint: info.hadCheckpoint,
4248
+ });
4249
+ });
4250
+ // Connection-level errors (e.g. child process 'error' after spawn).
4251
+ // Attaching this listener also keeps an unhandled EventEmitter 'error'
4252
+ // from crashing the host process.
4253
+ connection.on('error', (err) => {
4254
+ this.emitTrace({
4255
+ type: 'mcpl:server-error',
4256
+ serverId: connection.id,
4257
+ error: err.message,
4258
+ });
4259
+ });
4260
+ // Cleanup on disconnect. Feature-set state is always dropped (the server
4261
+ // can't be validated while down; reconnect re-registers it). Checkpoint
4262
+ // trees, however, are durable state: CheckpointManager.removeServer deletes
4263
+ // them AND persists the deletion to Chronicle. The close handler must
4264
+ // therefore NEVER destroy checkpoints — `willReconnect` is false on a clean
4265
+ // stop() too (reconnectEnabled is cleared before 'close' fires), so gating
4266
+ // on it would erase every checkpoint tree on an ordinary host restart while
4267
+ // a SIGKILL preserves them. Permanent removal is owned solely by
4268
+ // disconnectMcplServer, which deletes the trees explicitly.
4269
+ connection.on('close', (code, signal) => {
3455
4270
  this.featureSetManager?.removeServer(connection.id);
3456
- this.checkpointManager?.removeServer(connection.id);
4271
+ this.emitTrace({
4272
+ type: 'mcpl:server-closed',
4273
+ serverId: connection.id,
4274
+ code: code ?? null,
4275
+ signal: signal ?? null,
4276
+ willReconnect: connection.willReconnect,
4277
+ });
3457
4278
  this.emitTrace({ type: 'module:removed', moduleName: `mcpl:${connection.id}` });
3458
4279
  });
3459
4280
  }
@@ -3822,7 +4643,7 @@ export class AgentFramework {
3822
4643
  finish({ success: false, error: 'sleep: `seconds` must be a positive number', isError: true });
3823
4644
  return;
3824
4645
  }
3825
- const { until } = gate.setSleep(seconds, input.message);
4646
+ const { until } = gate.setSleep(seconds, input.message, agentName);
3826
4647
  const announce = input.announce !== false; // default true
3827
4648
  const human = seconds >= 3600 ? `${(seconds / 3600).toFixed(1)}h`
3828
4649
  : seconds >= 60 ? `${Math.round(seconds / 60)}m`