@animalabs/agent-framework 0.6.0 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/dist/src/agent.d.ts +4 -0
  2. package/dist/src/agent.d.ts.map +1 -1
  3. package/dist/src/agent.js +15 -0
  4. package/dist/src/agent.js.map +1 -1
  5. package/dist/src/framework.d.ts +235 -2
  6. package/dist/src/framework.d.ts.map +1 -1
  7. package/dist/src/framework.js +1141 -92
  8. package/dist/src/framework.js.map +1 -1
  9. package/dist/src/gate/event-gate.d.ts +20 -1
  10. package/dist/src/gate/event-gate.d.ts.map +1 -1
  11. package/dist/src/gate/event-gate.js +60 -4
  12. package/dist/src/gate/event-gate.js.map +1 -1
  13. package/dist/src/index.d.ts +3 -0
  14. package/dist/src/index.d.ts.map +1 -1
  15. package/dist/src/index.js +4 -0
  16. package/dist/src/index.js.map +1 -1
  17. package/dist/src/mcpl/channel-registry.d.ts +17 -1
  18. package/dist/src/mcpl/channel-registry.d.ts.map +1 -1
  19. package/dist/src/mcpl/channel-registry.js +34 -3
  20. package/dist/src/mcpl/channel-registry.js.map +1 -1
  21. package/dist/src/mcpl/push-handler.d.ts.map +1 -1
  22. package/dist/src/mcpl/push-handler.js +4 -0
  23. package/dist/src/mcpl/push-handler.js.map +1 -1
  24. package/dist/src/mcpl/server-connection.d.ts +29 -0
  25. package/dist/src/mcpl/server-connection.d.ts.map +1 -1
  26. package/dist/src/mcpl/server-connection.js +102 -4
  27. package/dist/src/mcpl/server-connection.js.map +1 -1
  28. package/dist/src/mcpl/types.d.ts +17 -1
  29. package/dist/src/mcpl/types.d.ts.map +1 -1
  30. package/dist/src/mcpl/types.js.map +1 -1
  31. package/dist/src/module-registry.d.ts +15 -1
  32. package/dist/src/module-registry.d.ts.map +1 -1
  33. package/dist/src/module-registry.js +92 -4
  34. package/dist/src/module-registry.js.map +1 -1
  35. package/dist/src/types/agent.d.ts +37 -0
  36. package/dist/src/types/agent.d.ts.map +1 -1
  37. package/dist/src/types/framework.d.ts +40 -0
  38. package/dist/src/types/framework.d.ts.map +1 -1
  39. package/dist/src/types/module.d.ts +46 -0
  40. package/dist/src/types/module.d.ts.map +1 -1
  41. package/dist/src/types/trace.d.ts +51 -0
  42. package/dist/src/types/trace.d.ts.map +1 -1
  43. package/dist/test/agent-cache-ttl.test.d.ts +2 -0
  44. package/dist/test/agent-cache-ttl.test.d.ts.map +1 -0
  45. package/dist/test/agent-cache-ttl.test.js +25 -0
  46. package/dist/test/agent-cache-ttl.test.js.map +1 -0
  47. package/dist/test/conversation-fork-injection-scoping.test.d.ts +2 -0
  48. package/dist/test/conversation-fork-injection-scoping.test.d.ts.map +1 -0
  49. package/dist/test/conversation-fork-injection-scoping.test.js +184 -0
  50. package/dist/test/conversation-fork-injection-scoping.test.js.map +1 -0
  51. package/dist/test/gather-context-timeout.test.d.ts +2 -0
  52. package/dist/test/gather-context-timeout.test.d.ts.map +1 -0
  53. package/dist/test/gather-context-timeout.test.js +120 -0
  54. package/dist/test/gather-context-timeout.test.js.map +1 -0
  55. package/dist/test/inference-exhaustion-rewind.test.d.ts +2 -0
  56. package/dist/test/inference-exhaustion-rewind.test.d.ts.map +1 -0
  57. package/dist/test/inference-exhaustion-rewind.test.js +181 -0
  58. package/dist/test/inference-exhaustion-rewind.test.js.map +1 -0
  59. package/dist/test/inference-failure-observability.test.js +9 -0
  60. package/dist/test/inference-failure-observability.test.js.map +1 -1
  61. package/dist/test/maintenance-timer.test.d.ts +2 -0
  62. package/dist/test/maintenance-timer.test.d.ts.map +1 -0
  63. package/dist/test/maintenance-timer.test.js +114 -0
  64. package/dist/test/maintenance-timer.test.js.map +1 -0
  65. package/dist/test/mcpl-reconnect-events.test.d.ts +2 -0
  66. package/dist/test/mcpl-reconnect-events.test.d.ts.map +1 -0
  67. package/dist/test/mcpl-reconnect-events.test.js +119 -0
  68. package/dist/test/mcpl-reconnect-events.test.js.map +1 -0
  69. package/dist/test/mcpl-reconnect-reregistration.test.d.ts +2 -0
  70. package/dist/test/mcpl-reconnect-reregistration.test.d.ts.map +1 -0
  71. package/dist/test/mcpl-reconnect-reregistration.test.js +181 -0
  72. package/dist/test/mcpl-reconnect-reregistration.test.js.map +1 -0
  73. package/dist/test/mcpl-request-timeout.test.d.ts +2 -0
  74. package/dist/test/mcpl-request-timeout.test.d.ts.map +1 -0
  75. package/dist/test/mcpl-request-timeout.test.js +123 -0
  76. package/dist/test/mcpl-request-timeout.test.js.map +1 -0
  77. package/dist/test/mcpl-runtime-lifecycle.test.d.ts +7 -0
  78. package/dist/test/mcpl-runtime-lifecycle.test.d.ts.map +1 -0
  79. package/dist/test/mcpl-runtime-lifecycle.test.js +142 -0
  80. package/dist/test/mcpl-runtime-lifecycle.test.js.map +1 -0
  81. package/dist/test/ops-alert.test.d.ts +2 -0
  82. package/dist/test/ops-alert.test.d.ts.map +1 -0
  83. package/dist/test/ops-alert.test.js +152 -0
  84. package/dist/test/ops-alert.test.js.map +1 -0
  85. package/dist/test/refusal-rewind.test.d.ts +2 -0
  86. package/dist/test/refusal-rewind.test.d.ts.map +1 -0
  87. package/dist/test/refusal-rewind.test.js +108 -0
  88. package/dist/test/refusal-rewind.test.js.map +1 -0
  89. package/dist/test/state-scaling-e2e.test.d.ts +22 -0
  90. package/dist/test/state-scaling-e2e.test.d.ts.map +1 -0
  91. package/dist/test/state-scaling-e2e.test.js +200 -0
  92. package/dist/test/state-scaling-e2e.test.js.map +1 -0
  93. package/dist/test/state-scaling.test.d.ts +12 -0
  94. package/dist/test/state-scaling.test.d.ts.map +1 -0
  95. package/dist/test/state-scaling.test.js +243 -0
  96. package/dist/test/state-scaling.test.js.map +1 -0
  97. package/dist/test/tool-result-content.test.d.ts +2 -0
  98. package/dist/test/tool-result-content.test.d.ts.map +1 -0
  99. package/dist/test/tool-result-content.test.js +37 -0
  100. package/dist/test/tool-result-content.test.js.map +1 -0
  101. package/dist/tsconfig.tsbuildinfo +1 -1
  102. package/package.json +3 -2
  103. package/src/agent.ts +16 -0
  104. package/src/framework.ts +1339 -99
  105. package/src/gate/event-gate.ts +56 -4
  106. package/src/index.ts +10 -0
  107. package/src/mcpl/channel-registry.ts +34 -3
  108. package/src/mcpl/push-handler.ts +4 -0
  109. package/src/mcpl/server-connection.ts +110 -4
  110. package/src/mcpl/types.ts +19 -1
  111. package/src/module-registry.ts +113 -4
  112. package/src/types/agent.ts +38 -0
  113. package/src/types/framework.ts +44 -0
  114. package/src/types/module.ts +52 -0
  115. package/src/types/trace.ts +67 -0
package/src/framework.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { join } from 'node:path';
2
+ import { appendFileSync, mkdirSync } from 'node:fs';
2
3
  import { JsStore } from '@animalabs/chronicle';
3
4
  import type { Membrane, ContentBlock, NormalizedRequest, YieldingStream, ToolResult as MembraneToolResult, ToolResultContentBlock } from '@animalabs/membrane';
4
5
  import { MembraneError } from '@animalabs/membrane';
@@ -39,10 +40,13 @@ import type {
39
40
  AgentState,
40
41
  Module,
41
42
  SpeechContext,
43
+ ContextMaintenanceRun,
44
+ ContextMaintenanceAgentRun,
45
+ ContextMaintenanceSnapshot,
42
46
  } from './types/index.js';
43
47
  import { ProcessQueueImpl } from './queue.js';
44
48
  import { Agent } from './agent.js';
45
- import { ModuleRegistry } from './module-registry.js';
49
+ import { ModuleRegistry, isStateExistsError } from './module-registry.js';
46
50
  import { McplServerRegistry } from './mcpl/server-registry.js';
47
51
  import { FeatureSetManager } from './mcpl/feature-set-manager.js';
48
52
  import { ScopeManager } from './mcpl/scope-manager.js';
@@ -80,7 +84,8 @@ const FRAMEWORK_STATE_ID = 'framework/state';
80
84
  const CONVERSATION_ROUTER_STATE_ID = 'framework/conversation-router';
81
85
  const INFERENCE_LOG_ID = 'framework/inference-log';
82
86
  const PROCESS_LOG_ID = 'framework/process-log';
83
- const TURN_CHECKPOINTS_ID = 'framework/turn-checkpoints';
87
+ const TURN_CHECKPOINTS_ID = 'framework/turn-checkpoints'; // legacy single-map layout, read-only fallback
88
+ const TURN_CHECKPOINTS_TREE_ID = 'framework/turn-checkpoints/tree';
84
89
 
85
90
  /** Maximum number of turn checkpoints to keep per agent. */
86
91
  const MAX_TURN_CHECKPOINTS = 20;
@@ -142,6 +147,10 @@ class DefaultErrorPolicy implements ErrorPolicy {
142
147
 
143
148
  /** Default sync interval in milliseconds */
144
149
  const DEFAULT_SYNC_INTERVAL_MS = 1000;
150
+ /** Poll pending context-strategy maintenance independently of user activity. */
151
+ const DEFAULT_MAINTENANCE_INTERVAL_MS = 5000;
152
+ /** Bound one pass so a large backlog yields to inference and other agents. */
153
+ const MAINTENANCE_TICKS_PER_PASS = 8;
145
154
 
146
155
  /**
147
156
  * Extract fields the EventGate cares about from a ProcessEvent. The set of
@@ -198,10 +207,34 @@ interface HostCommandParams {
198
207
  fromMessageId?: string;
199
208
  /** For the `hide` command: Discord message id ending an inclusive range. */
200
209
  toMessageId?: string;
210
+ /** For the `unstick` command: max rewind/retry attempts (default = the
211
+ * agent's refusalHandling.maxRewinds, else 3). */
212
+ maxRewinds?: number;
213
+ /** For the `unstick` command: raw channel id to post the outcome report to. */
214
+ channelId?: string;
201
215
  requesterId?: string;
202
216
  requesterName?: string;
203
217
  }
204
218
 
219
+ /** Descriptor of a single refusal-driven rewind (see rewindTriggeringTurn). */
220
+ interface RewindRecord {
221
+ /** tool = machine tool exchange; human = an ingested message; other = else. */
222
+ kind: 'tool' | 'human' | 'other';
223
+ /** Content-free, safe-to-replay description of what was withheld. */
224
+ descriptor: string;
225
+ /** All message ids removed in this shed (a tool exchange removes 2+). */
226
+ removedIds: MessageId[];
227
+ /** Discord (channelId, messageId) of the removed message, if it had one. */
228
+ discordRef?: { channelId: string; messageId: string };
229
+ }
230
+
231
+ /** Cap an API error message for inline use in a rewind marker: keep enough to
232
+ * identify the failure class without pasting a wall of provider JSON into the
233
+ * agent's context. */
234
+ function truncateReason(reason: string, max = 160): string {
235
+ return reason.length <= max ? reason : reason.slice(0, max) + '…';
236
+ }
237
+
205
238
  export class AgentFramework {
206
239
  private store: JsStore;
207
240
  private ownsStore: boolean;
@@ -227,6 +260,16 @@ export class AgentFramework {
227
260
  private traceListeners: TraceEventListener[] = [];
228
261
  private syncIntervalMs: number;
229
262
  private syncTimer: ReturnType<typeof setInterval> | null = null;
263
+ private maintenanceIntervalMs: number;
264
+ private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
265
+ private maintenancePass: Promise<void> | null = null;
266
+ private maintenanceRunId = 0;
267
+ private currentMaintenanceRun: ContextMaintenanceRun | null = null;
268
+ private maintenanceHistory: ContextMaintenanceRun[] = [];
269
+ /** Last time we console-warned about stale (busy-requeued) inference requests, per agent. */
270
+ private staleWarnAt = new Map<string, number>();
271
+ /** Per-agent last inference activity (epoch ms), for /healthz + doctor tooling. */
272
+ private lastInferenceAt = new Map<string, { startedAt?: number; endedAt?: number; failedAt?: number; lastError?: string }>();
230
273
  private processLoggingPersist: boolean;
231
274
  private processLoggingBroadcast: boolean;
232
275
  private activeStreams: Map<string, Promise<void>> = new Map();
@@ -237,6 +280,41 @@ export class AgentFramework {
237
280
  /** N consecutive failed inferences ⇒ the agent is treated as hard-down and
238
281
  * escalated loudly to stderr. */
239
282
  private readonly inferenceFailureEscalationThreshold = 3;
283
+ /** Per-(agent,kind) timestamp of the last ops webhook post. Throttles
284
+ * opsAlert() so a persistent failure re-posts once per cooldown window
285
+ * instead of on every occurrence. */
286
+ private opsAlertLastSent: Map<string, number> = new Map();
287
+ /** Cooldown between webhook posts for the same (agent, kind). */
288
+ private readonly opsAlertCooldownMs = 15 * 60_000;
289
+ /** Per-agent refusal bookkeeping for observability — exposed via
290
+ * healthSnapshot() and mirrored to failures.log / ops alerts. */
291
+ private refusalStats: Map<string, { total: number; byCategory: Record<string, number>; lastAt: number; lastCategory: string }> = new Map();
292
+ /** Per-agent count of consecutive refusals (reset on any non-refusal
293
+ * completion). Distinct from refusalRewinds, which budgets the auto-rewind
294
+ * loop — this one drives ops alerting. */
295
+ private refusalStreak: Map<string, number> = new Map();
296
+ /** Per-agent count of consecutive refusal-driven rewinds in the current turn
297
+ * chain (reset when a turn completes without a refusal). Bounds the auto
298
+ * rewind loop — see refusalHandling + rewindTriggeringTurn. */
299
+ private refusalRewinds: Map<string, number> = new Map();
300
+ /** Per-agent count of poison-history rewinds performed by the hard-down
301
+ * breaker (noteInferenceExhausted). Reset on any successful inference.
302
+ * Bounds the automatic quarantine loop the same way refusalRewinds bounds
303
+ * the refusal loop, so the breaker can never shed the whole history. */
304
+ private exhaustionRewinds: Map<string, number> = new Map();
305
+ /** Per-agent current rewind episode: the single consolidated marker's id and
306
+ * how many turns have been shed so far. One marker per episode, updated in
307
+ * place (see updateRewindMarker); cleared when the episode ends. */
308
+ private rewindEpisode: Map<string, { markerId: MessageId; count: number; category: string }> = new Map();
309
+ /** Active `/unstick` sessions: an admin-forced rewind-until-clean loop that
310
+ * runs even when the agent's autoRewind toggle is off. Tracks the remaining
311
+ * budget, what was shed (for the report), and where to post the outcome. */
312
+ private forcedRewind: Map<string, {
313
+ remaining: number;
314
+ removed: RewindRecord[];
315
+ serverId: string;
316
+ channelId: string;
317
+ }> = new Map();
240
318
  /** Name of the primary (non-ephemeral) agent for routing framework messages. */
241
319
  private primaryAgentName: string | null = null;
242
320
 
@@ -283,6 +361,13 @@ export class AgentFramework {
283
361
  private mcplPrefixMap: Map<string, string> = new Map();
284
362
  /** Maps serverId → McplServerConfig for prefix lookup. */
285
363
  private mcplServerConfigs: Map<string, import('./mcpl/types.js').McplServerConfig> = new Map();
364
+ /** Host capabilities advertised during the MCP handshake — stored so servers
365
+ * can be connected at runtime (connectMcplServer) after initialization. */
366
+ private mcplHostCapabilities: McplHostCapabilities | null = null;
367
+ /** Inference routing policy from FrameworkConfig — stored so the MCPL
368
+ * subsystem can be lazily initialized by connectMcplServer when the
369
+ * framework started with zero configured servers. */
370
+ private mcplInferenceRoutingConfig: import('./mcpl/types.js').InferenceRoutingPolicy | null = null;
286
371
 
287
372
  // EventGate (null when FrameworkConfig.gate is omitted)
288
373
  private eventGate: EventGate | null = null;
@@ -297,6 +382,7 @@ export class AgentFramework {
297
382
  inferencePolicy: InferencePolicy,
298
383
  errorPolicy: ErrorPolicy,
299
384
  syncIntervalMs: number,
385
+ maintenanceIntervalMs: number,
300
386
  processLoggingPersist: boolean,
301
387
  processLoggingBroadcast: boolean
302
388
  ) {
@@ -306,6 +392,7 @@ export class AgentFramework {
306
392
  this.inferencePolicy = inferencePolicy;
307
393
  this.errorPolicy = errorPolicy;
308
394
  this.syncIntervalMs = syncIntervalMs;
395
+ this.maintenanceIntervalMs = maintenanceIntervalMs;
309
396
  this.processLoggingPersist = processLoggingPersist;
310
397
  this.processLoggingBroadcast = processLoggingBroadcast;
311
398
  this.queue = new ProcessQueueImpl();
@@ -363,10 +450,12 @@ export class AgentFramework {
363
450
  // Already registered
364
451
  }
365
452
 
453
+ // The legacy single-map checkpoint state (TURN_CHECKPOINTS_ID) is no longer
454
+ // registered for new stores — it's read-only fallback data in old ones.
366
455
  try {
367
- store.registerState({ id: TURN_CHECKPOINTS_ID, strategy: 'snapshot' });
368
- } catch {
369
- // Already registered
456
+ store.registerState({ id: TURN_CHECKPOINTS_TREE_ID, strategy: 'tree' });
457
+ } catch (error) {
458
+ if (!isStateExistsError(error)) throw error;
370
459
  }
371
460
 
372
461
  // Process logging config (default: disabled)
@@ -394,6 +483,7 @@ export class AgentFramework {
394
483
  config.inferencePolicy ?? new DefaultInferencePolicy(),
395
484
  config.errorPolicy ?? new DefaultErrorPolicy(),
396
485
  config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS,
486
+ config.maintenanceIntervalMs ?? DEFAULT_MAINTENANCE_INTERVAL_MS,
397
487
  processLoggingPersist,
398
488
  processLoggingBroadcast
399
489
  );
@@ -472,6 +562,10 @@ export class AgentFramework {
472
562
  framework.livenessWatchdog.start();
473
563
  }
474
564
 
565
+ // Stored for lazy MCPL initialization (connectMcplServer on a framework
566
+ // that started with zero configured servers).
567
+ framework.mcplInferenceRoutingConfig = config.inferenceRouting ?? null;
568
+
475
569
  // Initialize MCPL subsystems if configured
476
570
  if (config.mcplServers && config.mcplServers.length > 0) {
477
571
  // Validate tool prefixes: no collisions with module names or between servers
@@ -496,6 +590,31 @@ export class AgentFramework {
496
590
  await framework.initializeMcpl(config.mcplServers, config.inferenceRouting);
497
591
  }
498
592
 
593
+ // Diagnostics: `kill -USR2 <pid>` dumps live wake/inference state to stderr
594
+ // (journal) without a restart — for catching the wake-wedge on the running
595
+ // process. Shows the gate's `inferring` set + buffered-event count (the
596
+ // wedge signature), active streams, and pending inference requests.
597
+ try {
598
+ process.on('SIGUSR2', () => {
599
+ try {
600
+ const gate = framework.eventGate?.inferenceDiagnostics() ?? null;
601
+ console.error(
602
+ '[diagnostics] ' + JSON.stringify({
603
+ at: new Date().toISOString(),
604
+ gate,
605
+ activeStreams: [...framework.activeStreams.keys()],
606
+ pendingRequests: framework.pendingRequests.length,
607
+ agents: [...framework.agents.keys()],
608
+ }),
609
+ );
610
+ } catch (err) {
611
+ console.error('[diagnostics] dump failed:', err instanceof Error ? err.message : err);
612
+ }
613
+ });
614
+ } catch {
615
+ // SIGUSR2 not available on this platform — non-fatal.
616
+ }
617
+
499
618
  return framework;
500
619
  }
501
620
 
@@ -520,6 +639,16 @@ export class AgentFramework {
520
639
  }
521
640
  }, this.syncIntervalMs);
522
641
  }
642
+
643
+ if (this.maintenanceIntervalMs > 0) {
644
+ this.maintenanceTimer = setInterval(() => {
645
+ this.startQueuedMaintenance();
646
+ }, this.maintenanceIntervalMs);
647
+ this.maintenanceTimer.unref?.();
648
+ // Do not make a restored queue wait a full interval before its first
649
+ // attempt. MCPL/module initialization has completed before start().
650
+ this.startQueuedMaintenance();
651
+ }
523
652
  }
524
653
 
525
654
  /**
@@ -548,6 +677,14 @@ export class AgentFramework {
548
677
  this.syncTimer = null;
549
678
  }
550
679
 
680
+ if (this.maintenanceTimer) {
681
+ clearInterval(this.maintenanceTimer);
682
+ this.maintenanceTimer = null;
683
+ }
684
+ if (this.maintenancePass) {
685
+ await this.maintenancePass;
686
+ }
687
+
551
688
  if (this.loopPromise) {
552
689
  await this.loopPromise;
553
690
  }
@@ -578,6 +715,103 @@ export class AgentFramework {
578
715
  }
579
716
  }
580
717
 
718
+ /**
719
+ * Advance queued ContextManager work without requiring a new message.
720
+ *
721
+ * Tool definitions are refreshed first because compression replays stored
722
+ * tool cycles and providers reject those requests when the corresponding
723
+ * schemas are absent. Passes never overlap; each agent gets a bounded drain
724
+ * so maintenance cannot monopolize the framework event loop.
725
+ */
726
+ private startQueuedMaintenance(): void {
727
+ if (this.maintenancePass || !this.running) return;
728
+ const pass = this.runQueuedMaintenance().catch((error) => {
729
+ console.error(
730
+ '[context-maintenance] pass failed:',
731
+ error instanceof Error ? error.message : error,
732
+ );
733
+ });
734
+ this.maintenancePass = pass;
735
+ void pass.finally(() => {
736
+ if (this.maintenancePass === pass) this.maintenancePass = null;
737
+ });
738
+ }
739
+
740
+ private async runQueuedMaintenance(): Promise<void> {
741
+ const allTools = this.getAllTools();
742
+ const queued = [...this.agents.values()].flatMap((agent) => {
743
+ const cm = agent.getContextManager();
744
+ const tools = allTools.filter((tool) => agent.canUseTool(tool.name));
745
+ cm.setToolDefinitions(tools);
746
+ if (cm.isReady()) return [];
747
+ const pending = cm.getPendingWork()?.description;
748
+ const progress = this.contextProgress(cm);
749
+ const record: ContextMaintenanceAgentRun = {
750
+ agentName: agent.name,
751
+ startedAt: Date.now(),
752
+ ticks: 0,
753
+ readyBefore: false,
754
+ ...(pending ? { pendingBefore: pending } : {}),
755
+ ...(progress ? { progressBefore: progress } : {}),
756
+ };
757
+ return [{
758
+ agent,
759
+ cm,
760
+ record,
761
+ }];
762
+ });
763
+ if (queued.length === 0) return;
764
+
765
+ const run: ContextMaintenanceRun = {
766
+ id: ++this.maintenanceRunId,
767
+ startedAt: Date.now(),
768
+ agents: queued.map(item => item.record),
769
+ };
770
+ this.currentMaintenanceRun = run;
771
+
772
+ try {
773
+ await Promise.all(queued.map(async ({ agent, cm, record }) => {
774
+ try {
775
+ for (
776
+ let i = 0;
777
+ i < MAINTENANCE_TICKS_PER_PASS && this.running && !cm.isReady();
778
+ i++
779
+ ) {
780
+ await cm.tick();
781
+ record.ticks++;
782
+ }
783
+ } catch (error) {
784
+ record.error = error instanceof Error ? error.message : String(error);
785
+ console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
786
+ } finally {
787
+ const pending = cm.getPendingWork()?.description;
788
+ record.finishedAt = Date.now();
789
+ record.readyAfter = cm.isReady();
790
+ if (pending) record.pendingAfter = pending;
791
+ const progress = this.contextProgress(cm);
792
+ if (progress) record.progressAfter = progress;
793
+ }
794
+ }));
795
+ } finally {
796
+ run.finishedAt = Date.now();
797
+ run.agents.sort((a, b) => a.agentName.localeCompare(b.agentName));
798
+ this.currentMaintenanceRun = null;
799
+ this.maintenanceHistory.unshift(run);
800
+ if (this.maintenanceHistory.length > 50) this.maintenanceHistory.length = 50;
801
+ }
802
+ }
803
+
804
+ private contextProgress(cm: ContextManager): Record<string, unknown> | undefined {
805
+ const strategy = cm.getStrategy() as {
806
+ getProgressSnapshot?: () => unknown;
807
+ };
808
+ if (typeof strategy.getProgressSnapshot !== 'function') return undefined;
809
+ const progress = strategy.getProgressSnapshot();
810
+ return progress && typeof progress === 'object'
811
+ ? progress as Record<string, unknown>
812
+ : undefined;
813
+ }
814
+
581
815
  /**
582
816
  * Push a process event to the queue.
583
817
  */
@@ -700,6 +934,28 @@ export class AgentFramework {
700
934
  return [...moduleTools, ...this.mcplTools, ...channelTools, ...gateTools];
701
935
  }
702
936
 
937
+ /** Counts-only context-maintenance diagnostics for authenticated debug UIs. */
938
+ getContextMaintenanceSnapshot(): ContextMaintenanceSnapshot {
939
+ const agents = [...this.agents.values()].map((agent) => {
940
+ const cm = agent.getContextManager();
941
+ const pending = cm.getPendingWork()?.description;
942
+ const progress = this.contextProgress(cm);
943
+ return {
944
+ agentName: agent.name,
945
+ ready: cm.isReady(),
946
+ ...(pending ? { pending } : {}),
947
+ ...(progress ? { progress } : {}),
948
+ };
949
+ });
950
+ return structuredClone({
951
+ intervalMs: this.maintenanceIntervalMs,
952
+ ticksPerPass: MAINTENANCE_TICKS_PER_PASS,
953
+ current: this.currentMaintenanceRun,
954
+ history: this.maintenanceHistory,
955
+ agents,
956
+ });
957
+ }
958
+
703
959
  /**
704
960
  * Build the membrane-normalized request that WOULD be emitted if `agentName`
705
961
  * were activated right now — WITHOUT running inference, opening a stream, or
@@ -749,8 +1005,9 @@ export class AgentFramework {
749
1005
  // Module gatherContext (fail-open, matches startAgentStream)
750
1006
  try {
751
1007
  const moduleInjections = await this.moduleRegistry.gatherContext(agentName);
752
- if (moduleInjections.length > 0) {
753
- injections = moduleInjections;
1008
+ const scopedModuleInjections = this.scopeInjectionsForAgent(agentName, moduleInjections);
1009
+ if (scopedModuleInjections.length > 0) {
1010
+ injections = scopedModuleInjections;
754
1011
  }
755
1012
  } catch (error) {
756
1013
  console.error('Module gatherContext error (preview):', error);
@@ -761,7 +1018,10 @@ export class AgentFramework {
761
1018
  if (this.hookOrchestrator) {
762
1019
  try {
763
1020
  const hookParams = this.buildBeforeInferenceParams(agent);
764
- const hookInjections = await this.hookOrchestrator.beforeInference(hookParams);
1021
+ const hookInjections = this.scopeInjectionsForAgent(
1022
+ agentName,
1023
+ await this.hookOrchestrator.beforeInference(hookParams),
1024
+ );
765
1025
  if (hookInjections.length > 0) {
766
1026
  injections = injections ? [...injections, ...hookInjections] : hookInjections;
767
1027
  }
@@ -951,6 +1211,7 @@ export class AgentFramework {
951
1211
  clearInterval(completionWatchdog);
952
1212
  this.offTrace(traceListener);
953
1213
  this.agents.delete(agent.name);
1214
+ this.evictTurnCheckpoints(agent.name);
954
1215
  };
955
1216
 
956
1217
  // Watchdog: if inference never starts within 30s, the agent is a zombie.
@@ -1309,6 +1570,194 @@ export class AgentFramework {
1309
1570
  }
1310
1571
  }
1311
1572
 
1573
+ /**
1574
+ * Rewind the single turn that fed a refused generation: redact the newest
1575
+ * message the agent did NOT author (a tool result, or an incoming message)
1576
+ * and inject a metadata-only marker in its place. The marker carries a
1577
+ * *description* of what was withheld (kind + size), never the content — so it
1578
+ * cannot itself re-trip the classifier — while the raw record survives in the
1579
+ * chronicle log for forensics. Returns a record describing the removal, or
1580
+ * null if there is nothing eligible to rewind (e.g. only the agent's own
1581
+ * turns, or a prior system marker, remain).
1582
+ *
1583
+ * Shared by the auto path (on `stop_reason: refusal`) and the `/rewind`
1584
+ * host command.
1585
+ */
1586
+ private shedNewestTurn(agent: Agent): RewindRecord | null {
1587
+ const cm = agent.getContextManager();
1588
+ const all = cm.getAllMessages();
1589
+ const typeOf = (b: unknown) => (b as { type?: string }).type;
1590
+ const hasBlock = (m: { content?: unknown } | undefined, t: string) =>
1591
+ Array.isArray(m?.content) && (m!.content as unknown[]).some((b) => typeOf(b) === t);
1592
+
1593
+ // Newest message that is not our own episode marker. We shed strictly
1594
+ // newest-first, in sequence — INCLUDING the agent's own turns (poison lives
1595
+ // in tool_use/narration turns too), because newest-first reaches whatever is
1596
+ // poisoning the context if allowed to run deep enough. The one invariant:
1597
+ // shed COMPLETE exchanges. Removing a `tool_result` also removes its paired
1598
+ // `tool_use` assistant turn, so we never leave an orphaned tool_use / signed
1599
+ // `thinking` block — which the API rejects with a 400 ("thinking blocks in
1600
+ // the latest assistant message cannot be modified"). We do NOT add a marker
1601
+ // here; the caller keeps one consolidated marker (updateRewindMarker).
1602
+ let idx = -1;
1603
+ for (let i = all.length - 1; i >= 0; i--) {
1604
+ const meta = (all[i].metadata ?? {}) as { system?: unknown };
1605
+ if (meta.system) continue;
1606
+ idx = i; break;
1607
+ }
1608
+ if (idx < 0) return null;
1609
+ const msg = all[idx];
1610
+ const md = (msg.metadata ?? {}) as {
1611
+ messageId?: unknown; channelId?: unknown; system?: unknown;
1612
+ };
1613
+
1614
+ const content = Array.isArray(msg.content) ? msg.content : [];
1615
+ const toolBlocks = content.filter((b) => typeOf(b) === 'tool_result');
1616
+ const images = content.filter((b) => typeOf(b) === 'image').length;
1617
+ const textLen = content
1618
+ .filter((b) => typeOf(b) === 'text')
1619
+ .reduce((n, b) => n + String((b as { text?: string }).text ?? '').length, 0);
1620
+
1621
+ // Determine the complete exchange to remove.
1622
+ const removedIds: MessageId[] = [msg.id];
1623
+ let kind: RewindRecord['kind'];
1624
+ let descriptor: string;
1625
+ if (toolBlocks.length > 0) {
1626
+ kind = 'tool';
1627
+ // Pair the tool_result with the tool_use assistant turn right before it.
1628
+ if (idx - 1 >= 0 && hasBlock(all[idx - 1], 'tool_use')) {
1629
+ removedIds.push(all[idx - 1].id);
1630
+ }
1631
+ const sz = toolBlocks.reduce(
1632
+ (n, b) => n + String((b as { content?: unknown }).content ?? '').length, 0);
1633
+ descriptor = `a tool exchange (~${Math.max(1, Math.round(sz / 1024))}KB` +
1634
+ `${images ? `, ${images} image(s)` : ''})`;
1635
+ } else if (md.messageId) {
1636
+ kind = 'human';
1637
+ descriptor = `an incoming message from ${msg.participant} ` +
1638
+ `(${textLen} chars${images ? `, ${images} image(s)` : ''})`;
1639
+ } else {
1640
+ kind = 'other';
1641
+ descriptor = `a ${msg.participant} turn ` +
1642
+ `(${textLen} chars${images ? `, ${images} image(s)` : ''})`;
1643
+ }
1644
+ const discordRef = md.messageId && md.channelId
1645
+ ? { channelId: String(md.channelId), messageId: String(md.messageId) }
1646
+ : undefined;
1647
+
1648
+ for (const id of removedIds) cm.removeMessage(id);
1649
+ return { kind, descriptor, removedIds, discordRef };
1650
+ }
1651
+
1652
+ /**
1653
+ * Maintain exactly ONE consolidated marker for the current rewind episode,
1654
+ * updated in place as more turns are shed. Six rewinds ⇒ one message that
1655
+ * says "the 6 most recent turns were set aside", not six separate notes — so
1656
+ * the context converges (shed N, add 1) instead of growing, and the marker
1657
+ * sits at the tail giving the model something actionable to answer once the
1658
+ * refusal clears. Returns the running count.
1659
+ */
1660
+ private updateRewindMarker(
1661
+ agent: Agent,
1662
+ category: string,
1663
+ cause: 'refusal' | 'inference-failure' = 'refusal',
1664
+ ): number {
1665
+ const cm = agent.getContextManager();
1666
+ const ep = this.rewindEpisode.get(agent.name);
1667
+ const count = (ep?.count ?? 0) + 1;
1668
+ const why = cause === 'refusal'
1669
+ ? `the model refused on them (content filter: ${category}). Their content is ` +
1670
+ `not reproduced here (so this note can't re-trigger the filter); the ` +
1671
+ `originals remain in the raw record`
1672
+ : `the model API kept rejecting the conversation on them (${category}). ` +
1673
+ `Their content is not reproduced here (so this note can't re-trigger the ` +
1674
+ `rejection); the originals remain in the raw record`;
1675
+ const text =
1676
+ `[refusal-rewind] The ${count} most recent turn(s) were set aside because ` +
1677
+ `${why}. You are clear to continue — if you ` +
1678
+ `were mid-task, take a different approach; otherwise carry on with whatever ` +
1679
+ `is now in front of you, or briefly acknowledge the gap and ask what's next.`;
1680
+ const blocks: ContentBlock[] = [{ type: 'text', text }];
1681
+ if (ep) {
1682
+ cm.editMessage(ep.markerId, blocks);
1683
+ this.rewindEpisode.set(agent.name, { markerId: ep.markerId, count, category });
1684
+ } else {
1685
+ const id = cm.addMessage('user', blocks, {
1686
+ system: true, kind: 'refusal-rewind', category, count, cause,
1687
+ });
1688
+ this.rewindEpisode.set(agent.name, { markerId: id, count, category });
1689
+ }
1690
+ return count;
1691
+ }
1692
+
1693
+ /**
1694
+ * Announce a refusal-rewind on the conversational surface (Discord), used
1695
+ * when the withheld turn was a *human* message so it isn't dropped silently.
1696
+ * Best-effort; mirrors reactToRefusal's locus resolution.
1697
+ */
1698
+ private async announceRewind(
1699
+ agentName: string,
1700
+ rec: RewindRecord,
1701
+ category: string,
1702
+ ): Promise<void> {
1703
+ try {
1704
+ const incoming = this.channelRegistry?.buildChannelContext()?.incoming;
1705
+ if (!incoming) return;
1706
+ const parts = incoming.channelId.split(':');
1707
+ if (parts[0] !== 'discord') return;
1708
+ const channelId = parts[parts.length - 1];
1709
+ const serverId = this.channelRegistry?.getChannelServerId(incoming.channelId);
1710
+ const server = serverId ? this.mcplServerRegistry?.getServer(serverId) : null;
1711
+ if (!server) return;
1712
+ await server.sendToolsCall('send_message', {
1713
+ channelId,
1714
+ content:
1715
+ `⚠️ I had to set aside ${rec.descriptor} — it tripped a content ` +
1716
+ `filter (${category}), so it's withheld from my context and I'm ` +
1717
+ `continuing without it. If it was important, please rephrase or re-send.`,
1718
+ });
1719
+ } catch (err) {
1720
+ console.error(
1721
+ '[refusal-rewind] announce failed:',
1722
+ err instanceof Error ? err.message : err,
1723
+ );
1724
+ }
1725
+ }
1726
+
1727
+ /**
1728
+ * Conclude an active `/unstick` session: post the outcome (what was shed and
1729
+ * whether the model stopped refusing) to the channel the command came from,
1730
+ * then clear the session. Idempotent — a no-op if there's no session.
1731
+ */
1732
+ private finishUnstick(agentName: string, success: boolean, category?: string): void {
1733
+ const s = this.forcedRewind.get(agentName);
1734
+ if (!s) return;
1735
+ this.forcedRewind.delete(agentName);
1736
+ const n = s.removed.length;
1737
+ const list = n
1738
+ ? '\n' + s.removed.map((r) => `• ${r.descriptor}`).join('\n')
1739
+ : '';
1740
+ const content = success
1741
+ ? `🔧 Unstuck **${agentName}** — shed ${n} turn(s); the model responded.${list}`
1742
+ : `⚠️ Couldn't unstick **${agentName}** — still refusing after ${n} rewind(s)` +
1743
+ `${category ? ` (category=${category})` : ''}.${list}`;
1744
+ console.error(
1745
+ `[unstick] agent=${agentName} ${success ? 'succeeded' : 'gave up'} after ${n} rewind(s)`,
1746
+ );
1747
+ if (s.channelId) void this.postToChannel(s.serverId, s.channelId, content);
1748
+ }
1749
+
1750
+ /** Best-effort send_message to a raw channel via a named MCPL server. */
1751
+ private async postToChannel(serverId: string, channelId: string, content: string): Promise<void> {
1752
+ try {
1753
+ const server = this.mcplServerRegistry?.getServer(serverId);
1754
+ if (!server) return;
1755
+ await server.sendToolsCall('send_message', { channelId, content });
1756
+ } catch (err) {
1757
+ console.error('[unstick] report post failed:', err instanceof Error ? err.message : err);
1758
+ }
1759
+ }
1760
+
1312
1761
  /**
1313
1762
  * Handle a `host/command` request from an MCPL surface server (e.g. a
1314
1763
  * Discord slash command). Currently supports:
@@ -1332,8 +1781,12 @@ export class AgentFramework {
1332
1781
  * that carried one — so the surface can mark them with a reaction. */
1333
1782
  hiddenRefs?: Array<{ channelId: string; messageId: string }>;
1334
1783
  lastVisible?: { participant?: string; role?: string; preview?: string } | null;
1784
+ /** For `unstick`: acknowledges the forced-rewind loop has started; the
1785
+ * outcome report is posted to the channel asynchronously. */
1786
+ started?: boolean;
1787
+ cap?: number;
1335
1788
  }> {
1336
- if (params.command !== 'undo' && params.command !== 'hide') {
1789
+ if (params.command !== 'undo' && params.command !== 'hide' && params.command !== 'unstick') {
1337
1790
  return { ok: false, error: `Unknown host command: ${String(params.command)}` };
1338
1791
  }
1339
1792
 
@@ -1342,6 +1795,38 @@ export class AgentFramework {
1342
1795
  return { ok: false, error: `Unknown agent: ${String(agentName)}` };
1343
1796
  }
1344
1797
 
1798
+ // unstick: force the refusal-rewind loop on demand (even if the agent's
1799
+ // autoRewind toggle is off). Redacts the turn that fed the refusal and
1800
+ // re-runs, up to `cap` times, until the model stops refusing. Kicks the
1801
+ // loop and returns immediately; the outcome (what was shed + whether it
1802
+ // cleared) is posted to the channel when the chain resolves.
1803
+ if (params.command === 'unstick') {
1804
+ const agent = this.agents.get(agentName)!;
1805
+ if (agent.state.status !== 'idle') {
1806
+ return { ok: false, error: `Cannot unstick while agent is ${agent.state.status}` };
1807
+ }
1808
+ const cap = Math.max(1, Math.min(10,
1809
+ Math.floor(params.maxRewinds ?? agent.refusalHandling?.maxRewinds ?? 3)));
1810
+ this.forcedRewind.set(agentName, {
1811
+ remaining: cap,
1812
+ removed: [],
1813
+ serverId,
1814
+ channelId: params.channelId ?? '',
1815
+ });
1816
+ this.refusalRewinds.set(agentName, 0);
1817
+ this.pendingRequests.push({
1818
+ agentName,
1819
+ reason: 'unstick',
1820
+ source: 'framework',
1821
+ timestamp: Date.now(),
1822
+ });
1823
+ console.error(
1824
+ `[unstick] agent=${agentName} started cap=${cap} ` +
1825
+ `by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`,
1826
+ );
1827
+ return { ok: true, started: true, cap };
1828
+ }
1829
+
1345
1830
  // hide: redact a single message (or an inclusive range) from the active
1346
1831
  // branch, addressed by Discord message id. Unlike undo this is a
1347
1832
  // removal-in-place (chronicle redact), not a branch rewind.
@@ -1959,18 +2444,91 @@ export class AgentFramework {
1959
2444
  this.saveTurnCheckpoints(agentName, checkpoints);
1960
2445
  }
1961
2446
 
2447
+ /**
2448
+ * Checkpoints live in one Tree state (TURN_CHECKPOINTS_TREE_ID) keyed by agent
2449
+ * name, each key pointing at a JSON blob of that agent's ≤MAX_TURN_CHECKPOINTS
2450
+ * list. This kills two unbounded-in-agents-ever dimensions at once:
2451
+ *
2452
+ * - the original single Record<agentName, list> map was rewritten whole on
2453
+ * every turn of every agent and never evicted a departed subagent's key;
2454
+ * - the intermediate design (one state slot per agent) fixed the writes but
2455
+ * leaked a permanent chronicle state *registration* per spawn — chronicle
2456
+ * has no deregistration, and the state index is rewritten and fsynced on
2457
+ * every sync tick, so per-tick cost grew with fleet history anyway.
2458
+ *
2459
+ * A tree gives per-key O(entry) writes, real key removal (treeRemove), and
2460
+ * exactly one registration for the life of the store. The legacy map state
2461
+ * (TURN_CHECKPOINTS_ID) is kept as a read-only fallback for stores written
2462
+ * before the split; eviction tombstones legacy keys so a reused agent name
2463
+ * can't inherit a dead agent's checkpoints through the fallback.
2464
+ */
2465
+ private legacyCheckpointKeys: Set<string> | null = null;
2466
+
2467
+ private hasLegacyCheckpoints(agentName: string): boolean {
2468
+ if (!this.legacyCheckpointKeys) {
2469
+ const legacy = this.store.getStateJson(TURN_CHECKPOINTS_ID);
2470
+ this.legacyCheckpointKeys = new Set(
2471
+ legacy && typeof legacy === 'object' ? Object.keys(legacy) : []
2472
+ );
2473
+ }
2474
+ return this.legacyCheckpointKeys.has(agentName);
2475
+ }
2476
+
1962
2477
  private getTurnCheckpoints(agentName: string): TurnCheckpoint[] {
1963
- const data = this.store.getStateJson(TURN_CHECKPOINTS_ID);
1964
- if (!data || typeof data !== 'object') return [];
1965
- const allCheckpoints = data as Record<string, TurnCheckpoint[]>;
1966
- return Array.isArray(allCheckpoints[agentName]) ? [...allCheckpoints[agentName]] : [];
2478
+ const entry = this.store.treeGet(TURN_CHECKPOINTS_TREE_ID, agentName);
2479
+ if (entry) {
2480
+ const blob = this.store.getBlob(entry.blobHash);
2481
+ if (!blob) return [];
2482
+ try {
2483
+ const parsed = JSON.parse(blob.toString());
2484
+ return Array.isArray(parsed) ? parsed : [];
2485
+ } catch {
2486
+ return [];
2487
+ }
2488
+ }
2489
+ if (!this.hasLegacyCheckpoints(agentName)) return [];
2490
+ const legacy = this.store.getStateJson(TURN_CHECKPOINTS_ID) as
2491
+ | Record<string, TurnCheckpoint[]>
2492
+ | null;
2493
+ const list = legacy?.[agentName];
2494
+ return Array.isArray(list) ? [...list] : [];
1967
2495
  }
1968
2496
 
1969
2497
  private saveTurnCheckpoints(agentName: string, checkpoints: TurnCheckpoint[]): void {
1970
- const data = this.store.getStateJson(TURN_CHECKPOINTS_ID);
1971
- const allCheckpoints = (data && typeof data === 'object' ? data : {}) as Record<string, TurnCheckpoint[]>;
1972
- allCheckpoints[agentName] = checkpoints;
1973
- this.store.setStateJson(TURN_CHECKPOINTS_ID, allCheckpoints);
2498
+ const bytes = Buffer.from(JSON.stringify(checkpoints));
2499
+ const blobHash = this.store.storeBlob(bytes, 'application/json');
2500
+ this.store.treeSet(TURN_CHECKPOINTS_TREE_ID, agentName, {
2501
+ blobHash,
2502
+ size: bytes.length,
2503
+ mode: 0o644,
2504
+ });
2505
+ }
2506
+
2507
+ /**
2508
+ * Drop a departed agent's checkpoint tree key and turn/redo bookkeeping.
2509
+ * Without this, spawn-and-dispose agents leave a key (and in-memory map
2510
+ * entries) behind for the life of the store/session.
2511
+ */
2512
+ private evictTurnCheckpoints(agentName: string): void {
2513
+ this.turnCounters.delete(agentName);
2514
+ this.redoStacks.delete(agentName);
2515
+ // Diagnostics maps are also keyed by agent name and never evicted —
2516
+ // spawn-and-dispose fleets would grow them for the session's life.
2517
+ this.staleWarnAt.delete(agentName);
2518
+ this.lastInferenceAt.delete(agentName);
2519
+ const entry = this.store.treeGet(TURN_CHECKPOINTS_TREE_ID, agentName);
2520
+ if (this.hasLegacyCheckpoints(agentName)) {
2521
+ // A bare treeRemove would resurrect the legacy map's list through the
2522
+ // fallback next time this name is reused — shadow it with an empty
2523
+ // tombstone instead. size <= 2 ⇔ the blob is already "[]".
2524
+ if (!entry || entry.size > 2) {
2525
+ this.saveTurnCheckpoints(agentName, []);
2526
+ }
2527
+ return;
2528
+ }
2529
+ if (entry) {
2530
+ this.store.treeRemove(TURN_CHECKPOINTS_TREE_ID, agentName);
2531
+ }
1974
2532
  }
1975
2533
 
1976
2534
  /**
@@ -1980,6 +2538,9 @@ export class AgentFramework {
1980
2538
  async runUntilIdle(): Promise<void> {
1981
2539
  while (
1982
2540
  !this.queue.isEmpty ||
2541
+ // Direct inference requests (e.g. runEphemeralToCompletion) bypass the
2542
+ // event queue — without this the loop can exit before they're drained.
2543
+ this.pendingRequests.length > 0 ||
1983
2544
  this.activeStreams.size > 0 ||
1984
2545
  Array.from(this.agents.values()).some((a) => a.state.status !== 'idle')
1985
2546
  ) {
@@ -2512,6 +3073,7 @@ export class AgentFramework {
2512
3073
  this.agents.delete(agentName);
2513
3074
  this.agentConfigs.delete(agentName);
2514
3075
  this.conversationAgentHomes.delete(agentName);
3076
+ this.evictTurnCheckpoints(agentName);
2515
3077
  this.emitTrace({
2516
3078
  type: 'mcpl:conversation-disposed',
2517
3079
  agentName,
@@ -2718,6 +3280,7 @@ export class AgentFramework {
2718
3280
  requestCount: requests.length,
2719
3281
  oldestRequestAge: now - oldest,
2720
3282
  });
3283
+ console.error(`[inference-dropped] agent=${agentName} reason=agent_not_found requests=${requests.length}`);
2721
3284
  continue;
2722
3285
  }
2723
3286
 
@@ -2733,6 +3296,14 @@ export class AgentFramework {
2733
3296
  requestCount: requests.length,
2734
3297
  oldestRequestAge: now - oldest,
2735
3298
  });
3299
+ // Loud (but throttled) note that requests are waiting on a busy agent.
3300
+ if ((this.staleWarnAt.get(agentName) ?? 0) < now - 60_000) {
3301
+ this.staleWarnAt.set(agentName, now);
3302
+ console.error(
3303
+ `[inference-stale] agent=${agentName} busy (${agent.state.status}) — ` +
3304
+ `${requests.length} request(s) waiting ${Math.round((now - oldest) / 1000)}s`,
3305
+ );
3306
+ }
2736
3307
  }
2737
3308
  this.pendingRequests.push(...requests);
2738
3309
  continue;
@@ -2740,6 +3311,13 @@ export class AgentFramework {
2740
3311
 
2741
3312
  // Check policy
2742
3313
  if (!this.inferencePolicy.shouldInfer(agentName, requests, state)) {
3314
+ // Loud drop: a queued request that dies here is otherwise invisible —
3315
+ // the 2026-07-09 mythos "not responding" diagnosis burned hours on
3316
+ // exactly this class of silent drop. One stderr line per drop.
3317
+ console.error(
3318
+ `[inference-dropped] agent=${agentName} reason=policy-skip ` +
3319
+ `requests=${requests.length} triggers=${requests.map((r) => r.reason).join(',')}`,
3320
+ );
2743
3321
  continue;
2744
3322
  }
2745
3323
 
@@ -2782,6 +3360,7 @@ export class AgentFramework {
2782
3360
 
2783
3361
  this.emitTrace({ type: 'inference:started', agentName: agent.name });
2784
3362
  this.eventGate?.onInferenceStarted(agent.name);
3363
+ this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), startedAt: Date.now() });
2785
3364
 
2786
3365
  try {
2787
3366
  const allTools = this.getAllTools();
@@ -2791,14 +3370,16 @@ export class AgentFramework {
2791
3370
  // Both produce ContextInjection[] that get merged before inference.
2792
3371
  let injections: ContextInjection[] | undefined;
2793
3372
 
2794
- // Module gatherContext (fail-open, 5s timeout per module)
2795
- // NOTE: module injections are NOT channel-scoped — only the MCPL hook
2796
- // injections below pass through scopeInjectionsForAgent (channel
2797
- // context arrives via beforeInference hooks by adapter convention). A
2798
- // module that starts emitting per-channel context must scope it per
2799
- // agent itself, or conversation forks will see cross-channel content.
3373
+ // Module gatherContext (fail-open, per-module timeout — the module's
3374
+ // contextTimeoutMs, else the registry default). Injections are
3375
+ // channel-scoped via scopeInjectionsForAgent, matching the MCPL hook
3376
+ // injections below, so conversation forks don't see cross-channel
3377
+ // content.
2800
3378
  try {
2801
- const moduleInjections = await this.moduleRegistry.gatherContext(agent.name);
3379
+ const moduleInjections = this.scopeInjectionsForAgent(
3380
+ agent.name,
3381
+ await this.moduleRegistry.gatherContext(agent.name),
3382
+ );
2802
3383
  if (moduleInjections.length > 0) {
2803
3384
  injections = moduleInjections;
2804
3385
  }
@@ -2845,6 +3426,13 @@ export class AgentFramework {
2845
3426
  type: 'inference:exhausted',
2846
3427
  agentName: agent.name,
2847
3428
  error: err.message,
3429
+ // Drives the poison-history breaker. `retryable` is kept for
3430
+ // observability, but the breaker gates on `errorType` — only an
3431
+ // `invalid_request` (a 400-class rejection of the history itself)
3432
+ // means retrying the same context can never succeed. auth/abort/
3433
+ // context_length are also non-retryable but must NOT cost history.
3434
+ retryable: err instanceof MembraneError ? err.retryable : undefined,
3435
+ errorType: err instanceof MembraneError ? err.type : undefined,
2848
3436
  });
2849
3437
  this.eventGate?.onInferenceEnded(agent.name);
2850
3438
  if (action.emit) {
@@ -3121,7 +3709,87 @@ export class AgentFramework {
3121
3709
  `[inference-refusal] agent=${agent.name} category=${category}` +
3122
3710
  (stopDetails?.explanation ? ` explanation=${stopDetails.explanation}` : ''),
3123
3711
  );
3124
- void this.reactToRefusal(agent.name, category);
3712
+
3713
+ this.noteRefusal(agent.name, category, tokenUsage);
3714
+
3715
+ // Rewind: excise the turn that fed the refusal, drop a
3716
+ // metadata-only marker, and retry — keeping the agent on its own
3717
+ // model instead of substituting a fallback. Driven either by an
3718
+ // admin `/unstick` (forced session) or the agent's autoRewind
3719
+ // config; both are bounded.
3720
+ let handledByRewind = false;
3721
+ const rh = agent.refusalHandling;
3722
+ const forced = this.forcedRewind.get(agent.name);
3723
+ // Shed exactly one more (newest, in-sequence) turn and keep the
3724
+ // single episode marker current. `budgetLeft` bounds the loop.
3725
+ const doRewind = (
3726
+ budgetLeft: boolean,
3727
+ onStep: (count: number) => void,
3728
+ onGiveUp: () => void,
3729
+ ): void => {
3730
+ if (!budgetLeft) { onGiveUp(); return; }
3731
+ const rec = this.shedNewestTurn(agent);
3732
+ if (!rec) { onGiveUp(); return; }
3733
+ const count = this.updateRewindMarker(agent, category);
3734
+ onStep(count);
3735
+ if (forced) forced.removed.push(rec);
3736
+ this.pendingRequests.push({
3737
+ agentName: agent.name,
3738
+ reason: forced ? 'unstick-retry' : 'refusal-rewind-retry',
3739
+ source: 'framework',
3740
+ timestamp: Date.now(),
3741
+ });
3742
+ handledByRewind = true;
3743
+ };
3744
+
3745
+ if (forced) {
3746
+ doRewind(
3747
+ forced.remaining > 0,
3748
+ (count) => {
3749
+ forced.remaining -= 1;
3750
+ console.error(
3751
+ `[unstick] agent=${agent.name} shed ${count} turn(s) ` +
3752
+ `(remaining ${forced.remaining})`,
3753
+ );
3754
+ },
3755
+ () => this.finishUnstick(agent.name, false, category),
3756
+ );
3757
+ } else if (rh?.autoRewind) {
3758
+ const cap = Math.max(1, rh.maxRewinds ?? 3);
3759
+ const used = this.refusalRewinds.get(agent.name) ?? 0;
3760
+ doRewind(
3761
+ used < cap,
3762
+ (count) => {
3763
+ this.refusalRewinds.set(agent.name, used + 1);
3764
+ console.error(
3765
+ `[refusal-rewind] agent=${agent.name} shed ${count} turn(s) so far ` +
3766
+ `(cap ${cap})`,
3767
+ );
3768
+ },
3769
+ () => {
3770
+ console.error(
3771
+ `[refusal-rewind] agent=${agent.name} gave up: ` +
3772
+ `${used >= cap ? `cap ${cap} reached` : 'nothing left to shed'}`,
3773
+ );
3774
+ this.refusalRewinds.set(agent.name, 0);
3775
+ this.rewindEpisode.delete(agent.name);
3776
+ },
3777
+ );
3778
+ }
3779
+
3780
+ if (!handledByRewind) void this.reactToRefusal(agent.name, category);
3781
+ } else {
3782
+ // A turn that completed WITHOUT a refusal ends the rewind episode:
3783
+ // the model responded. Leave the consolidated marker in place as
3784
+ // the durable record; just clear the per-episode counters.
3785
+ if (this.forcedRewind.has(agent.name)) {
3786
+ this.finishUnstick(agent.name, true);
3787
+ }
3788
+ if (this.refusalRewinds.get(agent.name)) {
3789
+ this.refusalRewinds.set(agent.name, 0);
3790
+ }
3791
+ this.rewindEpisode.delete(agent.name);
3792
+ this.refusalStreak.delete(agent.name);
3125
3793
  }
3126
3794
 
3127
3795
  // Dispatch speech (and thoughts if any)
@@ -3212,13 +3880,21 @@ export class AgentFramework {
3212
3880
  `(${silenced ? 'silencing tool / explicit send' : 'no prose'})`,
3213
3881
  );
3214
3882
  } else {
3883
+ // Snapshot the locus ONCE. This loop dispatches AFTER the agent
3884
+ // is idle (see PR #32 note below), so a queued inbound from
3885
+ // another channel can start a NEW turn and overwrite the per-agent
3886
+ // triggering channel (`activeTriggerChannels`) BETWEEN segments —
3887
+ // splitting one turn's reply across channels (the residual
3888
+ // multi-segment race left by 83dedf8's per-agent locus). Pin every
3889
+ // segment of THIS turn to the channel resolved right now.
3890
+ const turnLocus = this.channelRegistry.resolveLocus(agent.name);
3215
3891
  console.error(
3216
- `[routing] ${agent.name}: tool-call turn [${toolNames.join(', ')}] -> routing ${segments.length} prose segment(s) in order`,
3892
+ `[routing] ${agent.name}: tool-call turn [${toolNames.join(', ')}] -> routing ${segments.length} prose segment(s) in order -> ${turnLocus ?? '(default)'}`,
3217
3893
  );
3218
3894
  // Deliver sequentially (await each) so the segments land in order.
3219
3895
  for (const seg of segments) {
3220
3896
  try {
3221
- await this.channelRegistry.routeSpeech(agent.name, seg);
3897
+ await this.channelRegistry.routeSpeech(agent.name, seg, turnLocus);
3222
3898
  } catch (err) {
3223
3899
  console.error('speech routing failed:', err);
3224
3900
  }
@@ -3267,6 +3943,12 @@ export class AgentFramework {
3267
3943
  type: 'inference:exhausted',
3268
3944
  agentName: agent.name,
3269
3945
  error: err.message,
3946
+ // Drives the poison-history breaker. `retryable` is kept for
3947
+ // observability, but the breaker gates on `errorType` — only an
3948
+ // `invalid_request` (a 400-class rejection of the history
3949
+ // itself) means retrying the same context can never succeed.
3950
+ retryable: err instanceof MembraneError ? err.retryable : undefined,
3951
+ errorType: err instanceof MembraneError ? err.type : undefined,
3270
3952
  });
3271
3953
  this.eventGate?.onInferenceEnded(agent.name);
3272
3954
  if (action.emit) {
@@ -3281,19 +3963,58 @@ export class AgentFramework {
3281
3963
  // Only reset if this is still the active stream (a budget restart
3282
3964
  // may have already started a new stream, bumping streamId)
3283
3965
  if (agent.streamId === myStreamId) {
3966
+ const durationMs = Date.now() - startTime;
3284
3967
  agent.reset();
3285
3968
  this.emitTrace({
3286
3969
  type: 'inference:exhausted',
3287
3970
  agentName: agent.name,
3288
3971
  error: `Stream aborted: ${reason}`,
3289
3972
  });
3973
+ // Postmortem 2026-05-28 P2 #7: persist the abort to the
3974
+ // inference log so future investigations can attribute the
3975
+ // terminal cause without relying on live in-memory reducer
3976
+ // state. Without this, abort-terminated inferences are
3977
+ // invisible to forensic queries (only request-side telemetry
3978
+ // via llm-calls.jsonl shows them, and only by absence).
3979
+ this.logInference({
3980
+ timestamp: startTime,
3981
+ agentName: agent.name,
3982
+ requestId,
3983
+ success: false,
3984
+ error: `Stream aborted: ${reason}`,
3985
+ request: compiledRequest ?? { note: 'streaming request aborted' },
3986
+ durationMs,
3987
+ });
3290
3988
  this.eventGate?.onInferenceEnded(agent.name);
3291
3989
  }
3292
3990
  break;
3293
3991
  }
3294
3992
 
3295
- case 'usage':
3993
+ case 'usage': {
3296
3994
  agent.lastStreamInputTokens = event.usage.inputTokens;
3995
+
3996
+ // Closed-loop estimator calibration (2026-07-12). Sample the REAL
3997
+ // prefix size of THIS API call (fresh + cache write + cache read)
3998
+ // and hand it to the context strategy, which accepts exactly one
3999
+ // sample per compile (its arm-once gate) and rejects out-of-band
4000
+ // ratios. It must be sampled HERE, per call: `response.details.
4001
+ // usage` at turn completion is CUMULATIVE across the tool-use
4002
+ // loop (5 calls x ~160k reported as 884k), which is not a
4003
+ // window-shaped number and drove the multiplier to 2.37 before
4004
+ // the guards caught it.
4005
+ try {
4006
+ const realTotal =
4007
+ (event.usage.inputTokens ?? 0) +
4008
+ (event.usage.cacheCreationTokens ?? 0) +
4009
+ (event.usage.cacheReadTokens ?? 0);
4010
+ const strat = (agent as unknown as {
4011
+ getContextManager?: () => { getStrategy?: () => unknown };
4012
+ }).getContextManager?.()?.getStrategy?.() as
4013
+ | { reportRealInputTokens?: (n: number) => void }
4014
+ | undefined;
4015
+ strat?.reportRealInputTokens?.(realTotal);
4016
+ } catch { /* calibration is best-effort */ }
4017
+
3297
4018
  this.emitTrace({
3298
4019
  type: 'inference:usage',
3299
4020
  agentName: agent.name,
@@ -3305,12 +4026,14 @@ export class AgentFramework {
3305
4026
  },
3306
4027
  });
3307
4028
  break;
4029
+ }
3308
4030
  }
3309
4031
  }
3310
4032
  } catch (error) {
3311
4033
  // Stream itself threw (unexpected) — no retry path here, so also emit
3312
4034
  // inference:exhausted so ephemeral agent promises can settle.
3313
4035
  const err = error instanceof Error ? error : new Error(String(error));
4036
+ const durationMs = Date.now() - startTime;
3314
4037
  this.emitTrace({
3315
4038
  type: 'inference:failed',
3316
4039
  agentName: agent.name,
@@ -3322,9 +4045,31 @@ export class AgentFramework {
3322
4045
  agentName: agent.name,
3323
4046
  error: err.message,
3324
4047
  });
4048
+ // Postmortem 2026-05-28 P2 #7: catch-path failures (stream itself
4049
+ // threw) were previously only visible via in-memory trace listeners.
4050
+ // Persist to inference log for forensic attribution.
4051
+ this.logInference({
4052
+ timestamp: startTime,
4053
+ agentName: agent.name,
4054
+ requestId,
4055
+ success: false,
4056
+ error: `Stream threw: ${err.message}`,
4057
+ request: compiledRequest ?? { note: 'streaming request threw' },
4058
+ durationMs,
4059
+ });
3325
4060
  agent.reset();
3326
4061
  this.eventGate?.onInferenceEnded(agent.name);
3327
4062
  } finally {
4063
+ // Clear the gate's inference flag on EVERY exit path (paired with the
4064
+ // onInferenceStarted in startAgentStream). The branch-level calls above
4065
+ // are kept but are best-effort; if any exit path bypassed them the agent
4066
+ // stayed stuck in the gate's `inferring` set, which permanently BUFFERS
4067
+ // all incoming events → the agent silently stops waking on messages
4068
+ // (typing still stops, compression still runs — matching the observed
4069
+ // wedge). onInferenceEnded is idempotent, so a redundant call is safe.
4070
+ this.eventGate?.onInferenceEnded(agent.name);
4071
+ this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), endedAt: Date.now() });
4072
+
3328
4073
  // Stop the typing indicator on every exit path (complete, error,
3329
4074
  // exhausted, abort) so it never sticks after the turn ends.
3330
4075
  this.channelRegistry?.stopTyping();
@@ -3414,7 +4159,10 @@ export class AgentFramework {
3414
4159
  if (blocks) {
3415
4160
  return { toolUseId: callId, content: blocks, isError: false };
3416
4161
  }
3417
- let content = JSON.stringify(afResult.data);
4162
+ // JSON.stringify returns the VALUE undefined (not a string) for undefined
4163
+ // input — a module tool returning `{ success: true }` with no data would
4164
+ // otherwise make `content.length` below throw a TypeError mid-turn.
4165
+ let content = JSON.stringify(afResult.data) ?? '';
3418
4166
  if (maxChars && content.length > maxChars) {
3419
4167
  content = safeSlice(content, 0, maxChars)
3420
4168
  + '\n\n[truncated — original was ' + content.length + ' chars]';
@@ -3486,11 +4234,11 @@ export class AgentFramework {
3486
4234
  }
3487
4235
  }
3488
4236
 
3489
- // Append to the inference log state
3490
- const data = this.store.getStateJson(INFERENCE_LOG_ID);
3491
- const entries = Array.isArray(data) ? data : [];
3492
- entries.push(entryToStore);
3493
- this.store.setStateJson(INFERENCE_LOG_ID, entries);
4237
+ // Append one record per inference. A read-modify-write via setStateJson
4238
+ // would emit a whole-array Set — record size then grows with accumulated
4239
+ // history (O(n²) aggregate disk) and the append_log strategy registered
4240
+ // for this state never sees an append.
4241
+ this.store.appendToStateJson(INFERENCE_LOG_ID, entryToStore);
3494
4242
  }
3495
4243
 
3496
4244
  private logProcessEvent(event: ProcessEvent, responses: ModuleProcessResponse[]): void {
@@ -3510,11 +4258,8 @@ export class AgentFramework {
3510
4258
  entryToStore.responses = { blobId };
3511
4259
  }
3512
4260
 
3513
- // Append to the process log state
3514
- const data = this.store.getStateJson(PROCESS_LOG_ID);
3515
- const entries = Array.isArray(data) ? data : [];
3516
- entries.push(entryToStore);
3517
- this.store.setStateJson(PROCESS_LOG_ID, entries);
4261
+ // Append one record per event — same rationale as logInference.
4262
+ this.store.appendToStateJson(PROCESS_LOG_ID, entryToStore);
3518
4263
  }
3519
4264
 
3520
4265
  /**
@@ -3744,14 +4489,20 @@ export class AgentFramework {
3744
4489
  this.noteInferenceExhausted(
3745
4490
  (event.agentName as string) ?? 'unknown',
3746
4491
  (event.error as string) ?? 'unknown error',
4492
+ event.retryable as boolean | undefined,
4493
+ event.errorType as string | undefined,
3747
4494
  );
3748
4495
  } else if (event.type === 'inference:completed') {
3749
4496
  // A successful response — even mid-turn between tool calls — proves the
3750
- // agent isn't hard-down; clear its consecutive-failure streak.
4497
+ // agent isn't hard-down; clear its consecutive-failure streak (and the
4498
+ // poison-history breaker's rewind budget).
3751
4499
  const name = event.agentName as string | undefined;
3752
4500
  if (name && this.consecutiveInferenceFailures.get(name)) {
3753
4501
  this.consecutiveInferenceFailures.set(name, 0);
3754
4502
  }
4503
+ if (name && this.exhaustionRewinds.get(name)) {
4504
+ this.exhaustionRewinds.set(name, 0);
4505
+ }
3755
4506
  }
3756
4507
 
3757
4508
  const traceEvent = {
@@ -3782,14 +4533,152 @@ export class AgentFramework {
3782
4533
  * inference, so this never causes a retry/wake loop.
3783
4534
  * 3. escalation — after N consecutive failures the agent is hard-down;
3784
4535
  * log that loudly (a repeated identical failure is the textbook signal).
4536
+ * 4. breaker — when the failures are an `invalid_request` (a 400-class
4537
+ * rejection of the history itself, e.g. corrupted tool_use/tool_result
4538
+ * pairing or an oversized attachment), retrying the same context can
4539
+ * never succeed: every new push event wakes the agent onto the same
4540
+ * poisoned history forever. At the hard-down threshold, automatically
4541
+ * quarantine: shed the newest complete exchange (the same forced-rewind
4542
+ * primitive `/unstick` uses — shedNewestTurn never orphans a
4543
+ * tool_use/thinking block) and retry, bounded by the same rewind cap so
4544
+ * the breaker can never eat the whole history.
4545
+ */
4546
+ /**
4547
+ * Live health snapshot for /healthz and doctor tooling: gate state, queued
4548
+ * work, per-agent status + last inference activity. Cheap and read-only.
3785
4549
  */
3786
- private noteInferenceExhausted(agentName: string, reason: string): void {
4550
+ healthSnapshot(): Record<string, unknown> {
4551
+ return {
4552
+ at: new Date().toISOString(),
4553
+ uptimeSec: Math.round(process.uptime()),
4554
+ gate: this.eventGate?.inferenceDiagnostics() ?? null,
4555
+ pendingRequests: this.pendingRequests.length,
4556
+ activeStreams: [...this.activeStreams.keys()],
4557
+ agents: [...this.agents.entries()].map(([name, agent]) => ({
4558
+ name,
4559
+ status: agent.state.status,
4560
+ consecutiveInferenceFailures: this.consecutiveInferenceFailures.get(name) ?? 0,
4561
+ lastInference: this.lastInferenceAt.get(name) ?? null,
4562
+ refusalStats: this.refusalStats.get(name) ?? null,
4563
+ })),
4564
+ };
4565
+ }
4566
+
4567
+ /** Append a structured JSONL record to logs/failures.log (best-effort).
4568
+ * Durable and independent of journald/unit log redirects — this is what
4569
+ * connectome-doctor and fleet tooling read. Always stamps `at`. Legacy
4570
+ * records carry {at, agent, consecutive, reason}; new records add `kind`
4571
+ * and kind-specific fields (additive only — doctor parses by regex). */
4572
+ private logFailure(record: Record<string, unknown>): void {
4573
+ try {
4574
+ mkdirSync('logs', { recursive: true });
4575
+ appendFileSync(
4576
+ 'logs/failures.log',
4577
+ JSON.stringify({ at: new Date().toISOString(), ...record }) + '\n',
4578
+ );
4579
+ } catch { /* best-effort */ }
4580
+ }
4581
+
4582
+ /**
4583
+ * Ops alert — the one escalation path for "a human should hear about this":
4584
+ * (1) durable failures.log record (unless the caller already wrote one),
4585
+ * (2) an `ops:alert` trace so authorized observers get it on the wire,
4586
+ * (3) if CONNECTOME_OPS_WEBHOOK is set, a Discord post throttled to one per
4587
+ * (agent, kind) per cooldown window — a persistent failure re-posts
4588
+ * every ~15 min instead of flooding the channel on every occurrence.
4589
+ * See connectome docs/observability.md.
4590
+ */
4591
+ private opsAlert(
4592
+ kind: string,
4593
+ agentName: string,
4594
+ message: string,
4595
+ opts?: { data?: Record<string, unknown>; skipLog?: boolean },
4596
+ ): void {
4597
+ if (!opts?.skipLog) {
4598
+ this.logFailure({ agent: agentName, kind, reason: message, ...opts?.data });
4599
+ }
4600
+ this.emitTrace({ type: 'ops:alert', kind, agentName, message, data: opts?.data });
4601
+
4602
+ const hook = process.env.CONNECTOME_OPS_WEBHOOK;
4603
+ if (!hook) return;
4604
+ const key = `${agentName}:${kind}`;
4605
+ const now = Date.now();
4606
+ if (now - (this.opsAlertLastSent.get(key) ?? 0) < this.opsAlertCooldownMs) return;
4607
+ // Stamp BEFORE the async post so a burst can't race past the cooldown.
4608
+ this.opsAlertLastSent.set(key, now);
4609
+ fetch(hook, {
4610
+ method: 'POST',
4611
+ headers: { 'content-type': 'application/json' },
4612
+ body: JSON.stringify({
4613
+ content: `\u{1F6A8} **${agentName}** ${kind}: ${message.slice(0, 500)}`,
4614
+ }),
4615
+ }).catch((err) => console.error(
4616
+ `[ops-alert] webhook post failed (${key}):`,
4617
+ err instanceof Error ? err.message : err,
4618
+ ));
4619
+ }
4620
+
4621
+ /**
4622
+ * Record a model refusal for observability: per-agent stats + a
4623
+ * consecutive-refusal streak (exposed via healthSnapshot), a durable
4624
+ * failures.log record for EVERY refusal (previously invisible outside
4625
+ * stderr), and a throttled ops alert once refusals repeat — one refusal is
4626
+ * often survivable (auto-rewind may clear it); two in a row means stuck.
4627
+ * The streak is reset by any non-refusal completion in the stream driver.
4628
+ */
4629
+ private noteRefusal(
4630
+ agentName: string,
4631
+ category: string,
4632
+ tokens?: { input: number; output: number },
4633
+ ): number {
4634
+ const stats = this.refusalStats.get(agentName)
4635
+ ?? { total: 0, byCategory: {}, lastAt: 0, lastCategory: '' };
4636
+ stats.total += 1;
4637
+ stats.byCategory[category] = (stats.byCategory[category] ?? 0) + 1;
4638
+ stats.lastAt = Date.now();
4639
+ stats.lastCategory = category;
4640
+ this.refusalStats.set(agentName, stats);
4641
+
4642
+ const streak = (this.refusalStreak.get(agentName) ?? 0) + 1;
4643
+ this.refusalStreak.set(agentName, streak);
4644
+
4645
+ this.logFailure({
4646
+ agent: agentName,
4647
+ kind: 'refusal',
4648
+ category,
4649
+ streak,
4650
+ locus: this.channelRegistry?.buildChannelContext()?.incoming?.channelId ?? null,
4651
+ tokens: tokens ?? null,
4652
+ });
4653
+ if (streak >= 2) {
4654
+ this.opsAlert(
4655
+ 'refusal',
4656
+ agentName,
4657
+ `refusal streak ${streak}, category=${category}`,
4658
+ { skipLog: true, data: { category, streak } },
4659
+ );
4660
+ }
4661
+ return streak;
4662
+ }
4663
+
4664
+ private noteInferenceExhausted(
4665
+ agentName: string,
4666
+ reason: string,
4667
+ retryable?: boolean,
4668
+ errorType?: string,
4669
+ ): void {
3787
4670
  const streak = (this.consecutiveInferenceFailures.get(agentName) ?? 0) + 1;
3788
4671
  this.consecutiveInferenceFailures.set(agentName, streak);
4672
+ this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), failedAt: Date.now(), lastError: reason.slice(0, 300) });
3789
4673
 
3790
4674
  // (1) Durable stderr line — works in headless/daemon mode with no client.
3791
4675
  console.error(`[inference-failed] agent=${agentName} consecutive=${streak}: ${reason}`);
3792
4676
 
4677
+ // (1b) Machine-greppable durable record, independent of journald/unit log
4678
+ // redirects: logs/failures.log under the host's working directory. This is
4679
+ // what connectome-doctor reads. Legacy fields kept; `kind` is additive.
4680
+ this.logFailure({ agent: agentName, consecutive: streak, reason, kind: 'inference-exhausted' });
4681
+
3793
4682
  // (2) Agent-facing chronicle marker (no inference triggered → no loop).
3794
4683
  const agent = this.agents.get(agentName);
3795
4684
  if (agent && process.env.SUPPRESS_INFERENCE_FAILED_MARKER !== '1') {
@@ -3818,7 +4707,80 @@ export class AgentFramework {
3818
4707
  `[inference-hard-down] agent=${agentName} has FAILED ${streak} consecutive ` +
3819
4708
  `inferences — it cannot complete a turn. Last reason: ${reason}`,
3820
4709
  );
4710
+
4711
+ // Ops alert (webhook + ops:alert trace), throttled per (agent, kind) —
4712
+ // a hard-down agent re-posts every ~15 min, not on every failed retry.
4713
+ // failures.log already got the per-exhaustion record above.
4714
+ this.opsAlert(
4715
+ 'hard-down',
4716
+ agentName,
4717
+ `${streak} consecutive inference failures. Last reason: ${reason}`,
4718
+ { skipLog: true },
4719
+ );
4720
+
4721
+ // (4) Poison-history breaker: ONLY for `invalid_request` — a 400-class
4722
+ // rejection of the history itself. Deliberately NOT keyed on
4723
+ // `retryable === false`, which membrane also returns for auth (expired
4724
+ // key), abort (deliberate cancel), context_length (compression's job,
4725
+ // and shedding newest is the wrong direction), safety and unsupported —
4726
+ // none of which mean the history is poisoned, and all of which would
4727
+ // otherwise shed good exchanges and stamp a false "the API kept
4728
+ // rejecting your history" marker. `retryable` is kept only for the trace.
4729
+ void retryable;
4730
+ if (errorType === 'invalid_request' && agent) {
4731
+ this.quarantinePoisonedHistory(agent, reason);
4732
+ }
4733
+ }
4734
+ }
4735
+
4736
+ /**
4737
+ * Automatic poison-history quarantine (the actual "breaker"). Sheds the
4738
+ * newest complete exchange from the agent's history — reusing the same
4739
+ * primitives as the refusal auto-rewind / `/unstick` (shedNewestTurn +
4740
+ * the single consolidated episode marker) — and queues a retry so the
4741
+ * rewound history is verified immediately instead of waiting for the next
4742
+ * push event to wake the agent onto the same poisoned context.
4743
+ *
4744
+ * Bounded: at most `refusalHandling.maxRewinds` (default 3, hard cap 10)
4745
+ * sheds per failure episode; the budget resets on any successful inference.
4746
+ * At the cap (or with nothing left to shed) it stops — the agent stays up
4747
+ * and hard-down logging continues, but no further history is consumed.
4748
+ */
4749
+ private quarantinePoisonedHistory(agent: Agent, reason: string): void {
4750
+ const cap = Math.max(1, Math.min(10, agent.refusalHandling?.maxRewinds ?? 3));
4751
+ const used = this.exhaustionRewinds.get(agent.name) ?? 0;
4752
+ if (used >= cap) {
4753
+ console.error(
4754
+ `[inference-rewind] agent=${agent.name} rewind cap ${cap} reached — ` +
4755
+ `not shedding further history. Manual repair (/unstick or /undo) needed.`,
4756
+ );
4757
+ return;
4758
+ }
4759
+
4760
+ const rec = this.shedNewestTurn(agent);
4761
+ if (!rec) {
4762
+ console.error(
4763
+ `[inference-rewind] agent=${agent.name} has nothing left to shed — ` +
4764
+ `history is only system markers. Manual repair needed.`,
4765
+ );
4766
+ return;
3821
4767
  }
4768
+
4769
+ this.exhaustionRewinds.set(agent.name, used + 1);
4770
+ const count = this.updateRewindMarker(agent, truncateReason(reason), 'inference-failure');
4771
+ console.error(
4772
+ `[inference-rewind] agent=${agent.name} auto-quarantined ${rec.descriptor} ` +
4773
+ `(${count} turn(s) shed this episode, cap ${cap}) after non-retryable ` +
4774
+ `inference failures — retrying on the rewound history.`,
4775
+ );
4776
+
4777
+ // Retry immediately on the repaired history (bounded by the cap above).
4778
+ this.pendingRequests.push({
4779
+ agentName: agent.name,
4780
+ reason: 'inference-failure-rewind-retry',
4781
+ source: 'framework',
4782
+ timestamp: Date.now(),
4783
+ });
3822
4784
  }
3823
4785
 
3824
4786
  // ==========================================================================
@@ -3930,8 +4892,9 @@ export class AgentFramework {
3930
4892
  },
3931
4893
  );
3932
4894
 
3933
- // Host capabilities advertised during the MCP handshake
3934
- const hostCapabilities: McplHostCapabilities = {
4895
+ // Host capabilities advertised during the MCP handshake — stored so
4896
+ // servers can also be connected later at runtime (connectMcplServer).
4897
+ this.mcplHostCapabilities = {
3935
4898
  version: '0.4',
3936
4899
  pushEvents: true,
3937
4900
  contextHooks: {
@@ -3943,66 +4906,242 @@ export class AgentFramework {
3943
4906
 
3944
4907
  for (const config of serverConfigs) {
3945
4908
  try {
3946
- // Record per-server channel subscription policy before the server
3947
- // registers channels — handleRegister fires during the handshake.
3948
- if (config.channelSubscription !== undefined && this.channelRegistry) {
3949
- this.channelRegistry.setSubscriptionPolicy(config.id, config.channelSubscription);
3950
- }
4909
+ await this.connectMcplServerInternal(config);
4910
+ } catch (error) {
4911
+ // Fail-open: log and continue with remaining servers
4912
+ const err = error instanceof Error ? error : new Error(String(error));
4913
+ console.error(`Failed to connect MCPL server "${config.id}":`, err.message);
4914
+ // Reaching this catch means no stub was created, so nothing will
4915
+ // retry — except in the edge case where addServer succeeded (with a
4916
+ // reconnecting stub) and a later setup step threw; config.reconnect
4917
+ // is the best cheap approximation of that.
4918
+ this.emitTrace({
4919
+ type: 'mcpl:server-connect-failed',
4920
+ serverId: config.id,
4921
+ error: err.message,
4922
+ attempt: 0,
4923
+ willRetry: config.reconnect === true,
4924
+ });
4925
+ }
4926
+ }
3951
4927
 
3952
- const connection = await this.mcplServerRegistry.addServer(config, hostCapabilities);
3953
-
3954
- // Wire event listeners then flush any events that arrived during the
3955
- // handshake window (between setupMessageRouting and now).
3956
- this.wireMcplEvents(connection);
3957
- connection.ready();
3958
-
3959
- // Initialize feature sets if server advertises MCPL capabilities
3960
- if (connection.capabilities) {
3961
- const updateParams = this.featureSetManager.initializeServer(
3962
- config.id,
3963
- connection.capabilities,
3964
- {
3965
- enabledFeatureSets: config.enabledFeatureSets,
3966
- disabledFeatureSets: config.disabledFeatureSets,
3967
- },
3968
- );
4928
+ // Discover tools from all connected servers
4929
+ await this.refreshMcplTools();
4930
+ }
3969
4931
 
3970
- // Inform server which feature sets are enabled/disabled
3971
- if (updateParams.enabled?.length || updateParams.disabled?.length) {
3972
- connection.sendFeatureSetsUpdate(updateParams);
3973
- }
4932
+ /**
4933
+ * Connect a single MCPL server: register routing entries, open the
4934
+ * connection, wire events, and initialize feature sets / scopes /
4935
+ * checkpoints. Shared by startup (initializeMcpl) and the runtime
4936
+ * lifecycle API (connectMcplServer). Throws on connection failure.
4937
+ */
4938
+ private async connectMcplServerInternal(
4939
+ config: import('./mcpl/types.js').McplServerConfig,
4940
+ ): Promise<void> {
4941
+ if (!this.mcplServerRegistry || !this.mcplHostCapabilities) {
4942
+ throw new Error('MCPL subsystem is not initialized');
4943
+ }
3974
4944
 
3975
- // Configure scope whitelist/blacklist patterns
3976
- if (config.scopes) {
3977
- this.scopeManager.configureAll(config.scopes);
3978
- }
4945
+ // Register prefix + config for tool dispatch routing (idempotent with
4946
+ // the pre-registration pass in initializeMcpl).
4947
+ const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
4948
+ this.mcplPrefixMap.set(prefix, config.id);
4949
+ this.mcplServerConfigs.set(config.id, config);
3979
4950
 
3980
- // Register stateful feature sets with checkpoint manager (Step 8)
3981
- if (this.checkpointManager) {
3982
- const declared = this.featureSetManager.getDeclaredFeatureSets(config.id);
3983
- if (declared) {
3984
- for (const [fsName, fsDecl] of Object.entries(declared)) {
3985
- if (fsDecl.rollback || fsDecl.hostState) {
3986
- this.checkpointManager.registerFeatureSet(config.id, fsName, {
3987
- hostState: fsDecl.hostState ?? false,
3988
- rollback: fsDecl.rollback ?? false,
3989
- });
3990
- }
3991
- }
3992
- }
4951
+ // Record per-server channel subscription policy before the server
4952
+ // registers channels — handleRegister fires during the handshake.
4953
+ if (config.channelSubscription !== undefined && this.channelRegistry) {
4954
+ this.channelRegistry.setSubscriptionPolicy(config.id, config.channelSubscription);
4955
+ }
4956
+
4957
+ const connection = await this.mcplServerRegistry.addServer(config, this.mcplHostCapabilities);
4958
+
4959
+ // Wire event listeners then flush any events that arrived during the
4960
+ // handshake window (between setupMessageRouting and now).
4961
+ this.wireMcplEvents(connection);
4962
+ connection.ready();
4963
+
4964
+ // Initialize feature sets if server advertises MCPL capabilities
4965
+ this.registerMcplServerFeatures(config, connection);
4966
+
4967
+ this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` });
4968
+ }
4969
+
4970
+ /**
4971
+ * (Re-)establish a server's host-side MCPL registration: feature-set state
4972
+ * (plus the featureSets/update sent to the server), scope patterns, and
4973
+ * stateful-feature-set checkpoint registration.
4974
+ *
4975
+ * Called on initial connect (connectMcplServerInternal) AND after every
4976
+ * auto-reconnect. The 'close' handler drops the feature-set registration, so
4977
+ * without the reconnect re-run FeatureSetManager.validateInbound would throw
4978
+ * "Unknown server" forever — every push event and inference request from the
4979
+ * revived server silently rejected until a full host restart.
4980
+ *
4981
+ * CheckpointManager.registerFeatureSet is idempotent (no-op for a key that
4982
+ * already has a tree), so checkpoint trees preserved across a transient
4983
+ * disconnect are resumed, not reset.
4984
+ */
4985
+ private registerMcplServerFeatures(
4986
+ config: import('./mcpl/types.js').McplServerConfig,
4987
+ connection: McplServerConnection,
4988
+ ): void {
4989
+ if (!connection.capabilities) return;
4990
+
4991
+ const updateParams = this.featureSetManager!.initializeServer(
4992
+ config.id,
4993
+ connection.capabilities,
4994
+ {
4995
+ enabledFeatureSets: config.enabledFeatureSets,
4996
+ disabledFeatureSets: config.disabledFeatureSets,
4997
+ },
4998
+ );
4999
+
5000
+ // Inform server which feature sets are enabled/disabled
5001
+ if (updateParams.enabled?.length || updateParams.disabled?.length) {
5002
+ connection.sendFeatureSetsUpdate(updateParams);
5003
+ }
5004
+
5005
+ // Configure scope whitelist/blacklist patterns
5006
+ if (config.scopes) {
5007
+ this.scopeManager!.configureAll(config.scopes);
5008
+ }
5009
+
5010
+ // Register stateful feature sets with checkpoint manager (Step 8)
5011
+ if (this.checkpointManager) {
5012
+ const declared = this.featureSetManager!.getDeclaredFeatureSets(config.id);
5013
+ if (declared) {
5014
+ for (const [fsName, fsDecl] of Object.entries(declared)) {
5015
+ if (fsDecl.rollback || fsDecl.hostState) {
5016
+ this.checkpointManager.registerFeatureSet(config.id, fsName, {
5017
+ hostState: fsDecl.hostState ?? false,
5018
+ rollback: fsDecl.rollback ?? false,
5019
+ });
3993
5020
  }
3994
5021
  }
3995
-
3996
- this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` });
3997
- } catch (error) {
3998
- // Fail-open: log and continue with remaining servers
3999
- const err = error instanceof Error ? error : new Error(String(error));
4000
- console.error(`Failed to connect MCPL server "${config.id}":`, err.message);
4001
5022
  }
4002
5023
  }
5024
+ }
4003
5025
 
4004
- // Discover tools from all connected servers
5026
+ // ==========================================================================
5027
+ // Runtime MCPL server lifecycle (agent-facing hot deploy/restart/unload)
5028
+ // ==========================================================================
5029
+
5030
+ /**
5031
+ * Connect a new MCPL server at runtime. Refreshes the tool list and
5032
+ * notifies the agent of newly available tools. Throws if the MCPL
5033
+ * subsystem is not initialized, the id is already connected, or the
5034
+ * connection/handshake fails.
5035
+ */
5036
+ async connectMcplServer(
5037
+ config: import('./mcpl/types.js').McplServerConfig,
5038
+ ): Promise<void> {
5039
+ // Lazily bring up the MCPL subsystem — a framework that started with zero
5040
+ // configured servers can still deploy its first one at runtime.
5041
+ if (!this.mcplServerRegistry || !this.mcplHostCapabilities) {
5042
+ await this.initializeMcpl([], this.mcplInferenceRoutingConfig ?? undefined);
5043
+ }
5044
+
5045
+ // Same collision rules create() enforces at startup: the tool prefix must
5046
+ // not shadow a module name or another server's prefix.
5047
+ const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
5048
+ if (this.moduleRegistry.getAllModules().some(m => m.name === prefix)) {
5049
+ throw new Error(
5050
+ `MCPL server "${config.id}" toolPrefix "${prefix}" collides with module "${prefix}"`,
5051
+ );
5052
+ }
5053
+ const prefixOwner = this.mcplPrefixMap.get(prefix);
5054
+ if (prefixOwner && prefixOwner !== config.id) {
5055
+ throw new Error(
5056
+ `MCPL server "${config.id}" toolPrefix "${prefix}" collides with server "${prefixOwner}"`,
5057
+ );
5058
+ }
5059
+
5060
+ const oldToolNames = new Set(this.mcplTools.map(t => t.name));
5061
+ await this.connectMcplServerInternal(config);
4005
5062
  await this.refreshMcplTools();
5063
+ this.emitMcplToolDiff(oldToolNames, config.id);
5064
+ }
5065
+
5066
+ /**
5067
+ * Disconnect an MCPL server at runtime: close the connection, destroy its
5068
+ * feature-set and checkpoint state (permanent removal — unlike a transient
5069
+ * transport close, which preserves checkpoints for the reconnect), remove
5070
+ * its channels from the registry, drop routing entries, and refresh tools.
5071
+ * No-op-ish if the server is not connected (still clears routing state).
5072
+ */
5073
+ async disconnectMcplServer(id: string): Promise<void> {
5074
+ if (!this.mcplServerRegistry) {
5075
+ throw new Error('MCPL subsystem is not initialized');
5076
+ }
5077
+ const config = this.mcplServerConfigs.get(id);
5078
+ const oldToolNames = new Set(this.mcplTools.map(t => t.name));
5079
+
5080
+ await this.mcplServerRegistry.removeServer(id);
5081
+ this.channelRegistry?.removeServer(id);
5082
+
5083
+ // Permanent removal: also destroy feature-set and checkpoint state
5084
+ // explicitly. The 'close' handler usually does this, but a connection that
5085
+ // already transiently closed (reconnect pending) emits no second 'close'
5086
+ // from close(), and the transient path deliberately preserves checkpoints.
5087
+ this.featureSetManager?.removeServer(id);
5088
+ this.checkpointManager?.removeServer(id);
5089
+
5090
+ const prefix = config?.toolPrefix ?? `mcpl--${id}`;
5091
+ this.mcplPrefixMap.delete(prefix);
5092
+ this.mcplServerConfigs.delete(id);
5093
+
5094
+ await this.refreshMcplTools();
5095
+ this.emitMcplToolDiff(oldToolNames, id);
5096
+ }
5097
+
5098
+ /**
5099
+ * Restart an MCPL server at runtime: disconnect, then reconnect with the
5100
+ * same (or an updated) config. This actually respawns a stdio child —
5101
+ * unlike `reconnect: true`, which only retries after transport-level
5102
+ * failures. Throws if the server has no stored config and none is given.
5103
+ */
5104
+ async restartMcplServer(
5105
+ id: string,
5106
+ newConfig?: import('./mcpl/types.js').McplServerConfig,
5107
+ ): Promise<void> {
5108
+ const config = newConfig ?? this.mcplServerConfigs.get(id);
5109
+ if (!config) {
5110
+ throw new Error(`MCPL server "${id}" is not configured`);
5111
+ }
5112
+ await this.disconnectMcplServer(id);
5113
+ await this.connectMcplServer(config);
5114
+ }
5115
+
5116
+ /**
5117
+ * List configured MCPL servers with live connection status and the number
5118
+ * of tools each currently contributes.
5119
+ */
5120
+ listMcplServers(): Array<{
5121
+ id: string;
5122
+ connected: boolean;
5123
+ toolPrefix: string;
5124
+ toolCount: number;
5125
+ command?: string;
5126
+ url?: string;
5127
+ }> {
5128
+ const result: Array<{
5129
+ id: string; connected: boolean; toolPrefix: string; toolCount: number;
5130
+ command?: string; url?: string;
5131
+ }> = [];
5132
+ for (const [id, config] of this.mcplServerConfigs) {
5133
+ const prefix = config.toolPrefix ?? `mcpl--${id}`;
5134
+ const connection = this.mcplServerRegistry?.getServer(id) ?? null;
5135
+ result.push({
5136
+ id,
5137
+ connected: connection?.isConnected ?? false,
5138
+ toolPrefix: prefix,
5139
+ toolCount: this.mcplTools.filter(t => t.name.startsWith(`${prefix}--`)).length,
5140
+ command: config.command,
5141
+ url: config.url,
5142
+ });
5143
+ }
5144
+ return result;
4006
5145
  }
4007
5146
 
4008
5147
  /**
@@ -4100,15 +5239,116 @@ export class AgentFramework {
4100
5239
  this.handleToolsListChanged(connection.id);
4101
5240
  });
4102
5241
 
4103
- // Also refresh tools on reconnect (server may have different tools)
4104
- connection.on('reconnect', () => {
5242
+ // Re-establish full server registration on reconnect. The 'close' handler
5243
+ // removed the feature-set state, so pushes / inference requests from the
5244
+ // revived server would otherwise be rejected with "Unknown server" until a
5245
+ // host restart. Re-runs the same init as the initial connect path (the
5246
+ // fresh handshake refreshed connection.capabilities); checkpoint trees
5247
+ // preserved across the transient close are resumed by idempotent
5248
+ // registration. Then refresh tools (server may have different tools).
5249
+ connection.on('reconnect', (info?: { attempts?: number }) => {
5250
+ try {
5251
+ const config = this.mcplServerConfigs.get(connection.id);
5252
+ if (config) {
5253
+ this.registerMcplServerFeatures(config, connection);
5254
+ }
5255
+ } catch (error) {
5256
+ console.error(
5257
+ `MCPL server "${connection.id}" re-registration after reconnect failed:`,
5258
+ error instanceof Error ? error.message : error,
5259
+ );
5260
+ }
4105
5261
  this.handleToolsListChanged(connection.id);
5262
+ this.emitTrace({
5263
+ type: 'mcpl:server-reconnected',
5264
+ serverId: connection.id,
5265
+ attempts: info?.attempts ?? 0,
5266
+ });
5267
+ // Mirror the module:removed emitted on 'close' so module-lifecycle
5268
+ // consumers see the server come back, not just vanish.
5269
+ this.emitTrace({ type: 'module:added', moduleName: `mcpl:${connection.id}` });
5270
+ });
5271
+
5272
+ // Surface connect/reconnect failures. Before these traces existed the
5273
+ // only receipt was a console.error on the host's own stderr — invisible
5274
+ // unless someone ssh'd in and read the process log.
5275
+ connection.on('connect-failed', (params: { error: string; attempt: number }) => {
5276
+ this.emitTrace({
5277
+ type: 'mcpl:server-connect-failed',
5278
+ serverId: connection.id,
5279
+ error: params.error,
5280
+ attempt: params.attempt,
5281
+ willRetry: connection.willReconnect,
5282
+ });
5283
+ });
5284
+ connection.on('reconnect-failed', (params: { error: string; attempt: number }) => {
5285
+ this.emitTrace({
5286
+ type: 'mcpl:server-connect-failed',
5287
+ serverId: connection.id,
5288
+ error: params.error,
5289
+ attempt: params.attempt,
5290
+ willRetry: connection.willReconnect,
5291
+ });
5292
+ // The reconnect loop never gives up (backoff caps at ~300s), so
5293
+ // "the server is effectively down" is an attempt-count judgment:
5294
+ // 5 failed attempts ≈ a few minutes of outage. Throttled per
5295
+ // (serverId, kind), so a long outage re-posts every ~15 min.
5296
+ if (params.attempt >= 5) {
5297
+ this.opsAlert(
5298
+ 'mcpl-down',
5299
+ connection.id,
5300
+ `MCPL server unreachable (attempt ${params.attempt}): ${params.error}`,
5301
+ );
5302
+ }
4106
5303
  });
4107
5304
 
4108
- // Cleanup on disconnect
4109
- connection.on('close', () => {
5305
+ // A late response to a tools/call that already timed out, carrying
5306
+ // stateful data the server advanced to. The host can't re-inject it (the
5307
+ // dispatch context is gone), but surfacing it makes the checkpoint-tree
5308
+ // divergence greppable instead of a silent drift into stale state.
5309
+ connection.on('orphaned-response', (info: {
5310
+ id: string | number;
5311
+ hadState: boolean;
5312
+ hadCheckpoint: boolean;
5313
+ }) => {
5314
+ this.emitTrace({
5315
+ type: 'mcpl:orphaned-response',
5316
+ serverId: connection.id,
5317
+ responseId: info.id,
5318
+ hadState: info.hadState,
5319
+ hadCheckpoint: info.hadCheckpoint,
5320
+ });
5321
+ });
5322
+
5323
+ // Connection-level errors (e.g. child process 'error' after spawn).
5324
+ // Attaching this listener also keeps an unhandled EventEmitter 'error'
5325
+ // from crashing the host process.
5326
+ connection.on('error', (err: Error) => {
5327
+ this.emitTrace({
5328
+ type: 'mcpl:server-error',
5329
+ serverId: connection.id,
5330
+ error: err.message,
5331
+ });
5332
+ });
5333
+
5334
+ // Cleanup on disconnect. Feature-set state is always dropped (the server
5335
+ // can't be validated while down; reconnect re-registers it). Checkpoint
5336
+ // trees, however, are durable state: CheckpointManager.removeServer deletes
5337
+ // them AND persists the deletion to Chronicle. The close handler must
5338
+ // therefore NEVER destroy checkpoints — `willReconnect` is false on a clean
5339
+ // stop() too (reconnectEnabled is cleared before 'close' fires), so gating
5340
+ // on it would erase every checkpoint tree on an ordinary host restart while
5341
+ // a SIGKILL preserves them. Permanent removal is owned solely by
5342
+ // disconnectMcplServer, which deletes the trees explicitly.
5343
+ connection.on('close', (code?: number | null, signal?: string | null) => {
4110
5344
  this.featureSetManager?.removeServer(connection.id);
4111
- this.checkpointManager?.removeServer(connection.id);
5345
+ this.emitTrace({
5346
+ type: 'mcpl:server-closed',
5347
+ serverId: connection.id,
5348
+ code: code ?? null,
5349
+ signal: signal ?? null,
5350
+ willReconnect: connection.willReconnect,
5351
+ });
4112
5352
  this.emitTrace({ type: 'module:removed', moduleName: `mcpl:${connection.id}` });
4113
5353
  });
4114
5354
  }
@@ -4507,7 +5747,7 @@ export class AgentFramework {
4507
5747
  return;
4508
5748
  }
4509
5749
 
4510
- const { until } = gate.setSleep(seconds, input.message);
5750
+ const { until } = gate.setSleep(seconds, input.message, agentName);
4511
5751
  const announce = input.announce !== false; // default true
4512
5752
  const human =
4513
5753
  seconds >= 3600 ? `${(seconds / 3600).toFixed(1)}h`