@animalabs/agent-framework 0.6.0 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/dist/src/agent.d.ts +4 -0
  2. package/dist/src/agent.d.ts.map +1 -1
  3. package/dist/src/agent.js +15 -0
  4. package/dist/src/agent.js.map +1 -1
  5. package/dist/src/framework.d.ts +235 -2
  6. package/dist/src/framework.d.ts.map +1 -1
  7. package/dist/src/framework.js +1141 -92
  8. package/dist/src/framework.js.map +1 -1
  9. package/dist/src/gate/event-gate.d.ts +20 -1
  10. package/dist/src/gate/event-gate.d.ts.map +1 -1
  11. package/dist/src/gate/event-gate.js +60 -4
  12. package/dist/src/gate/event-gate.js.map +1 -1
  13. package/dist/src/index.d.ts +3 -0
  14. package/dist/src/index.d.ts.map +1 -1
  15. package/dist/src/index.js +4 -0
  16. package/dist/src/index.js.map +1 -1
  17. package/dist/src/mcpl/channel-registry.d.ts +17 -1
  18. package/dist/src/mcpl/channel-registry.d.ts.map +1 -1
  19. package/dist/src/mcpl/channel-registry.js +34 -3
  20. package/dist/src/mcpl/channel-registry.js.map +1 -1
  21. package/dist/src/mcpl/push-handler.d.ts.map +1 -1
  22. package/dist/src/mcpl/push-handler.js +4 -0
  23. package/dist/src/mcpl/push-handler.js.map +1 -1
  24. package/dist/src/mcpl/server-connection.d.ts +29 -0
  25. package/dist/src/mcpl/server-connection.d.ts.map +1 -1
  26. package/dist/src/mcpl/server-connection.js +102 -4
  27. package/dist/src/mcpl/server-connection.js.map +1 -1
  28. package/dist/src/mcpl/types.d.ts +17 -1
  29. package/dist/src/mcpl/types.d.ts.map +1 -1
  30. package/dist/src/mcpl/types.js.map +1 -1
  31. package/dist/src/module-registry.d.ts +15 -1
  32. package/dist/src/module-registry.d.ts.map +1 -1
  33. package/dist/src/module-registry.js +92 -4
  34. package/dist/src/module-registry.js.map +1 -1
  35. package/dist/src/types/agent.d.ts +37 -0
  36. package/dist/src/types/agent.d.ts.map +1 -1
  37. package/dist/src/types/framework.d.ts +40 -0
  38. package/dist/src/types/framework.d.ts.map +1 -1
  39. package/dist/src/types/module.d.ts +46 -0
  40. package/dist/src/types/module.d.ts.map +1 -1
  41. package/dist/src/types/trace.d.ts +51 -0
  42. package/dist/src/types/trace.d.ts.map +1 -1
  43. package/dist/test/agent-cache-ttl.test.d.ts +2 -0
  44. package/dist/test/agent-cache-ttl.test.d.ts.map +1 -0
  45. package/dist/test/agent-cache-ttl.test.js +25 -0
  46. package/dist/test/agent-cache-ttl.test.js.map +1 -0
  47. package/dist/test/conversation-fork-injection-scoping.test.d.ts +2 -0
  48. package/dist/test/conversation-fork-injection-scoping.test.d.ts.map +1 -0
  49. package/dist/test/conversation-fork-injection-scoping.test.js +184 -0
  50. package/dist/test/conversation-fork-injection-scoping.test.js.map +1 -0
  51. package/dist/test/gather-context-timeout.test.d.ts +2 -0
  52. package/dist/test/gather-context-timeout.test.d.ts.map +1 -0
  53. package/dist/test/gather-context-timeout.test.js +120 -0
  54. package/dist/test/gather-context-timeout.test.js.map +1 -0
  55. package/dist/test/inference-exhaustion-rewind.test.d.ts +2 -0
  56. package/dist/test/inference-exhaustion-rewind.test.d.ts.map +1 -0
  57. package/dist/test/inference-exhaustion-rewind.test.js +181 -0
  58. package/dist/test/inference-exhaustion-rewind.test.js.map +1 -0
  59. package/dist/test/inference-failure-observability.test.js +9 -0
  60. package/dist/test/inference-failure-observability.test.js.map +1 -1
  61. package/dist/test/maintenance-timer.test.d.ts +2 -0
  62. package/dist/test/maintenance-timer.test.d.ts.map +1 -0
  63. package/dist/test/maintenance-timer.test.js +114 -0
  64. package/dist/test/maintenance-timer.test.js.map +1 -0
  65. package/dist/test/mcpl-reconnect-events.test.d.ts +2 -0
  66. package/dist/test/mcpl-reconnect-events.test.d.ts.map +1 -0
  67. package/dist/test/mcpl-reconnect-events.test.js +119 -0
  68. package/dist/test/mcpl-reconnect-events.test.js.map +1 -0
  69. package/dist/test/mcpl-reconnect-reregistration.test.d.ts +2 -0
  70. package/dist/test/mcpl-reconnect-reregistration.test.d.ts.map +1 -0
  71. package/dist/test/mcpl-reconnect-reregistration.test.js +181 -0
  72. package/dist/test/mcpl-reconnect-reregistration.test.js.map +1 -0
  73. package/dist/test/mcpl-request-timeout.test.d.ts +2 -0
  74. package/dist/test/mcpl-request-timeout.test.d.ts.map +1 -0
  75. package/dist/test/mcpl-request-timeout.test.js +123 -0
  76. package/dist/test/mcpl-request-timeout.test.js.map +1 -0
  77. package/dist/test/mcpl-runtime-lifecycle.test.d.ts +7 -0
  78. package/dist/test/mcpl-runtime-lifecycle.test.d.ts.map +1 -0
  79. package/dist/test/mcpl-runtime-lifecycle.test.js +142 -0
  80. package/dist/test/mcpl-runtime-lifecycle.test.js.map +1 -0
  81. package/dist/test/ops-alert.test.d.ts +2 -0
  82. package/dist/test/ops-alert.test.d.ts.map +1 -0
  83. package/dist/test/ops-alert.test.js +152 -0
  84. package/dist/test/ops-alert.test.js.map +1 -0
  85. package/dist/test/refusal-rewind.test.d.ts +2 -0
  86. package/dist/test/refusal-rewind.test.d.ts.map +1 -0
  87. package/dist/test/refusal-rewind.test.js +108 -0
  88. package/dist/test/refusal-rewind.test.js.map +1 -0
  89. package/dist/test/state-scaling-e2e.test.d.ts +22 -0
  90. package/dist/test/state-scaling-e2e.test.d.ts.map +1 -0
  91. package/dist/test/state-scaling-e2e.test.js +200 -0
  92. package/dist/test/state-scaling-e2e.test.js.map +1 -0
  93. package/dist/test/state-scaling.test.d.ts +12 -0
  94. package/dist/test/state-scaling.test.d.ts.map +1 -0
  95. package/dist/test/state-scaling.test.js +243 -0
  96. package/dist/test/state-scaling.test.js.map +1 -0
  97. package/dist/test/tool-result-content.test.d.ts +2 -0
  98. package/dist/test/tool-result-content.test.d.ts.map +1 -0
  99. package/dist/test/tool-result-content.test.js +37 -0
  100. package/dist/test/tool-result-content.test.js.map +1 -0
  101. package/dist/tsconfig.tsbuildinfo +1 -1
  102. package/package.json +3 -2
  103. package/src/agent.ts +16 -0
  104. package/src/framework.ts +1339 -99
  105. package/src/gate/event-gate.ts +56 -4
  106. package/src/index.ts +10 -0
  107. package/src/mcpl/channel-registry.ts +34 -3
  108. package/src/mcpl/push-handler.ts +4 -0
  109. package/src/mcpl/server-connection.ts +110 -4
  110. package/src/mcpl/types.ts +19 -1
  111. package/src/module-registry.ts +113 -4
  112. package/src/types/agent.ts +38 -0
  113. package/src/types/framework.ts +44 -0
  114. package/src/types/module.ts +52 -0
  115. package/src/types/trace.ts +67 -0
@@ -1,10 +1,11 @@
1
1
  import { join } from 'node:path';
2
+ import { appendFileSync, mkdirSync } from 'node:fs';
2
3
  import { JsStore } from '@animalabs/chronicle';
3
4
  import { MembraneError } from '@animalabs/membrane';
4
5
  import { ContextManager, PassthroughStrategy } from '@animalabs/context-manager';
5
6
  import { ProcessQueueImpl } from './queue.js';
6
7
  import { Agent } from './agent.js';
7
- import { ModuleRegistry } from './module-registry.js';
8
+ import { ModuleRegistry, isStateExistsError } from './module-registry.js';
8
9
  import { McplServerRegistry } from './mcpl/server-registry.js';
9
10
  import { FeatureSetManager } from './mcpl/feature-set-manager.js';
10
11
  import { ScopeManager } from './mcpl/scope-manager.js';
@@ -24,7 +25,8 @@ const FRAMEWORK_STATE_ID = 'framework/state';
24
25
  const CONVERSATION_ROUTER_STATE_ID = 'framework/conversation-router';
25
26
  const INFERENCE_LOG_ID = 'framework/inference-log';
26
27
  const PROCESS_LOG_ID = 'framework/process-log';
27
- const TURN_CHECKPOINTS_ID = 'framework/turn-checkpoints';
28
+ const TURN_CHECKPOINTS_ID = 'framework/turn-checkpoints'; // legacy single-map layout, read-only fallback
29
+ const TURN_CHECKPOINTS_TREE_ID = 'framework/turn-checkpoints/tree';
28
30
  /** Maximum number of turn checkpoints to keep per agent. */
29
31
  const MAX_TURN_CHECKPOINTS = 20;
30
32
  /**
@@ -64,6 +66,10 @@ class DefaultErrorPolicy {
64
66
  }
65
67
  /** Default sync interval in milliseconds */
66
68
  const DEFAULT_SYNC_INTERVAL_MS = 1000;
69
+ /** Poll pending context-strategy maintenance independently of user activity. */
70
+ const DEFAULT_MAINTENANCE_INTERVAL_MS = 5000;
71
+ /** Bound one pass so a large backlog yields to inference and other agents. */
72
+ const MAINTENANCE_TICKS_PER_PASS = 8;
67
73
  /**
68
74
  * Extract fields the EventGate cares about from a ProcessEvent. The set of
69
75
  * event variants that carry `content`/`mount`/`paths`/`metadata` is open-ended
@@ -95,6 +101,12 @@ function extractGateFields(event) {
95
101
  : undefined;
96
102
  return { content, metadata, mount, paths };
97
103
  }
104
+ /** Cap an API error message for inline use in a rewind marker: keep enough to
105
+ * identify the failure class without pasting a wall of provider JSON into the
106
+ * agent's context. */
107
+ function truncateReason(reason, max = 160) {
108
+ return reason.length <= max ? reason : reason.slice(0, max) + '…';
109
+ }
98
110
  export class AgentFramework {
99
111
  store;
100
112
  ownsStore;
@@ -120,6 +132,16 @@ export class AgentFramework {
120
132
  traceListeners = [];
121
133
  syncIntervalMs;
122
134
  syncTimer = null;
135
+ maintenanceIntervalMs;
136
+ maintenanceTimer = null;
137
+ maintenancePass = null;
138
+ maintenanceRunId = 0;
139
+ currentMaintenanceRun = null;
140
+ maintenanceHistory = [];
141
+ /** Last time we console-warned about stale (busy-requeued) inference requests, per agent. */
142
+ staleWarnAt = new Map();
143
+ /** Per-agent last inference activity (epoch ms), for /healthz + doctor tooling. */
144
+ lastInferenceAt = new Map();
123
145
  processLoggingPersist;
124
146
  processLoggingBroadcast;
125
147
  activeStreams = new Map();
@@ -130,6 +152,36 @@ export class AgentFramework {
130
152
  /** N consecutive failed inferences ⇒ the agent is treated as hard-down and
131
153
  * escalated loudly to stderr. */
132
154
  inferenceFailureEscalationThreshold = 3;
155
+ /** Per-(agent,kind) timestamp of the last ops webhook post. Throttles
156
+ * opsAlert() so a persistent failure re-posts once per cooldown window
157
+ * instead of on every occurrence. */
158
+ opsAlertLastSent = new Map();
159
+ /** Cooldown between webhook posts for the same (agent, kind). */
160
+ opsAlertCooldownMs = 15 * 60_000;
161
+ /** Per-agent refusal bookkeeping for observability — exposed via
162
+ * healthSnapshot() and mirrored to failures.log / ops alerts. */
163
+ refusalStats = new Map();
164
+ /** Per-agent count of consecutive refusals (reset on any non-refusal
165
+ * completion). Distinct from refusalRewinds, which budgets the auto-rewind
166
+ * loop — this one drives ops alerting. */
167
+ refusalStreak = new Map();
168
+ /** Per-agent count of consecutive refusal-driven rewinds in the current turn
169
+ * chain (reset when a turn completes without a refusal). Bounds the auto
170
+ * rewind loop — see refusalHandling + rewindTriggeringTurn. */
171
+ refusalRewinds = new Map();
172
+ /** Per-agent count of poison-history rewinds performed by the hard-down
173
+ * breaker (noteInferenceExhausted). Reset on any successful inference.
174
+ * Bounds the automatic quarantine loop the same way refusalRewinds bounds
175
+ * the refusal loop, so the breaker can never shed the whole history. */
176
+ exhaustionRewinds = new Map();
177
+ /** Per-agent current rewind episode: the single consolidated marker's id and
178
+ * how many turns have been shed so far. One marker per episode, updated in
179
+ * place (see updateRewindMarker); cleared when the episode ends. */
180
+ rewindEpisode = new Map();
181
+ /** Active `/unstick` sessions: an admin-forced rewind-until-clean loop that
182
+ * runs even when the agent's autoRewind toggle is off. Tracks the remaining
183
+ * budget, what was shed (for the report), and where to post the outcome. */
184
+ forcedRewind = new Map();
133
185
  /** Name of the primary (non-ephemeral) agent for routing framework messages. */
134
186
  primaryAgentName = null;
135
187
  // Messages deferred while an agent is waiting_for_tools (to preserve
@@ -171,17 +223,25 @@ export class AgentFramework {
171
223
  mcplPrefixMap = new Map();
172
224
  /** Maps serverId → McplServerConfig for prefix lookup. */
173
225
  mcplServerConfigs = new Map();
226
+ /** Host capabilities advertised during the MCP handshake — stored so servers
227
+ * can be connected at runtime (connectMcplServer) after initialization. */
228
+ mcplHostCapabilities = null;
229
+ /** Inference routing policy from FrameworkConfig — stored so the MCPL
230
+ * subsystem can be lazily initialized by connectMcplServer when the
231
+ * framework started with zero configured servers. */
232
+ mcplInferenceRoutingConfig = null;
174
233
  // EventGate (null when FrameworkConfig.gate is omitted)
175
234
  eventGate = null;
176
235
  // Session-level token usage tracking (always-on)
177
236
  usageTracker;
178
- constructor(store, ownsStore, membrane, inferencePolicy, errorPolicy, syncIntervalMs, processLoggingPersist, processLoggingBroadcast) {
237
+ constructor(store, ownsStore, membrane, inferencePolicy, errorPolicy, syncIntervalMs, maintenanceIntervalMs, processLoggingPersist, processLoggingBroadcast) {
179
238
  this.store = store;
180
239
  this.ownsStore = ownsStore;
181
240
  this.membrane = membrane;
182
241
  this.inferencePolicy = inferencePolicy;
183
242
  this.errorPolicy = errorPolicy;
184
243
  this.syncIntervalMs = syncIntervalMs;
244
+ this.maintenanceIntervalMs = maintenanceIntervalMs;
185
245
  this.processLoggingPersist = processLoggingPersist;
186
246
  this.processLoggingBroadcast = processLoggingBroadcast;
187
247
  this.queue = new ProcessQueueImpl();
@@ -237,11 +297,14 @@ export class AgentFramework {
237
297
  catch {
238
298
  // Already registered
239
299
  }
300
+ // The legacy single-map checkpoint state (TURN_CHECKPOINTS_ID) is no longer
301
+ // registered for new stores — it's read-only fallback data in old ones.
240
302
  try {
241
- store.registerState({ id: TURN_CHECKPOINTS_ID, strategy: 'snapshot' });
303
+ store.registerState({ id: TURN_CHECKPOINTS_TREE_ID, strategy: 'tree' });
242
304
  }
243
- catch {
244
- // Already registered
305
+ catch (error) {
306
+ if (!isStateExistsError(error))
307
+ throw error;
245
308
  }
246
309
  // Process logging config (default: disabled)
247
310
  const processLoggingPersist = config.processLogging?.persist ?? false;
@@ -260,7 +323,7 @@ export class AgentFramework {
260
323
  // Already registered
261
324
  }
262
325
  }
263
- const framework = new AgentFramework(store, ownsStore, config.membrane, config.inferencePolicy ?? new DefaultInferencePolicy(), config.errorPolicy ?? new DefaultErrorPolicy(), config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS, processLoggingPersist, processLoggingBroadcast);
326
+ const framework = new AgentFramework(store, ownsStore, config.membrane, config.inferencePolicy ?? new DefaultInferencePolicy(), config.errorPolicy ?? new DefaultErrorPolicy(), config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS, config.maintenanceIntervalMs ?? DEFAULT_MAINTENANCE_INTERVAL_MS, processLoggingPersist, processLoggingBroadcast);
264
327
  // Restore persisted usage data (if any) from prior session
265
328
  framework.restoreUsageState();
266
329
  // Create agents
@@ -327,6 +390,9 @@ export class AgentFramework {
327
390
  });
328
391
  framework.livenessWatchdog.start();
329
392
  }
393
+ // Stored for lazy MCPL initialization (connectMcplServer on a framework
394
+ // that started with zero configured servers).
395
+ framework.mcplInferenceRoutingConfig = config.inferenceRouting ?? null;
330
396
  // Initialize MCPL subsystems if configured
331
397
  if (config.mcplServers && config.mcplServers.length > 0) {
332
398
  // Validate tool prefixes: no collisions with module names or between servers
@@ -345,6 +411,30 @@ export class AgentFramework {
345
411
  }
346
412
  await framework.initializeMcpl(config.mcplServers, config.inferenceRouting);
347
413
  }
414
+ // Diagnostics: `kill -USR2 <pid>` dumps live wake/inference state to stderr
415
+ // (journal) without a restart — for catching the wake-wedge on the running
416
+ // process. Shows the gate's `inferring` set + buffered-event count (the
417
+ // wedge signature), active streams, and pending inference requests.
418
+ try {
419
+ process.on('SIGUSR2', () => {
420
+ try {
421
+ const gate = framework.eventGate?.inferenceDiagnostics() ?? null;
422
+ console.error('[diagnostics] ' + JSON.stringify({
423
+ at: new Date().toISOString(),
424
+ gate,
425
+ activeStreams: [...framework.activeStreams.keys()],
426
+ pendingRequests: framework.pendingRequests.length,
427
+ agents: [...framework.agents.keys()],
428
+ }));
429
+ }
430
+ catch (err) {
431
+ console.error('[diagnostics] dump failed:', err instanceof Error ? err.message : err);
432
+ }
433
+ });
434
+ }
435
+ catch {
436
+ // SIGUSR2 not available on this platform — non-fatal.
437
+ }
348
438
  return framework;
349
439
  }
350
440
  /**
@@ -367,6 +457,15 @@ export class AgentFramework {
367
457
  }
368
458
  }, this.syncIntervalMs);
369
459
  }
460
+ if (this.maintenanceIntervalMs > 0) {
461
+ this.maintenanceTimer = setInterval(() => {
462
+ this.startQueuedMaintenance();
463
+ }, this.maintenanceIntervalMs);
464
+ this.maintenanceTimer.unref?.();
465
+ // Do not make a restored queue wait a full interval before its first
466
+ // attempt. MCPL/module initialization has completed before start().
467
+ this.startQueuedMaintenance();
468
+ }
370
469
  }
371
470
  /**
372
471
  * Stop the event loop.
@@ -390,6 +489,13 @@ export class AgentFramework {
390
489
  clearInterval(this.syncTimer);
391
490
  this.syncTimer = null;
392
491
  }
492
+ if (this.maintenanceTimer) {
493
+ clearInterval(this.maintenanceTimer);
494
+ this.maintenanceTimer = null;
495
+ }
496
+ if (this.maintenancePass) {
497
+ await this.maintenancePass;
498
+ }
393
499
  if (this.loopPromise) {
394
500
  await this.loopPromise;
395
501
  }
@@ -415,6 +521,100 @@ export class AgentFramework {
415
521
  this.store.close();
416
522
  }
417
523
  }
524
+ /**
525
+ * Advance queued ContextManager work without requiring a new message.
526
+ *
527
+ * Tool definitions are refreshed first because compression replays stored
528
+ * tool cycles and providers reject those requests when the corresponding
529
+ * schemas are absent. Passes never overlap; each agent gets a bounded drain
530
+ * so maintenance cannot monopolize the framework event loop.
531
+ */
532
+ startQueuedMaintenance() {
533
+ if (this.maintenancePass || !this.running)
534
+ return;
535
+ const pass = this.runQueuedMaintenance().catch((error) => {
536
+ console.error('[context-maintenance] pass failed:', error instanceof Error ? error.message : error);
537
+ });
538
+ this.maintenancePass = pass;
539
+ void pass.finally(() => {
540
+ if (this.maintenancePass === pass)
541
+ this.maintenancePass = null;
542
+ });
543
+ }
544
+ async runQueuedMaintenance() {
545
+ const allTools = this.getAllTools();
546
+ const queued = [...this.agents.values()].flatMap((agent) => {
547
+ const cm = agent.getContextManager();
548
+ const tools = allTools.filter((tool) => agent.canUseTool(tool.name));
549
+ cm.setToolDefinitions(tools);
550
+ if (cm.isReady())
551
+ return [];
552
+ const pending = cm.getPendingWork()?.description;
553
+ const progress = this.contextProgress(cm);
554
+ const record = {
555
+ agentName: agent.name,
556
+ startedAt: Date.now(),
557
+ ticks: 0,
558
+ readyBefore: false,
559
+ ...(pending ? { pendingBefore: pending } : {}),
560
+ ...(progress ? { progressBefore: progress } : {}),
561
+ };
562
+ return [{
563
+ agent,
564
+ cm,
565
+ record,
566
+ }];
567
+ });
568
+ if (queued.length === 0)
569
+ return;
570
+ const run = {
571
+ id: ++this.maintenanceRunId,
572
+ startedAt: Date.now(),
573
+ agents: queued.map(item => item.record),
574
+ };
575
+ this.currentMaintenanceRun = run;
576
+ try {
577
+ await Promise.all(queued.map(async ({ agent, cm, record }) => {
578
+ try {
579
+ for (let i = 0; i < MAINTENANCE_TICKS_PER_PASS && this.running && !cm.isReady(); i++) {
580
+ await cm.tick();
581
+ record.ticks++;
582
+ }
583
+ }
584
+ catch (error) {
585
+ record.error = error instanceof Error ? error.message : String(error);
586
+ console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
587
+ }
588
+ finally {
589
+ const pending = cm.getPendingWork()?.description;
590
+ record.finishedAt = Date.now();
591
+ record.readyAfter = cm.isReady();
592
+ if (pending)
593
+ record.pendingAfter = pending;
594
+ const progress = this.contextProgress(cm);
595
+ if (progress)
596
+ record.progressAfter = progress;
597
+ }
598
+ }));
599
+ }
600
+ finally {
601
+ run.finishedAt = Date.now();
602
+ run.agents.sort((a, b) => a.agentName.localeCompare(b.agentName));
603
+ this.currentMaintenanceRun = null;
604
+ this.maintenanceHistory.unshift(run);
605
+ if (this.maintenanceHistory.length > 50)
606
+ this.maintenanceHistory.length = 50;
607
+ }
608
+ }
609
+ contextProgress(cm) {
610
+ const strategy = cm.getStrategy();
611
+ if (typeof strategy.getProgressSnapshot !== 'function')
612
+ return undefined;
613
+ const progress = strategy.getProgressSnapshot();
614
+ return progress && typeof progress === 'object'
615
+ ? progress
616
+ : undefined;
617
+ }
418
618
  /**
419
619
  * Push a process event to the queue.
420
620
  */
@@ -526,6 +726,27 @@ export class AgentFramework {
526
726
  }
527
727
  return [...moduleTools, ...this.mcplTools, ...channelTools, ...gateTools];
528
728
  }
729
+ /** Counts-only context-maintenance diagnostics for authenticated debug UIs. */
730
+ getContextMaintenanceSnapshot() {
731
+ const agents = [...this.agents.values()].map((agent) => {
732
+ const cm = agent.getContextManager();
733
+ const pending = cm.getPendingWork()?.description;
734
+ const progress = this.contextProgress(cm);
735
+ return {
736
+ agentName: agent.name,
737
+ ready: cm.isReady(),
738
+ ...(pending ? { pending } : {}),
739
+ ...(progress ? { progress } : {}),
740
+ };
741
+ });
742
+ return structuredClone({
743
+ intervalMs: this.maintenanceIntervalMs,
744
+ ticksPerPass: MAINTENANCE_TICKS_PER_PASS,
745
+ current: this.currentMaintenanceRun,
746
+ history: this.maintenanceHistory,
747
+ agents,
748
+ });
749
+ }
529
750
  /**
530
751
  * Build the membrane-normalized request that WOULD be emitted if `agentName`
531
752
  * were activated right now — WITHOUT running inference, opening a stream, or
@@ -568,8 +789,9 @@ export class AgentFramework {
568
789
  // Module gatherContext (fail-open, matches startAgentStream)
569
790
  try {
570
791
  const moduleInjections = await this.moduleRegistry.gatherContext(agentName);
571
- if (moduleInjections.length > 0) {
572
- injections = moduleInjections;
792
+ const scopedModuleInjections = this.scopeInjectionsForAgent(agentName, moduleInjections);
793
+ if (scopedModuleInjections.length > 0) {
794
+ injections = scopedModuleInjections;
573
795
  }
574
796
  }
575
797
  catch (error) {
@@ -580,7 +802,7 @@ export class AgentFramework {
580
802
  if (this.hookOrchestrator) {
581
803
  try {
582
804
  const hookParams = this.buildBeforeInferenceParams(agent);
583
- const hookInjections = await this.hookOrchestrator.beforeInference(hookParams);
805
+ const hookInjections = this.scopeInjectionsForAgent(agentName, await this.hookOrchestrator.beforeInference(hookParams));
584
806
  if (hookInjections.length > 0) {
585
807
  injections = injections ? [...injections, ...hookInjections] : hookInjections;
586
808
  }
@@ -743,6 +965,7 @@ export class AgentFramework {
743
965
  clearInterval(completionWatchdog);
744
966
  this.offTrace(traceListener);
745
967
  this.agents.delete(agent.name);
968
+ this.evictTurnCheckpoints(agent.name);
746
969
  };
747
970
  // Watchdog: if inference never starts within 30s, the agent is a zombie.
748
971
  // This catches cases where the event loop stalled, the agent was
@@ -1080,6 +1303,181 @@ export class AgentFramework {
1080
1303
  console.error('[inference-refusal] reaction failed:', err instanceof Error ? err.message : err);
1081
1304
  }
1082
1305
  }
1306
+ /**
1307
+ * Rewind the single turn that fed a refused generation: redact the newest
1308
+ * message the agent did NOT author (a tool result, or an incoming message)
1309
+ * and inject a metadata-only marker in its place. The marker carries a
1310
+ * *description* of what was withheld (kind + size), never the content — so it
1311
+ * cannot itself re-trip the classifier — while the raw record survives in the
1312
+ * chronicle log for forensics. Returns a record describing the removal, or
1313
+ * null if there is nothing eligible to rewind (e.g. only the agent's own
1314
+ * turns, or a prior system marker, remain).
1315
+ *
1316
+ * Shared by the auto path (on `stop_reason: refusal`) and the `/rewind`
1317
+ * host command.
1318
+ */
1319
+ shedNewestTurn(agent) {
1320
+ const cm = agent.getContextManager();
1321
+ const all = cm.getAllMessages();
1322
+ const typeOf = (b) => b.type;
1323
+ const hasBlock = (m, t) => Array.isArray(m?.content) && m.content.some((b) => typeOf(b) === t);
1324
+ // Newest message that is not our own episode marker. We shed strictly
1325
+ // newest-first, in sequence — INCLUDING the agent's own turns (poison lives
1326
+ // in tool_use/narration turns too), because newest-first reaches whatever is
1327
+ // poisoning the context if allowed to run deep enough. The one invariant:
1328
+ // shed COMPLETE exchanges. Removing a `tool_result` also removes its paired
1329
+ // `tool_use` assistant turn, so we never leave an orphaned tool_use / signed
1330
+ // `thinking` block — which the API rejects with a 400 ("thinking blocks in
1331
+ // the latest assistant message cannot be modified"). We do NOT add a marker
1332
+ // here; the caller keeps one consolidated marker (updateRewindMarker).
1333
+ let idx = -1;
1334
+ for (let i = all.length - 1; i >= 0; i--) {
1335
+ const meta = (all[i].metadata ?? {});
1336
+ if (meta.system)
1337
+ continue;
1338
+ idx = i;
1339
+ break;
1340
+ }
1341
+ if (idx < 0)
1342
+ return null;
1343
+ const msg = all[idx];
1344
+ const md = (msg.metadata ?? {});
1345
+ const content = Array.isArray(msg.content) ? msg.content : [];
1346
+ const toolBlocks = content.filter((b) => typeOf(b) === 'tool_result');
1347
+ const images = content.filter((b) => typeOf(b) === 'image').length;
1348
+ const textLen = content
1349
+ .filter((b) => typeOf(b) === 'text')
1350
+ .reduce((n, b) => n + String(b.text ?? '').length, 0);
1351
+ // Determine the complete exchange to remove.
1352
+ const removedIds = [msg.id];
1353
+ let kind;
1354
+ let descriptor;
1355
+ if (toolBlocks.length > 0) {
1356
+ kind = 'tool';
1357
+ // Pair the tool_result with the tool_use assistant turn right before it.
1358
+ if (idx - 1 >= 0 && hasBlock(all[idx - 1], 'tool_use')) {
1359
+ removedIds.push(all[idx - 1].id);
1360
+ }
1361
+ const sz = toolBlocks.reduce((n, b) => n + String(b.content ?? '').length, 0);
1362
+ descriptor = `a tool exchange (~${Math.max(1, Math.round(sz / 1024))}KB` +
1363
+ `${images ? `, ${images} image(s)` : ''})`;
1364
+ }
1365
+ else if (md.messageId) {
1366
+ kind = 'human';
1367
+ descriptor = `an incoming message from ${msg.participant} ` +
1368
+ `(${textLen} chars${images ? `, ${images} image(s)` : ''})`;
1369
+ }
1370
+ else {
1371
+ kind = 'other';
1372
+ descriptor = `a ${msg.participant} turn ` +
1373
+ `(${textLen} chars${images ? `, ${images} image(s)` : ''})`;
1374
+ }
1375
+ const discordRef = md.messageId && md.channelId
1376
+ ? { channelId: String(md.channelId), messageId: String(md.messageId) }
1377
+ : undefined;
1378
+ for (const id of removedIds)
1379
+ cm.removeMessage(id);
1380
+ return { kind, descriptor, removedIds, discordRef };
1381
+ }
1382
+ /**
1383
+ * Maintain exactly ONE consolidated marker for the current rewind episode,
1384
+ * updated in place as more turns are shed. Six rewinds ⇒ one message that
1385
+ * says "the 6 most recent turns were set aside", not six separate notes — so
1386
+ * the context converges (shed N, add 1) instead of growing, and the marker
1387
+ * sits at the tail giving the model something actionable to answer once the
1388
+ * refusal clears. Returns the running count.
1389
+ */
1390
+ updateRewindMarker(agent, category, cause = 'refusal') {
1391
+ const cm = agent.getContextManager();
1392
+ const ep = this.rewindEpisode.get(agent.name);
1393
+ const count = (ep?.count ?? 0) + 1;
1394
+ const why = cause === 'refusal'
1395
+ ? `the model refused on them (content filter: ${category}). Their content is ` +
1396
+ `not reproduced here (so this note can't re-trigger the filter); the ` +
1397
+ `originals remain in the raw record`
1398
+ : `the model API kept rejecting the conversation on them (${category}). ` +
1399
+ `Their content is not reproduced here (so this note can't re-trigger the ` +
1400
+ `rejection); the originals remain in the raw record`;
1401
+ const text = `[refusal-rewind] The ${count} most recent turn(s) were set aside because ` +
1402
+ `${why}. You are clear to continue — if you ` +
1403
+ `were mid-task, take a different approach; otherwise carry on with whatever ` +
1404
+ `is now in front of you, or briefly acknowledge the gap and ask what's next.`;
1405
+ const blocks = [{ type: 'text', text }];
1406
+ if (ep) {
1407
+ cm.editMessage(ep.markerId, blocks);
1408
+ this.rewindEpisode.set(agent.name, { markerId: ep.markerId, count, category });
1409
+ }
1410
+ else {
1411
+ const id = cm.addMessage('user', blocks, {
1412
+ system: true, kind: 'refusal-rewind', category, count, cause,
1413
+ });
1414
+ this.rewindEpisode.set(agent.name, { markerId: id, count, category });
1415
+ }
1416
+ return count;
1417
+ }
1418
+ /**
1419
+ * Announce a refusal-rewind on the conversational surface (Discord), used
1420
+ * when the withheld turn was a *human* message so it isn't dropped silently.
1421
+ * Best-effort; mirrors reactToRefusal's locus resolution.
1422
+ */
1423
+ async announceRewind(agentName, rec, category) {
1424
+ try {
1425
+ const incoming = this.channelRegistry?.buildChannelContext()?.incoming;
1426
+ if (!incoming)
1427
+ return;
1428
+ const parts = incoming.channelId.split(':');
1429
+ if (parts[0] !== 'discord')
1430
+ return;
1431
+ const channelId = parts[parts.length - 1];
1432
+ const serverId = this.channelRegistry?.getChannelServerId(incoming.channelId);
1433
+ const server = serverId ? this.mcplServerRegistry?.getServer(serverId) : null;
1434
+ if (!server)
1435
+ return;
1436
+ await server.sendToolsCall('send_message', {
1437
+ channelId,
1438
+ content: `⚠️ I had to set aside ${rec.descriptor} — it tripped a content ` +
1439
+ `filter (${category}), so it's withheld from my context and I'm ` +
1440
+ `continuing without it. If it was important, please rephrase or re-send.`,
1441
+ });
1442
+ }
1443
+ catch (err) {
1444
+ console.error('[refusal-rewind] announce failed:', err instanceof Error ? err.message : err);
1445
+ }
1446
+ }
1447
+ /**
1448
+ * Conclude an active `/unstick` session: post the outcome (what was shed and
1449
+ * whether the model stopped refusing) to the channel the command came from,
1450
+ * then clear the session. Idempotent — a no-op if there's no session.
1451
+ */
1452
+ finishUnstick(agentName, success, category) {
1453
+ const s = this.forcedRewind.get(agentName);
1454
+ if (!s)
1455
+ return;
1456
+ this.forcedRewind.delete(agentName);
1457
+ const n = s.removed.length;
1458
+ const list = n
1459
+ ? '\n' + s.removed.map((r) => `• ${r.descriptor}`).join('\n')
1460
+ : '';
1461
+ const content = success
1462
+ ? `🔧 Unstuck **${agentName}** — shed ${n} turn(s); the model responded.${list}`
1463
+ : `⚠️ Couldn't unstick **${agentName}** — still refusing after ${n} rewind(s)` +
1464
+ `${category ? ` (category=${category})` : ''}.${list}`;
1465
+ console.error(`[unstick] agent=${agentName} ${success ? 'succeeded' : 'gave up'} after ${n} rewind(s)`);
1466
+ if (s.channelId)
1467
+ void this.postToChannel(s.serverId, s.channelId, content);
1468
+ }
1469
+ /** Best-effort send_message to a raw channel via a named MCPL server. */
1470
+ async postToChannel(serverId, channelId, content) {
1471
+ try {
1472
+ const server = this.mcplServerRegistry?.getServer(serverId);
1473
+ if (!server)
1474
+ return;
1475
+ await server.sendToolsCall('send_message', { channelId, content });
1476
+ }
1477
+ catch (err) {
1478
+ console.error('[unstick] report post failed:', err instanceof Error ? err.message : err);
1479
+ }
1480
+ }
1083
1481
  /**
1084
1482
  * Handle a `host/command` request from an MCPL surface server (e.g. a
1085
1483
  * Discord slash command). Currently supports:
@@ -1090,13 +1488,41 @@ export class AgentFramework {
1090
1488
  * `previewActivation` render (no inference, no Chronicle writes).
1091
1489
  */
1092
1490
  async handleHostCommand(serverId, params) {
1093
- if (params.command !== 'undo' && params.command !== 'hide') {
1491
+ if (params.command !== 'undo' && params.command !== 'hide' && params.command !== 'unstick') {
1094
1492
  return { ok: false, error: `Unknown host command: ${String(params.command)}` };
1095
1493
  }
1096
1494
  const agentName = params.agentName ?? [...this.agents.keys()][0];
1097
1495
  if (!agentName || !this.agents.has(agentName)) {
1098
1496
  return { ok: false, error: `Unknown agent: ${String(agentName)}` };
1099
1497
  }
1498
+ // unstick: force the refusal-rewind loop on demand (even if the agent's
1499
+ // autoRewind toggle is off). Redacts the turn that fed the refusal and
1500
+ // re-runs, up to `cap` times, until the model stops refusing. Kicks the
1501
+ // loop and returns immediately; the outcome (what was shed + whether it
1502
+ // cleared) is posted to the channel when the chain resolves.
1503
+ if (params.command === 'unstick') {
1504
+ const agent = this.agents.get(agentName);
1505
+ if (agent.state.status !== 'idle') {
1506
+ return { ok: false, error: `Cannot unstick while agent is ${agent.state.status}` };
1507
+ }
1508
+ const cap = Math.max(1, Math.min(10, Math.floor(params.maxRewinds ?? agent.refusalHandling?.maxRewinds ?? 3)));
1509
+ this.forcedRewind.set(agentName, {
1510
+ remaining: cap,
1511
+ removed: [],
1512
+ serverId,
1513
+ channelId: params.channelId ?? '',
1514
+ });
1515
+ this.refusalRewinds.set(agentName, 0);
1516
+ this.pendingRequests.push({
1517
+ agentName,
1518
+ reason: 'unstick',
1519
+ source: 'framework',
1520
+ timestamp: Date.now(),
1521
+ });
1522
+ console.error(`[unstick] agent=${agentName} started cap=${cap} ` +
1523
+ `by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`);
1524
+ return { ok: true, started: true, cap };
1525
+ }
1100
1526
  // hide: redact a single message (or an inclusive range) from the active
1101
1527
  // branch, addressed by Discord message id. Unlike undo this is a
1102
1528
  // removal-in-place (chronicle redact), not a branch rewind.
@@ -1615,18 +2041,86 @@ export class AgentFramework {
1615
2041
  }
1616
2042
  this.saveTurnCheckpoints(agentName, checkpoints);
1617
2043
  }
2044
+ /**
2045
+ * Checkpoints live in one Tree state (TURN_CHECKPOINTS_TREE_ID) keyed by agent
2046
+ * name, each key pointing at a JSON blob of that agent's ≤MAX_TURN_CHECKPOINTS
2047
+ * list. This kills two unbounded-in-agents-ever dimensions at once:
2048
+ *
2049
+ * - the original single Record<agentName, list> map was rewritten whole on
2050
+ * every turn of every agent and never evicted a departed subagent's key;
2051
+ * - the intermediate design (one state slot per agent) fixed the writes but
2052
+ * leaked a permanent chronicle state *registration* per spawn — chronicle
2053
+ * has no deregistration, and the state index is rewritten and fsynced on
2054
+ * every sync tick, so per-tick cost grew with fleet history anyway.
2055
+ *
2056
+ * A tree gives per-key O(entry) writes, real key removal (treeRemove), and
2057
+ * exactly one registration for the life of the store. The legacy map state
2058
+ * (TURN_CHECKPOINTS_ID) is kept as a read-only fallback for stores written
2059
+ * before the split; eviction tombstones legacy keys so a reused agent name
2060
+ * can't inherit a dead agent's checkpoints through the fallback.
2061
+ */
2062
+ legacyCheckpointKeys = null;
2063
+ hasLegacyCheckpoints(agentName) {
2064
+ if (!this.legacyCheckpointKeys) {
2065
+ const legacy = this.store.getStateJson(TURN_CHECKPOINTS_ID);
2066
+ this.legacyCheckpointKeys = new Set(legacy && typeof legacy === 'object' ? Object.keys(legacy) : []);
2067
+ }
2068
+ return this.legacyCheckpointKeys.has(agentName);
2069
+ }
1618
2070
  getTurnCheckpoints(agentName) {
1619
- const data = this.store.getStateJson(TURN_CHECKPOINTS_ID);
1620
- if (!data || typeof data !== 'object')
2071
+ const entry = this.store.treeGet(TURN_CHECKPOINTS_TREE_ID, agentName);
2072
+ if (entry) {
2073
+ const blob = this.store.getBlob(entry.blobHash);
2074
+ if (!blob)
2075
+ return [];
2076
+ try {
2077
+ const parsed = JSON.parse(blob.toString());
2078
+ return Array.isArray(parsed) ? parsed : [];
2079
+ }
2080
+ catch {
2081
+ return [];
2082
+ }
2083
+ }
2084
+ if (!this.hasLegacyCheckpoints(agentName))
1621
2085
  return [];
1622
- const allCheckpoints = data;
1623
- return Array.isArray(allCheckpoints[agentName]) ? [...allCheckpoints[agentName]] : [];
2086
+ const legacy = this.store.getStateJson(TURN_CHECKPOINTS_ID);
2087
+ const list = legacy?.[agentName];
2088
+ return Array.isArray(list) ? [...list] : [];
1624
2089
  }
1625
2090
  saveTurnCheckpoints(agentName, checkpoints) {
1626
- const data = this.store.getStateJson(TURN_CHECKPOINTS_ID);
1627
- const allCheckpoints = (data && typeof data === 'object' ? data : {});
1628
- allCheckpoints[agentName] = checkpoints;
1629
- this.store.setStateJson(TURN_CHECKPOINTS_ID, allCheckpoints);
2091
+ const bytes = Buffer.from(JSON.stringify(checkpoints));
2092
+ const blobHash = this.store.storeBlob(bytes, 'application/json');
2093
+ this.store.treeSet(TURN_CHECKPOINTS_TREE_ID, agentName, {
2094
+ blobHash,
2095
+ size: bytes.length,
2096
+ mode: 0o644,
2097
+ });
2098
+ }
2099
+ /**
2100
+ * Drop a departed agent's checkpoint tree key and turn/redo bookkeeping.
2101
+ * Without this, spawn-and-dispose agents leave a key (and in-memory map
2102
+ * entries) behind for the life of the store/session.
2103
+ */
2104
+ evictTurnCheckpoints(agentName) {
2105
+ this.turnCounters.delete(agentName);
2106
+ this.redoStacks.delete(agentName);
2107
+ // Diagnostics maps are also keyed by agent name and never evicted —
2108
+ // spawn-and-dispose fleets would grow them for the session's life.
2109
+ this.staleWarnAt.delete(agentName);
2110
+ this.lastInferenceAt.delete(agentName);
2111
+ const entry = this.store.treeGet(TURN_CHECKPOINTS_TREE_ID, agentName);
2112
+ if (this.hasLegacyCheckpoints(agentName)) {
2113
+ // A bare treeRemove would resurrect the legacy map's list through the
2114
+ // fallback next time this name is reused — shadow it with an empty
2115
+ // tombstone instead. size <= 2 ⇔ the blob is already "[]".
2116
+ if (!entry || entry.size > 2) {
2117
+ this.saveTurnCheckpoints(agentName, []);
2118
+ }
2119
+ return;
2120
+ }
2121
+ if (entry) {
2122
+ this.store.treeRemove(TURN_CHECKPOINTS_TREE_ID, agentName);
2123
+ }
1630
2124
  }
1631
2125
  /**
1632
2126
  * Run until the queue is empty and all agents are idle.
@@ -1634,6 +2128,9 @@ export class AgentFramework {
1634
2128
  */
1635
2129
  async runUntilIdle() {
1636
2130
  while (!this.queue.isEmpty ||
2131
+ // Direct inference requests (e.g. runEphemeralToCompletion) bypass the
2132
+ // event queue — without this the loop can exit before they're drained.
2133
+ this.pendingRequests.length > 0 ||
1637
2134
  this.activeStreams.size > 0 ||
1638
2135
  Array.from(this.agents.values()).some((a) => a.state.status !== 'idle')) {
1639
2136
  await this.processNextEvent();
@@ -2083,6 +2580,7 @@ export class AgentFramework {
2083
2580
  this.agents.delete(agentName);
2084
2581
  this.agentConfigs.delete(agentName);
2085
2582
  this.conversationAgentHomes.delete(agentName);
2583
+ this.evictTurnCheckpoints(agentName);
2086
2584
  this.emitTrace({
2087
2585
  type: 'mcpl:conversation-disposed',
2088
2586
  agentName,
@@ -2257,6 +2755,7 @@ export class AgentFramework {
2257
2755
  requestCount: requests.length,
2258
2756
  oldestRequestAge: now - oldest,
2259
2757
  });
2758
+ console.error(`[inference-dropped] agent=${agentName} reason=agent_not_found requests=${requests.length}`);
2260
2759
  continue;
2261
2760
  }
2262
2761
  // Skip if agent is busy (inferring, streaming, or waiting for tools)
@@ -2271,12 +2770,23 @@ export class AgentFramework {
2271
2770
  requestCount: requests.length,
2272
2771
  oldestRequestAge: now - oldest,
2273
2772
  });
2773
+ // Loud (but throttled) note that requests are waiting on a busy agent.
2774
+ if ((this.staleWarnAt.get(agentName) ?? 0) < now - 60_000) {
2775
+ this.staleWarnAt.set(agentName, now);
2776
+ console.error(`[inference-stale] agent=${agentName} busy (${agent.state.status}) — ` +
2777
+ `${requests.length} request(s) waiting ${Math.round((now - oldest) / 1000)}s`);
2778
+ }
2274
2779
  }
2275
2780
  this.pendingRequests.push(...requests);
2276
2781
  continue;
2277
2782
  }
2278
2783
  // Check policy
2279
2784
  if (!this.inferencePolicy.shouldInfer(agentName, requests, state)) {
2785
+ // Loud drop: a queued request that dies here is otherwise invisible —
2786
+ // the 2026-07-09 mythos "not responding" diagnosis burned hours on
2787
+ // exactly this class of silent drop. One stderr line per drop.
2788
+ console.error(`[inference-dropped] agent=${agentName} reason=policy-skip ` +
2789
+ `requests=${requests.length} triggers=${requests.map((r) => r.reason).join(',')}`);
2280
2790
  continue;
2281
2791
  }
2282
2792
  // Start streaming inference (non-blocking — driveStream runs in background)
@@ -2313,20 +2823,20 @@ export class AgentFramework {
2313
2823
  }
2314
2824
  this.emitTrace({ type: 'inference:started', agentName: agent.name });
2315
2825
  this.eventGate?.onInferenceStarted(agent.name);
2826
+ this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), startedAt: Date.now() });
2316
2827
  try {
2317
2828
  const allTools = this.getAllTools();
2318
2829
  const tools = allTools.filter((t) => agent.canUseTool(t.name));
2319
2830
  // Gather context from modules (pull-based) and MCPL hooks (push-based)
2320
2831
  // Both produce ContextInjection[] that get merged before inference.
2321
2832
  let injections;
2322
- // Module gatherContext (fail-open, 5s timeout per module)
2323
- // NOTE: module injections are NOT channel-scoped — only the MCPL hook
2324
- // injections below pass through scopeInjectionsForAgent (channel
2325
- // context arrives via beforeInference hooks by adapter convention). A
2326
- // module that starts emitting per-channel context must scope it per
2327
- // agent itself, or conversation forks will see cross-channel content.
2833
+ // Module gatherContext (fail-open, per-module timeout — the module's
2834
+ // contextTimeoutMs, else the registry default). Injections are
2835
+ // channel-scoped via scopeInjectionsForAgent, matching the MCPL hook
2836
+ // injections below, so conversation forks don't see cross-channel
2837
+ // content.
2328
2838
  try {
2329
- const moduleInjections = await this.moduleRegistry.gatherContext(agent.name);
2839
+ const moduleInjections = this.scopeInjectionsForAgent(agent.name, await this.moduleRegistry.gatherContext(agent.name));
2330
2840
  if (moduleInjections.length > 0) {
2331
2841
  injections = moduleInjections;
2332
2842
  }
@@ -2370,6 +2880,13 @@ export class AgentFramework {
2370
2880
  type: 'inference:exhausted',
2371
2881
  agentName: agent.name,
2372
2882
  error: err.message,
2883
+ // Drives the poison-history breaker. `retryable` is kept for
2884
+ // observability, but the breaker gates on `errorType` — only an
2885
+ // `invalid_request` (a 400-class rejection of the history itself)
2886
+ // means retrying the same context can never succeed. auth/abort/
2887
+ // context_length are also non-retryable but must NOT cost history.
2888
+ retryable: err instanceof MembraneError ? err.retryable : undefined,
2889
+ errorType: err instanceof MembraneError ? err.type : undefined,
2373
2890
  });
2374
2891
  this.eventGate?.onInferenceEnded(agent.name);
2375
2892
  if (action.emit) {
@@ -2608,7 +3125,75 @@ export class AgentFramework {
2608
3125
  const category = stopDetails?.category ?? 'unknown';
2609
3126
  console.error(`[inference-refusal] agent=${agent.name} category=${category}` +
2610
3127
  (stopDetails?.explanation ? ` explanation=${stopDetails.explanation}` : ''));
2611
- void this.reactToRefusal(agent.name, category);
3128
+ this.noteRefusal(agent.name, category, tokenUsage);
3129
+ // Rewind: excise the turn that fed the refusal, drop a
3130
+ // metadata-only marker, and retry — keeping the agent on its own
3131
+ // model instead of substituting a fallback. Driven either by an
3132
+ // admin `/unstick` (forced session) or the agent's autoRewind
3133
+ // config; both are bounded.
3134
+ let handledByRewind = false;
3135
+ const rh = agent.refusalHandling;
3136
+ const forced = this.forcedRewind.get(agent.name);
3137
+ // Shed exactly one more (newest, in-sequence) turn and keep the
3138
+ // single episode marker current. `budgetLeft` bounds the loop.
3139
+ const doRewind = (budgetLeft, onStep, onGiveUp) => {
3140
+ if (!budgetLeft) {
3141
+ onGiveUp();
3142
+ return;
3143
+ }
3144
+ const rec = this.shedNewestTurn(agent);
3145
+ if (!rec) {
3146
+ onGiveUp();
3147
+ return;
3148
+ }
3149
+ const count = this.updateRewindMarker(agent, category);
3150
+ onStep(count);
3151
+ if (forced)
3152
+ forced.removed.push(rec);
3153
+ this.pendingRequests.push({
3154
+ agentName: agent.name,
3155
+ reason: forced ? 'unstick-retry' : 'refusal-rewind-retry',
3156
+ source: 'framework',
3157
+ timestamp: Date.now(),
3158
+ });
3159
+ handledByRewind = true;
3160
+ };
3161
+ if (forced) {
3162
+ doRewind(forced.remaining > 0, (count) => {
3163
+ forced.remaining -= 1;
3164
+ console.error(`[unstick] agent=${agent.name} shed ${count} turn(s) ` +
3165
+ `(remaining ${forced.remaining})`);
3166
+ }, () => this.finishUnstick(agent.name, false, category));
3167
+ }
3168
+ else if (rh?.autoRewind) {
3169
+ const cap = Math.max(1, rh.maxRewinds ?? 3);
3170
+ const used = this.refusalRewinds.get(agent.name) ?? 0;
3171
+ doRewind(used < cap, (count) => {
3172
+ this.refusalRewinds.set(agent.name, used + 1);
3173
+ console.error(`[refusal-rewind] agent=${agent.name} shed ${count} turn(s) so far ` +
3174
+ `(cap ${cap})`);
3175
+ }, () => {
3176
+ console.error(`[refusal-rewind] agent=${agent.name} gave up: ` +
3177
+ `${used >= cap ? `cap ${cap} reached` : 'nothing left to shed'}`);
3178
+ this.refusalRewinds.set(agent.name, 0);
3179
+ this.rewindEpisode.delete(agent.name);
3180
+ });
3181
+ }
3182
+ if (!handledByRewind)
3183
+ void this.reactToRefusal(agent.name, category);
3184
+ }
3185
+ else {
3186
+ // A turn that completed WITHOUT a refusal ends the rewind episode:
3187
+ // the model responded. Leave the consolidated marker in place as
3188
+ // the durable record; just clear the per-episode counters.
3189
+ if (this.forcedRewind.has(agent.name)) {
3190
+ this.finishUnstick(agent.name, true);
3191
+ }
3192
+ if (this.refusalRewinds.get(agent.name)) {
3193
+ this.refusalRewinds.set(agent.name, 0);
3194
+ }
3195
+ this.rewindEpisode.delete(agent.name);
3196
+ this.refusalStreak.delete(agent.name);
2612
3197
  }
2613
3198
  // Dispatch speech (and thoughts if any)
2614
3199
  if (speechContent.length > 0 || thoughts.length > 0) {
@@ -2692,11 +3277,19 @@ export class AgentFramework {
2692
3277
  `(${silenced ? 'silencing tool / explicit send' : 'no prose'})`);
2693
3278
  }
2694
3279
  else {
2695
- console.error(`[routing] ${agent.name}: tool-call turn [${toolNames.join(', ')}] -> routing ${segments.length} prose segment(s) in order`);
3280
+ // Snapshot the locus ONCE. This loop dispatches AFTER the agent
3281
+ // is idle (see PR #32 note below), so a queued inbound from
3282
+ // another channel can start a NEW turn and overwrite the per-agent
3283
+ // triggering channel (`activeTriggerChannels`) BETWEEN segments —
3284
+ // splitting one turn's reply across channels (the residual
3285
+ // multi-segment race left by 83dedf8's per-agent locus). Pin every
3286
+ // segment of THIS turn to the channel resolved right now.
3287
+ const turnLocus = this.channelRegistry.resolveLocus(agent.name);
3288
+ console.error(`[routing] ${agent.name}: tool-call turn [${toolNames.join(', ')}] -> routing ${segments.length} prose segment(s) in order -> ${turnLocus ?? '(default)'}`);
2696
3289
  // Deliver sequentially (await each) so the segments land in order.
2697
3290
  for (const seg of segments) {
2698
3291
  try {
2699
- await this.channelRegistry.routeSpeech(agent.name, seg);
3292
+ await this.channelRegistry.routeSpeech(agent.name, seg, turnLocus);
2700
3293
  }
2701
3294
  catch (err) {
2702
3295
  console.error('speech routing failed:', err);
@@ -2742,6 +3335,12 @@ export class AgentFramework {
2742
3335
  type: 'inference:exhausted',
2743
3336
  agentName: agent.name,
2744
3337
  error: err.message,
3338
+ // Drives the poison-history breaker. `retryable` is kept for
3339
+ // observability, but the breaker gates on `errorType` — only an
3340
+ // `invalid_request` (a 400-class rejection of the history
3341
+ // itself) means retrying the same context can never succeed.
3342
+ retryable: err instanceof MembraneError ? err.retryable : undefined,
3343
+ errorType: err instanceof MembraneError ? err.type : undefined,
2745
3344
  });
2746
3345
  this.eventGate?.onInferenceEnded(agent.name);
2747
3346
  if (action.emit) {
@@ -2755,18 +3354,51 @@ export class AgentFramework {
2755
3354
  // Only reset if this is still the active stream (a budget restart
2756
3355
  // may have already started a new stream, bumping streamId)
2757
3356
  if (agent.streamId === myStreamId) {
3357
+ const durationMs = Date.now() - startTime;
2758
3358
  agent.reset();
2759
3359
  this.emitTrace({
2760
3360
  type: 'inference:exhausted',
2761
3361
  agentName: agent.name,
2762
3362
  error: `Stream aborted: ${reason}`,
2763
3363
  });
3364
+ // Postmortem 2026-05-28 P2 #7: persist the abort to the
3365
+ // inference log so future investigations can attribute the
3366
+ // terminal cause without relying on live in-memory reducer
3367
+ // state. Without this, abort-terminated inferences are
3368
+ // invisible to forensic queries (only request-side telemetry
3369
+ // via llm-calls.jsonl shows them, and only by absence).
3370
+ this.logInference({
3371
+ timestamp: startTime,
3372
+ agentName: agent.name,
3373
+ requestId,
3374
+ success: false,
3375
+ error: `Stream aborted: ${reason}`,
3376
+ request: compiledRequest ?? { note: 'streaming request aborted' },
3377
+ durationMs,
3378
+ });
2764
3379
  this.eventGate?.onInferenceEnded(agent.name);
2765
3380
  }
2766
3381
  break;
2767
3382
  }
2768
- case 'usage':
3383
+ case 'usage': {
2769
3384
  agent.lastStreamInputTokens = event.usage.inputTokens;
3385
+ // Closed-loop estimator calibration (2026-07-12). Sample the REAL
3386
+ // prefix size of THIS API call (fresh + cache write + cache read)
3387
+ // and hand it to the context strategy, which accepts exactly one
3388
+ // sample per compile (its arm-once gate) and rejects out-of-band
3389
+ // ratios. It must be sampled HERE, per call: `response.details.
3390
+ // usage` at turn completion is CUMULATIVE across the tool-use
3391
+ // loop (5 calls x ~160k reported as 884k), which is not a
3392
+ // window-shaped number and drove the multiplier to 2.37 before
3393
+ // the guards caught it.
3394
+ try {
3395
+ const realTotal = (event.usage.inputTokens ?? 0) +
3396
+ (event.usage.cacheCreationTokens ?? 0) +
3397
+ (event.usage.cacheReadTokens ?? 0);
3398
+ const strat = agent.getContextManager?.()?.getStrategy?.();
3399
+ strat?.reportRealInputTokens?.(realTotal);
3400
+ }
3401
+ catch { /* calibration is best-effort */ }
2770
3402
  this.emitTrace({
2771
3403
  type: 'inference:usage',
2772
3404
  agentName: agent.name,
@@ -2778,6 +3410,7 @@ export class AgentFramework {
2778
3410
  },
2779
3411
  });
2780
3412
  break;
3413
+ }
2781
3414
  }
2782
3415
  }
2783
3416
  }
@@ -2785,6 +3418,7 @@ export class AgentFramework {
2785
3418
  // Stream itself threw (unexpected) — no retry path here, so also emit
2786
3419
  // inference:exhausted so ephemeral agent promises can settle.
2787
3420
  const err = error instanceof Error ? error : new Error(String(error));
3421
+ const durationMs = Date.now() - startTime;
2788
3422
  this.emitTrace({
2789
3423
  type: 'inference:failed',
2790
3424
  agentName: agent.name,
@@ -2796,10 +3430,31 @@ export class AgentFramework {
2796
3430
  agentName: agent.name,
2797
3431
  error: err.message,
2798
3432
  });
3433
+ // Postmortem 2026-05-28 P2 #7: catch-path failures (stream itself
3434
+ // threw) were previously only visible via in-memory trace listeners.
3435
+ // Persist to inference log for forensic attribution.
3436
+ this.logInference({
3437
+ timestamp: startTime,
3438
+ agentName: agent.name,
3439
+ requestId,
3440
+ success: false,
3441
+ error: `Stream threw: ${err.message}`,
3442
+ request: compiledRequest ?? { note: 'streaming request threw' },
3443
+ durationMs,
3444
+ });
2799
3445
  agent.reset();
2800
3446
  this.eventGate?.onInferenceEnded(agent.name);
2801
3447
  }
2802
3448
  finally {
3449
+ // Clear the gate's inference flag on EVERY exit path (paired with the
3450
+ // onInferenceStarted in startAgentStream). The branch-level calls above
3451
+ // are kept but are best-effort; if any exit path bypassed them the agent
3452
+ // stayed stuck in the gate's `inferring` set, which permanently BUFFERS
3453
+ // all incoming events → the agent silently stops waking on messages
3454
+ // (typing still stops, compression still runs — matching the observed
3455
+ // wedge). onInferenceEnded is idempotent, so a redundant call is safe.
3456
+ this.eventGate?.onInferenceEnded(agent.name);
3457
+ this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), endedAt: Date.now() });
2803
3458
  // Stop the typing indicator on every exit path (complete, error,
2804
3459
  // exhausted, abort) so it never sticks after the turn ends.
2805
3460
  this.channelRegistry?.stopTyping();
@@ -2884,7 +3539,10 @@ export class AgentFramework {
2884
3539
  if (blocks) {
2885
3540
  return { toolUseId: callId, content: blocks, isError: false };
2886
3541
  }
2887
- let content = JSON.stringify(afResult.data);
3542
+ // JSON.stringify returns the VALUE undefined (not a string) for undefined
3543
+ // input — a module tool returning `{ success: true }` with no data would
3544
+ // otherwise make `content.length` below throw a TypeError mid-turn.
3545
+ let content = JSON.stringify(afResult.data) ?? '';
2888
3546
  if (maxChars && content.length > maxChars) {
2889
3547
  content = safeSlice(content, 0, maxChars)
2890
3548
  + '\n\n[truncated — original was ' + content.length + ' chars]';
@@ -2954,11 +3612,11 @@ export class AgentFramework {
2954
3612
  entryToStore.response = { blobId };
2955
3613
  }
2956
3614
  }
2957
- // Append to the inference log state
2958
- const data = this.store.getStateJson(INFERENCE_LOG_ID);
2959
- const entries = Array.isArray(data) ? data : [];
2960
- entries.push(entryToStore);
2961
- this.store.setStateJson(INFERENCE_LOG_ID, entries);
3615
+ // Append one record per inference. A read-modify-write via setStateJson
3616
+ // would emit a whole-array Set — record size then grows with accumulated
3617
+ // history (O(n²) aggregate disk) and the append_log strategy registered
3618
+ // for this state never sees an append.
3619
+ this.store.appendToStateJson(INFERENCE_LOG_ID, entryToStore);
2962
3620
  }
2963
3621
  logProcessEvent(event, responses) {
2964
3622
  const entry = {
@@ -2974,11 +3632,8 @@ export class AgentFramework {
2974
3632
  const blobId = this.store.storeBlob(Buffer.from(responsesJson), 'application/json');
2975
3633
  entryToStore.responses = { blobId };
2976
3634
  }
2977
- // Append to the process log state
2978
- const data = this.store.getStateJson(PROCESS_LOG_ID);
2979
- const entries = Array.isArray(data) ? data : [];
2980
- entries.push(entryToStore);
2981
- this.store.setStateJson(PROCESS_LOG_ID, entries);
3635
+ // Append one record per event — same rationale as logInference.
3636
+ this.store.appendToStateJson(PROCESS_LOG_ID, entryToStore);
2982
3637
  }
2983
3638
  /**
2984
3639
  * Find the MCPL server for a tool call by checking against the prefix map.
@@ -3175,15 +3830,19 @@ export class AgentFramework {
3175
3830
  // without this the only durable record of a failed inference is a field in
3176
3831
  // llm-calls.jsonl — invisible to operator, agent, and monitoring.
3177
3832
  if (event.type === 'inference:exhausted') {
3178
- this.noteInferenceExhausted(event.agentName ?? 'unknown', event.error ?? 'unknown error');
3833
+ this.noteInferenceExhausted(event.agentName ?? 'unknown', event.error ?? 'unknown error', event.retryable, event.errorType);
3179
3834
  }
3180
3835
  else if (event.type === 'inference:completed') {
3181
3836
  // A successful response — even mid-turn between tool calls — proves the
3182
- // agent isn't hard-down; clear its consecutive-failure streak.
3837
+ // agent isn't hard-down; clear its consecutive-failure streak (and the
3838
+ // poison-history breaker's rewind budget).
3183
3839
  const name = event.agentName;
3184
3840
  if (name && this.consecutiveInferenceFailures.get(name)) {
3185
3841
  this.consecutiveInferenceFailures.set(name, 0);
3186
3842
  }
3843
+ if (name && this.exhaustionRewinds.get(name)) {
3844
+ this.exhaustionRewinds.set(name, 0);
3845
+ }
3187
3846
  }
3188
3847
  const traceEvent = {
3189
3848
  ...event,
@@ -3212,12 +3871,120 @@ export class AgentFramework {
3212
3871
  * inference, so this never causes a retry/wake loop.
3213
3872
  * 3. escalation — after N consecutive failures the agent is hard-down;
3214
3873
  * log that loudly (a repeated identical failure is the textbook signal).
3874
+ * 4. breaker — when the failures are an `invalid_request` (a 400-class
3875
+ * rejection of the history itself, e.g. corrupted tool_use/tool_result
3876
+ * pairing or an oversized attachment), retrying the same context can
3877
+ * never succeed: every new push event wakes the agent onto the same
3878
+ * poisoned history forever. At the hard-down threshold, automatically
3879
+ * quarantine: shed the newest complete exchange (the same forced-rewind
3880
+ * primitive `/unstick` uses — shedNewestTurn never orphans a
3881
+ * tool_use/thinking block) and retry, bounded by the same rewind cap so
3882
+ * the breaker can never eat the whole history.
3215
3883
  */
3216
- noteInferenceExhausted(agentName, reason) {
3884
+ /**
3885
+ * Live health snapshot for /healthz and doctor tooling: gate state, queued
3886
+ * work, per-agent status + last inference activity. Cheap and read-only.
3887
+ */
3888
+ healthSnapshot() {
3889
+ return {
3890
+ at: new Date().toISOString(),
3891
+ uptimeSec: Math.round(process.uptime()),
3892
+ gate: this.eventGate?.inferenceDiagnostics() ?? null,
3893
+ pendingRequests: this.pendingRequests.length,
3894
+ activeStreams: [...this.activeStreams.keys()],
3895
+ agents: [...this.agents.entries()].map(([name, agent]) => ({
3896
+ name,
3897
+ status: agent.state.status,
3898
+ consecutiveInferenceFailures: this.consecutiveInferenceFailures.get(name) ?? 0,
3899
+ lastInference: this.lastInferenceAt.get(name) ?? null,
3900
+ refusalStats: this.refusalStats.get(name) ?? null,
3901
+ })),
3902
+ };
3903
+ }
3904
+ /** Append a structured JSONL record to logs/failures.log (best-effort).
3905
+ * Durable and independent of journald/unit log redirects — this is what
3906
+ * connectome-doctor and fleet tooling read. Always stamps `at`. Legacy
3907
+ * records carry {at, agent, consecutive, reason}; new records add `kind`
3908
+ * and kind-specific fields (additive only — doctor parses by regex). */
3909
+ logFailure(record) {
3910
+ try {
3911
+ mkdirSync('logs', { recursive: true });
3912
+ appendFileSync('logs/failures.log', JSON.stringify({ at: new Date().toISOString(), ...record }) + '\n');
3913
+ }
3914
+ catch { /* best-effort */ }
3915
+ }
3916
+ /**
3917
+ * Ops alert — the one escalation path for "a human should hear about this":
3918
+ * (1) durable failures.log record (unless the caller already wrote one),
3919
+ * (2) an `ops:alert` trace so authorized observers get it on the wire,
3920
+ * (3) if CONNECTOME_OPS_WEBHOOK is set, a Discord post throttled to one per
3921
+ * (agent, kind) per cooldown window — a persistent failure re-posts
3922
+ * every ~15 min instead of flooding the channel on every occurrence.
3923
+ * See connectome docs/observability.md.
3924
+ */
3925
+ opsAlert(kind, agentName, message, opts) {
3926
+ if (!opts?.skipLog) {
3927
+ this.logFailure({ agent: agentName, kind, reason: message, ...opts?.data });
3928
+ }
3929
+ this.emitTrace({ type: 'ops:alert', kind, agentName, message, data: opts?.data });
3930
+ const hook = process.env.CONNECTOME_OPS_WEBHOOK;
3931
+ if (!hook)
3932
+ return;
3933
+ const key = `${agentName}:${kind}`;
3934
+ const now = Date.now();
3935
+ if (now - (this.opsAlertLastSent.get(key) ?? 0) < this.opsAlertCooldownMs)
3936
+ return;
3937
+ // Stamp BEFORE the async post so a burst can't race past the cooldown.
3938
+ this.opsAlertLastSent.set(key, now);
3939
+ fetch(hook, {
3940
+ method: 'POST',
3941
+ headers: { 'content-type': 'application/json' },
3942
+ body: JSON.stringify({
3943
+ content: `\u{1F6A8} **${agentName}** ${kind}: ${message.slice(0, 500)}`,
3944
+ }),
3945
+ }).catch((err) => console.error(`[ops-alert] webhook post failed (${key}):`, err instanceof Error ? err.message : err));
3946
+ }
3947
+ /**
3948
+ * Record a model refusal for observability: per-agent stats + a
3949
+ * consecutive-refusal streak (exposed via healthSnapshot), a durable
3950
+ * failures.log record for EVERY refusal (previously invisible outside
3951
+ * stderr), and a throttled ops alert once refusals repeat — one refusal is
3952
+ * often survivable (auto-rewind may clear it); two in a row means stuck.
3953
+ * The streak is reset by any non-refusal completion in the stream driver.
3954
+ */
3955
+ noteRefusal(agentName, category, tokens) {
3956
+ const stats = this.refusalStats.get(agentName)
3957
+ ?? { total: 0, byCategory: {}, lastAt: 0, lastCategory: '' };
3958
+ stats.total += 1;
3959
+ stats.byCategory[category] = (stats.byCategory[category] ?? 0) + 1;
3960
+ stats.lastAt = Date.now();
3961
+ stats.lastCategory = category;
3962
+ this.refusalStats.set(agentName, stats);
3963
+ const streak = (this.refusalStreak.get(agentName) ?? 0) + 1;
3964
+ this.refusalStreak.set(agentName, streak);
3965
+ this.logFailure({
3966
+ agent: agentName,
3967
+ kind: 'refusal',
3968
+ category,
3969
+ streak,
3970
+ locus: this.channelRegistry?.buildChannelContext()?.incoming?.channelId ?? null,
3971
+ tokens: tokens ?? null,
3972
+ });
3973
+ if (streak >= 2) {
3974
+ this.opsAlert('refusal', agentName, `refusal streak ${streak}, category=${category}`, { skipLog: true, data: { category, streak } });
3975
+ }
3976
+ return streak;
3977
+ }
3978
+ noteInferenceExhausted(agentName, reason, retryable, errorType) {
3217
3979
  const streak = (this.consecutiveInferenceFailures.get(agentName) ?? 0) + 1;
3218
3980
  this.consecutiveInferenceFailures.set(agentName, streak);
3981
+ this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), failedAt: Date.now(), lastError: reason.slice(0, 300) });
3219
3982
  // (1) Durable stderr line — works in headless/daemon mode with no client.
3220
3983
  console.error(`[inference-failed] agent=${agentName} consecutive=${streak}: ${reason}`);
3984
+ // (1b) Machine-greppable durable record, independent of journald/unit log
3985
+ // redirects: logs/failures.log under the host's working directory. This is
3986
+ // what connectome-doctor reads. Legacy fields kept; `kind` is additive.
3987
+ this.logFailure({ agent: agentName, consecutive: streak, reason, kind: 'inference-exhausted' });
3221
3988
  // (2) Agent-facing chronicle marker (no inference triggered → no loop).
3222
3989
  const agent = this.agents.get(agentName);
3223
3990
  if (agent && process.env.SUPPRESS_INFERENCE_FAILED_MARKER !== '1') {
@@ -3239,8 +4006,64 @@ export class AgentFramework {
3239
4006
  if (streak >= this.inferenceFailureEscalationThreshold) {
3240
4007
  console.error(`[inference-hard-down] agent=${agentName} has FAILED ${streak} consecutive ` +
3241
4008
  `inferences — it cannot complete a turn. Last reason: ${reason}`);
4009
+ // Ops alert (webhook + ops:alert trace), throttled per (agent, kind) —
4010
+ // a hard-down agent re-posts every ~15 min, not on every failed retry.
4011
+ // failures.log already got the per-exhaustion record above.
4012
+ this.opsAlert('hard-down', agentName, `${streak} consecutive inference failures. Last reason: ${reason}`, { skipLog: true });
4013
+ // (4) Poison-history breaker: ONLY for `invalid_request` — a 400-class
4014
+ // rejection of the history itself. Deliberately NOT keyed on
4015
+ // `retryable === false`, which membrane also returns for auth (expired
4016
+ // key), abort (deliberate cancel), context_length (compression's job,
4017
+ // and shedding newest is the wrong direction), safety and unsupported —
4018
+ // none of which mean the history is poisoned, and all of which would
4019
+ // otherwise shed good exchanges and stamp a false "the API kept
4020
+ // rejecting your history" marker. `retryable` is kept only for the trace.
4021
+ void retryable;
4022
+ if (errorType === 'invalid_request' && agent) {
4023
+ this.quarantinePoisonedHistory(agent, reason);
4024
+ }
3242
4025
  }
3243
4026
  }
4027
+ /**
4028
+ * Automatic poison-history quarantine (the actual "breaker"). Sheds the
4029
+ * newest complete exchange from the agent's history — reusing the same
4030
+ * primitives as the refusal auto-rewind / `/unstick` (shedNewestTurn +
4031
+ * the single consolidated episode marker) — and queues a retry so the
4032
+ * rewound history is verified immediately instead of waiting for the next
4033
+ * push event to wake the agent onto the same poisoned context.
4034
+ *
4035
+ * Bounded: at most `refusalHandling.maxRewinds` (default 3, hard cap 10)
4036
+ * sheds per failure episode; the budget resets on any successful inference.
4037
+ * At the cap (or with nothing left to shed) it stops — the agent stays up
4038
+ * and hard-down logging continues, but no further history is consumed.
4039
+ */
4040
+ quarantinePoisonedHistory(agent, reason) {
4041
+ const cap = Math.max(1, Math.min(10, agent.refusalHandling?.maxRewinds ?? 3));
4042
+ const used = this.exhaustionRewinds.get(agent.name) ?? 0;
4043
+ if (used >= cap) {
4044
+ console.error(`[inference-rewind] agent=${agent.name} rewind cap ${cap} reached — ` +
4045
+ `not shedding further history. Manual repair (/unstick or /undo) needed.`);
4046
+ return;
4047
+ }
4048
+ const rec = this.shedNewestTurn(agent);
4049
+ if (!rec) {
4050
+ console.error(`[inference-rewind] agent=${agent.name} has nothing left to shed — ` +
4051
+ `history is only system markers. Manual repair needed.`);
4052
+ return;
4053
+ }
4054
+ this.exhaustionRewinds.set(agent.name, used + 1);
4055
+ const count = this.updateRewindMarker(agent, truncateReason(reason), 'inference-failure');
4056
+ console.error(`[inference-rewind] agent=${agent.name} auto-quarantined ${rec.descriptor} ` +
4057
+ `(${count} turn(s) shed this episode, cap ${cap}) after non-retryable ` +
4058
+ `inference failures — retrying on the rewound history.`);
4059
+ // Retry immediately on the repaired history (bounded by the cap above).
4060
+ this.pendingRequests.push({
4061
+ agentName: agent.name,
4062
+ reason: 'inference-failure-rewind-retry',
4063
+ source: 'framework',
4064
+ timestamp: Date.now(),
4065
+ });
4066
+ }
3244
4067
  // ==========================================================================
3245
4068
  // MCPL subsystem wiring
3246
4069
  // ==========================================================================
@@ -3314,8 +4137,9 @@ export class AgentFramework {
3314
4137
  }
3315
4138
  },
3316
4139
  });
3317
- // Host capabilities advertised during the MCP handshake
3318
- const hostCapabilities = {
4140
+ // Host capabilities advertised during the MCP handshake — stored so
4141
+ // servers can also be connected later at runtime (connectMcplServer).
4142
+ this.mcplHostCapabilities = {
3319
4143
  version: '0.4',
3320
4144
  pushEvents: true,
3321
4145
  contextHooks: {
@@ -3326,56 +4150,193 @@ export class AgentFramework {
3326
4150
  };
3327
4151
  for (const config of serverConfigs) {
3328
4152
  try {
3329
- // Record per-server channel subscription policy before the server
3330
- // registers channels — handleRegister fires during the handshake.
3331
- if (config.channelSubscription !== undefined && this.channelRegistry) {
3332
- this.channelRegistry.setSubscriptionPolicy(config.id, config.channelSubscription);
3333
- }
3334
- const connection = await this.mcplServerRegistry.addServer(config, hostCapabilities);
3335
- // Wire event listeners then flush any events that arrived during the
3336
- // handshake window (between setupMessageRouting and now).
3337
- this.wireMcplEvents(connection);
3338
- connection.ready();
3339
- // Initialize feature sets if server advertises MCPL capabilities
3340
- if (connection.capabilities) {
3341
- const updateParams = this.featureSetManager.initializeServer(config.id, connection.capabilities, {
3342
- enabledFeatureSets: config.enabledFeatureSets,
3343
- disabledFeatureSets: config.disabledFeatureSets,
3344
- });
3345
- // Inform server which feature sets are enabled/disabled
3346
- if (updateParams.enabled?.length || updateParams.disabled?.length) {
3347
- connection.sendFeatureSetsUpdate(updateParams);
3348
- }
3349
- // Configure scope whitelist/blacklist patterns
3350
- if (config.scopes) {
3351
- this.scopeManager.configureAll(config.scopes);
3352
- }
3353
- // Register stateful feature sets with checkpoint manager (Step 8)
3354
- if (this.checkpointManager) {
3355
- const declared = this.featureSetManager.getDeclaredFeatureSets(config.id);
3356
- if (declared) {
3357
- for (const [fsName, fsDecl] of Object.entries(declared)) {
3358
- if (fsDecl.rollback || fsDecl.hostState) {
3359
- this.checkpointManager.registerFeatureSet(config.id, fsName, {
3360
- hostState: fsDecl.hostState ?? false,
3361
- rollback: fsDecl.rollback ?? false,
3362
- });
3363
- }
3364
- }
3365
- }
3366
- }
3367
- }
3368
- this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` });
4153
+ await this.connectMcplServerInternal(config);
3369
4154
  }
3370
4155
  catch (error) {
3371
4156
  // Fail-open: log and continue with remaining servers
3372
4157
  const err = error instanceof Error ? error : new Error(String(error));
3373
4158
  console.error(`Failed to connect MCPL server "${config.id}":`, err.message);
4159
+ // Reaching this catch means no stub was created, so nothing will
4160
+ // retry — except in the edge case where addServer succeeded (with a
4161
+ // reconnecting stub) and a later setup step threw; config.reconnect
4162
+ // is the best cheap approximation of that.
4163
+ this.emitTrace({
4164
+ type: 'mcpl:server-connect-failed',
4165
+ serverId: config.id,
4166
+ error: err.message,
4167
+ attempt: 0,
4168
+ willRetry: config.reconnect === true,
4169
+ });
3374
4170
  }
3375
4171
  }
3376
4172
  // Discover tools from all connected servers
3377
4173
  await this.refreshMcplTools();
3378
4174
  }
4175
+ /**
4176
+ * Connect a single MCPL server: register routing entries, open the
4177
+ * connection, wire events, and initialize feature sets / scopes /
4178
+ * checkpoints. Shared by startup (initializeMcpl) and the runtime
4179
+ * lifecycle API (connectMcplServer). Throws on connection failure.
4180
+ */
4181
+ async connectMcplServerInternal(config) {
4182
+ if (!this.mcplServerRegistry || !this.mcplHostCapabilities) {
4183
+ throw new Error('MCPL subsystem is not initialized');
4184
+ }
4185
+ // Register prefix + config for tool dispatch routing (idempotent with
4186
+ // the pre-registration pass in initializeMcpl).
4187
+ const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
4188
+ this.mcplPrefixMap.set(prefix, config.id);
4189
+ this.mcplServerConfigs.set(config.id, config);
4190
+ // Record per-server channel subscription policy before the server
4191
+ // registers channels — handleRegister fires during the handshake.
4192
+ if (config.channelSubscription !== undefined && this.channelRegistry) {
4193
+ this.channelRegistry.setSubscriptionPolicy(config.id, config.channelSubscription);
4194
+ }
4195
+ const connection = await this.mcplServerRegistry.addServer(config, this.mcplHostCapabilities);
4196
+ // Wire event listeners then flush any events that arrived during the
4197
+ // handshake window (between setupMessageRouting and now).
4198
+ this.wireMcplEvents(connection);
4199
+ connection.ready();
4200
+ // Initialize feature sets if server advertises MCPL capabilities
4201
+ this.registerMcplServerFeatures(config, connection);
4202
+ this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` });
4203
+ }
4204
+ /**
4205
+ * (Re-)establish a server's host-side MCPL registration: feature-set state
4206
+ * (plus the featureSets/update sent to the server), scope patterns, and
4207
+ * stateful-feature-set checkpoint registration.
4208
+ *
4209
+ * Called on initial connect (connectMcplServerInternal) AND after every
4210
+ * auto-reconnect. The 'close' handler drops the feature-set registration, so
4211
+ * without the reconnect re-run FeatureSetManager.validateInbound would throw
4212
+ * "Unknown server" forever — every push event and inference request from the
4213
+ * revived server silently rejected until a full host restart.
4214
+ *
4215
+ * CheckpointManager.registerFeatureSet is idempotent (no-op for a key that
4216
+ * already has a tree), so checkpoint trees preserved across a transient
4217
+ * disconnect are resumed, not reset.
4218
+ */
4219
+ registerMcplServerFeatures(config, connection) {
4220
+ if (!connection.capabilities)
4221
+ return;
4222
+ const updateParams = this.featureSetManager.initializeServer(config.id, connection.capabilities, {
4223
+ enabledFeatureSets: config.enabledFeatureSets,
4224
+ disabledFeatureSets: config.disabledFeatureSets,
4225
+ });
4226
+ // Inform server which feature sets are enabled/disabled
4227
+ if (updateParams.enabled?.length || updateParams.disabled?.length) {
4228
+ connection.sendFeatureSetsUpdate(updateParams);
4229
+ }
4230
+ // Configure scope whitelist/blacklist patterns
4231
+ if (config.scopes) {
4232
+ this.scopeManager.configureAll(config.scopes);
4233
+ }
4234
+ // Register stateful feature sets with checkpoint manager (Step 8)
4235
+ if (this.checkpointManager) {
4236
+ const declared = this.featureSetManager.getDeclaredFeatureSets(config.id);
4237
+ if (declared) {
4238
+ for (const [fsName, fsDecl] of Object.entries(declared)) {
4239
+ if (fsDecl.rollback || fsDecl.hostState) {
4240
+ this.checkpointManager.registerFeatureSet(config.id, fsName, {
4241
+ hostState: fsDecl.hostState ?? false,
4242
+ rollback: fsDecl.rollback ?? false,
4243
+ });
4244
+ }
4245
+ }
4246
+ }
4247
+ }
4248
+ }
4249
+ // ==========================================================================
4250
+ // Runtime MCPL server lifecycle (agent-facing hot deploy/restart/unload)
4251
+ // ==========================================================================
4252
+ /**
4253
+ * Connect a new MCPL server at runtime. Refreshes the tool list and
4254
+ * notifies the agent of newly available tools. Throws if the MCPL
4255
+ * subsystem is not initialized, the id is already connected, or the
4256
+ * connection/handshake fails.
4257
+ */
4258
+ async connectMcplServer(config) {
4259
+ // Lazily bring up the MCPL subsystem — a framework that started with zero
4260
+ // configured servers can still deploy its first one at runtime.
4261
+ if (!this.mcplServerRegistry || !this.mcplHostCapabilities) {
4262
+ await this.initializeMcpl([], this.mcplInferenceRoutingConfig ?? undefined);
4263
+ }
4264
+ // Same collision rules create() enforces at startup: the tool prefix must
4265
+ // not shadow a module name or another server's prefix.
4266
+ const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
4267
+ if (this.moduleRegistry.getAllModules().some(m => m.name === prefix)) {
4268
+ throw new Error(`MCPL server "${config.id}" toolPrefix "${prefix}" collides with module "${prefix}"`);
4269
+ }
4270
+ const prefixOwner = this.mcplPrefixMap.get(prefix);
4271
+ if (prefixOwner && prefixOwner !== config.id) {
4272
+ throw new Error(`MCPL server "${config.id}" toolPrefix "${prefix}" collides with server "${prefixOwner}"`);
4273
+ }
4274
+ const oldToolNames = new Set(this.mcplTools.map(t => t.name));
4275
+ await this.connectMcplServerInternal(config);
4276
+ await this.refreshMcplTools();
4277
+ this.emitMcplToolDiff(oldToolNames, config.id);
4278
+ }
4279
+ /**
4280
+ * Disconnect an MCPL server at runtime: close the connection, destroy its
4281
+ * feature-set and checkpoint state (permanent removal — unlike a transient
4282
+ * transport close, which preserves checkpoints for the reconnect), remove
4283
+ * its channels from the registry, drop routing entries, and refresh tools.
4284
+ * No-op-ish if the server is not connected (still clears routing state).
4285
+ */
4286
+ async disconnectMcplServer(id) {
4287
+ if (!this.mcplServerRegistry) {
4288
+ throw new Error('MCPL subsystem is not initialized');
4289
+ }
4290
+ const config = this.mcplServerConfigs.get(id);
4291
+ const oldToolNames = new Set(this.mcplTools.map(t => t.name));
4292
+ await this.mcplServerRegistry.removeServer(id);
4293
+ this.channelRegistry?.removeServer(id);
4294
+ // Permanent removal: also destroy feature-set and checkpoint state
4295
+ // explicitly. The 'close' handler usually does this, but a connection that
4296
+ // already transiently closed (reconnect pending) emits no second 'close'
4297
+ // from close(), and the transient path deliberately preserves checkpoints.
4298
+ this.featureSetManager?.removeServer(id);
4299
+ this.checkpointManager?.removeServer(id);
4300
+ const prefix = config?.toolPrefix ?? `mcpl--${id}`;
4301
+ this.mcplPrefixMap.delete(prefix);
4302
+ this.mcplServerConfigs.delete(id);
4303
+ await this.refreshMcplTools();
4304
+ this.emitMcplToolDiff(oldToolNames, id);
4305
+ }
4306
+ /**
4307
+ * Restart an MCPL server at runtime: disconnect, then reconnect with the
4308
+ * same (or an updated) config. This actually respawns a stdio child —
4309
+ * unlike `reconnect: true`, which only retries after transport-level
4310
+ * failures. Throws if the server has no stored config and none is given.
4311
+ */
4312
+ async restartMcplServer(id, newConfig) {
4313
+ const config = newConfig ?? this.mcplServerConfigs.get(id);
4314
+ if (!config) {
4315
+ throw new Error(`MCPL server "${id}" is not configured`);
4316
+ }
4317
+ await this.disconnectMcplServer(id);
4318
+ await this.connectMcplServer(config);
4319
+ }
4320
+ /**
4321
+ * List configured MCPL servers with live connection status and the number
4322
+ * of tools each currently contributes.
4323
+ */
4324
+ listMcplServers() {
4325
+ const result = [];
4326
+ for (const [id, config] of this.mcplServerConfigs) {
4327
+ const prefix = config.toolPrefix ?? `mcpl--${id}`;
4328
+ const connection = this.mcplServerRegistry?.getServer(id) ?? null;
4329
+ result.push({
4330
+ id,
4331
+ connected: connection?.isConnected ?? false,
4332
+ toolPrefix: prefix,
4333
+ toolCount: this.mcplTools.filter(t => t.name.startsWith(`${prefix}--`)).length,
4334
+ command: config.command,
4335
+ url: config.url,
4336
+ });
4337
+ }
4338
+ return result;
4339
+ }
3379
4340
  /**
3380
4341
  * Wire event listeners on an MCPL server connection.
3381
4342
  * Push events and inference requests are deferred to Steps 6/7.
@@ -3446,14 +4407,102 @@ export class AgentFramework {
3446
4407
  connection.on('tools-list-changed', () => {
3447
4408
  this.handleToolsListChanged(connection.id);
3448
4409
  });
3449
- // Also refresh tools on reconnect (server may have different tools)
3450
- connection.on('reconnect', () => {
4410
+ // Re-establish full server registration on reconnect. The 'close' handler
4411
+ // removed the feature-set state, so pushes / inference requests from the
4412
+ // revived server would otherwise be rejected with "Unknown server" until a
4413
+ // host restart. Re-runs the same init as the initial connect path (the
4414
+ // fresh handshake refreshed connection.capabilities); checkpoint trees
4415
+ // preserved across the transient close are resumed by idempotent
4416
+ // registration. Then refresh tools (server may have different tools).
4417
+ connection.on('reconnect', (info) => {
4418
+ try {
4419
+ const config = this.mcplServerConfigs.get(connection.id);
4420
+ if (config) {
4421
+ this.registerMcplServerFeatures(config, connection);
4422
+ }
4423
+ }
4424
+ catch (error) {
4425
+ console.error(`MCPL server "${connection.id}" re-registration after reconnect failed:`, error instanceof Error ? error.message : error);
4426
+ }
3451
4427
  this.handleToolsListChanged(connection.id);
4428
+ this.emitTrace({
4429
+ type: 'mcpl:server-reconnected',
4430
+ serverId: connection.id,
4431
+ attempts: info?.attempts ?? 0,
4432
+ });
4433
+ // Mirror the module:removed emitted on 'close' so module-lifecycle
4434
+ // consumers see the server come back, not just vanish.
4435
+ this.emitTrace({ type: 'module:added', moduleName: `mcpl:${connection.id}` });
3452
4436
  });
3453
- // Cleanup on disconnect
3454
- connection.on('close', () => {
4437
+ // Surface connect/reconnect failures. Before these traces existed the
4438
+ // only receipt was a console.error on the host's own stderr — invisible
4439
+ // unless someone ssh'd in and read the process log.
4440
+ connection.on('connect-failed', (params) => {
4441
+ this.emitTrace({
4442
+ type: 'mcpl:server-connect-failed',
4443
+ serverId: connection.id,
4444
+ error: params.error,
4445
+ attempt: params.attempt,
4446
+ willRetry: connection.willReconnect,
4447
+ });
4448
+ });
4449
+ connection.on('reconnect-failed', (params) => {
4450
+ this.emitTrace({
4451
+ type: 'mcpl:server-connect-failed',
4452
+ serverId: connection.id,
4453
+ error: params.error,
4454
+ attempt: params.attempt,
4455
+ willRetry: connection.willReconnect,
4456
+ });
4457
+ // The reconnect loop never gives up (backoff caps at ~300s), so
4458
+ // "the server is effectively down" is an attempt-count judgment:
4459
+ // 5 failed attempts ≈ a few minutes of outage. Throttled per
4460
+ // (serverId, kind), so a long outage re-posts every ~15 min.
4461
+ if (params.attempt >= 5) {
4462
+ this.opsAlert('mcpl-down', connection.id, `MCPL server unreachable (attempt ${params.attempt}): ${params.error}`);
4463
+ }
4464
+ });
4465
+ // A late response to a tools/call that already timed out, carrying
4466
+ // stateful data the server advanced to. The host can't re-inject it (the
4467
+ // dispatch context is gone), but surfacing it makes the checkpoint-tree
4468
+ // divergence greppable instead of a silent drift into stale state.
4469
+ connection.on('orphaned-response', (info) => {
4470
+ this.emitTrace({
4471
+ type: 'mcpl:orphaned-response',
4472
+ serverId: connection.id,
4473
+ responseId: info.id,
4474
+ hadState: info.hadState,
4475
+ hadCheckpoint: info.hadCheckpoint,
4476
+ });
4477
+ });
4478
+ // Connection-level errors (e.g. child process 'error' after spawn).
4479
+ // Attaching this listener also keeps an unhandled EventEmitter 'error'
4480
+ // from crashing the host process.
4481
+ connection.on('error', (err) => {
4482
+ this.emitTrace({
4483
+ type: 'mcpl:server-error',
4484
+ serverId: connection.id,
4485
+ error: err.message,
4486
+ });
4487
+ });
4488
+ // Cleanup on disconnect. Feature-set state is always dropped (the server
4489
+ // can't be validated while down; reconnect re-registers it). Checkpoint
4490
+ // trees, however, are durable state: CheckpointManager.removeServer deletes
4491
+ // them AND persists the deletion to Chronicle. The close handler must
4492
+ // therefore NEVER destroy checkpoints — `willReconnect` is false on a clean
4493
+ // stop() too (reconnectEnabled is cleared before 'close' fires), so gating
4494
+ // on it would erase every checkpoint tree on an ordinary host restart while
4495
+ // a SIGKILL preserves them. Permanent removal is owned solely by
4496
+ // disconnectMcplServer, which deletes the trees explicitly.
4497
+ connection.on('close', (code, signal) => {
3455
4498
  this.featureSetManager?.removeServer(connection.id);
3456
- this.checkpointManager?.removeServer(connection.id);
4499
+ this.emitTrace({
4500
+ type: 'mcpl:server-closed',
4501
+ serverId: connection.id,
4502
+ code: code ?? null,
4503
+ signal: signal ?? null,
4504
+ willReconnect: connection.willReconnect,
4505
+ });
3457
4506
  this.emitTrace({ type: 'module:removed', moduleName: `mcpl:${connection.id}` });
3458
4507
  });
3459
4508
  }
@@ -3822,7 +4871,7 @@ export class AgentFramework {
3822
4871
  finish({ success: false, error: 'sleep: `seconds` must be a positive number', isError: true });
3823
4872
  return;
3824
4873
  }
3825
- const { until } = gate.setSleep(seconds, input.message);
4874
+ const { until } = gate.setSleep(seconds, input.message, agentName);
3826
4875
  const announce = input.announce !== false; // default true
3827
4876
  const human = seconds >= 3600 ? `${(seconds / 3600).toFixed(1)}h`
3828
4877
  : seconds >= 60 ? `${Math.round(seconds / 60)}m`