@animalabs/agent-framework 0.6.0 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/agent.d.ts +4 -0
- package/dist/src/agent.d.ts.map +1 -1
- package/dist/src/agent.js +15 -0
- package/dist/src/agent.js.map +1 -1
- package/dist/src/framework.d.ts +235 -2
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +1141 -92
- package/dist/src/framework.js.map +1 -1
- package/dist/src/gate/event-gate.d.ts +20 -1
- package/dist/src/gate/event-gate.d.ts.map +1 -1
- package/dist/src/gate/event-gate.js +60 -4
- package/dist/src/gate/event-gate.js.map +1 -1
- package/dist/src/index.d.ts +3 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +4 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/mcpl/channel-registry.d.ts +17 -1
- package/dist/src/mcpl/channel-registry.d.ts.map +1 -1
- package/dist/src/mcpl/channel-registry.js +34 -3
- package/dist/src/mcpl/channel-registry.js.map +1 -1
- package/dist/src/mcpl/push-handler.d.ts.map +1 -1
- package/dist/src/mcpl/push-handler.js +4 -0
- package/dist/src/mcpl/push-handler.js.map +1 -1
- package/dist/src/mcpl/server-connection.d.ts +29 -0
- package/dist/src/mcpl/server-connection.d.ts.map +1 -1
- package/dist/src/mcpl/server-connection.js +102 -4
- package/dist/src/mcpl/server-connection.js.map +1 -1
- package/dist/src/mcpl/types.d.ts +17 -1
- package/dist/src/mcpl/types.d.ts.map +1 -1
- package/dist/src/mcpl/types.js.map +1 -1
- package/dist/src/module-registry.d.ts +15 -1
- package/dist/src/module-registry.d.ts.map +1 -1
- package/dist/src/module-registry.js +92 -4
- package/dist/src/module-registry.js.map +1 -1
- package/dist/src/types/agent.d.ts +37 -0
- package/dist/src/types/agent.d.ts.map +1 -1
- package/dist/src/types/framework.d.ts +40 -0
- package/dist/src/types/framework.d.ts.map +1 -1
- package/dist/src/types/module.d.ts +46 -0
- package/dist/src/types/module.d.ts.map +1 -1
- package/dist/src/types/trace.d.ts +51 -0
- package/dist/src/types/trace.d.ts.map +1 -1
- package/dist/test/agent-cache-ttl.test.d.ts +2 -0
- package/dist/test/agent-cache-ttl.test.d.ts.map +1 -0
- package/dist/test/agent-cache-ttl.test.js +25 -0
- package/dist/test/agent-cache-ttl.test.js.map +1 -0
- package/dist/test/conversation-fork-injection-scoping.test.d.ts +2 -0
- package/dist/test/conversation-fork-injection-scoping.test.d.ts.map +1 -0
- package/dist/test/conversation-fork-injection-scoping.test.js +184 -0
- package/dist/test/conversation-fork-injection-scoping.test.js.map +1 -0
- package/dist/test/gather-context-timeout.test.d.ts +2 -0
- package/dist/test/gather-context-timeout.test.d.ts.map +1 -0
- package/dist/test/gather-context-timeout.test.js +120 -0
- package/dist/test/gather-context-timeout.test.js.map +1 -0
- package/dist/test/inference-exhaustion-rewind.test.d.ts +2 -0
- package/dist/test/inference-exhaustion-rewind.test.d.ts.map +1 -0
- package/dist/test/inference-exhaustion-rewind.test.js +181 -0
- package/dist/test/inference-exhaustion-rewind.test.js.map +1 -0
- package/dist/test/inference-failure-observability.test.js +9 -0
- package/dist/test/inference-failure-observability.test.js.map +1 -1
- package/dist/test/maintenance-timer.test.d.ts +2 -0
- package/dist/test/maintenance-timer.test.d.ts.map +1 -0
- package/dist/test/maintenance-timer.test.js +114 -0
- package/dist/test/maintenance-timer.test.js.map +1 -0
- package/dist/test/mcpl-reconnect-events.test.d.ts +2 -0
- package/dist/test/mcpl-reconnect-events.test.d.ts.map +1 -0
- package/dist/test/mcpl-reconnect-events.test.js +119 -0
- package/dist/test/mcpl-reconnect-events.test.js.map +1 -0
- package/dist/test/mcpl-reconnect-reregistration.test.d.ts +2 -0
- package/dist/test/mcpl-reconnect-reregistration.test.d.ts.map +1 -0
- package/dist/test/mcpl-reconnect-reregistration.test.js +181 -0
- package/dist/test/mcpl-reconnect-reregistration.test.js.map +1 -0
- package/dist/test/mcpl-request-timeout.test.d.ts +2 -0
- package/dist/test/mcpl-request-timeout.test.d.ts.map +1 -0
- package/dist/test/mcpl-request-timeout.test.js +123 -0
- package/dist/test/mcpl-request-timeout.test.js.map +1 -0
- package/dist/test/mcpl-runtime-lifecycle.test.d.ts +7 -0
- package/dist/test/mcpl-runtime-lifecycle.test.d.ts.map +1 -0
- package/dist/test/mcpl-runtime-lifecycle.test.js +142 -0
- package/dist/test/mcpl-runtime-lifecycle.test.js.map +1 -0
- package/dist/test/ops-alert.test.d.ts +2 -0
- package/dist/test/ops-alert.test.d.ts.map +1 -0
- package/dist/test/ops-alert.test.js +152 -0
- package/dist/test/ops-alert.test.js.map +1 -0
- package/dist/test/refusal-rewind.test.d.ts +2 -0
- package/dist/test/refusal-rewind.test.d.ts.map +1 -0
- package/dist/test/refusal-rewind.test.js +108 -0
- package/dist/test/refusal-rewind.test.js.map +1 -0
- package/dist/test/state-scaling-e2e.test.d.ts +22 -0
- package/dist/test/state-scaling-e2e.test.d.ts.map +1 -0
- package/dist/test/state-scaling-e2e.test.js +200 -0
- package/dist/test/state-scaling-e2e.test.js.map +1 -0
- package/dist/test/state-scaling.test.d.ts +12 -0
- package/dist/test/state-scaling.test.d.ts.map +1 -0
- package/dist/test/state-scaling.test.js +243 -0
- package/dist/test/state-scaling.test.js.map +1 -0
- package/dist/test/tool-result-content.test.d.ts +2 -0
- package/dist/test/tool-result-content.test.d.ts.map +1 -0
- package/dist/test/tool-result-content.test.js +37 -0
- package/dist/test/tool-result-content.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +3 -2
- package/src/agent.ts +16 -0
- package/src/framework.ts +1339 -99
- package/src/gate/event-gate.ts +56 -4
- package/src/index.ts +10 -0
- package/src/mcpl/channel-registry.ts +34 -3
- package/src/mcpl/push-handler.ts +4 -0
- package/src/mcpl/server-connection.ts +110 -4
- package/src/mcpl/types.ts +19 -1
- package/src/module-registry.ts +113 -4
- package/src/types/agent.ts +38 -0
- package/src/types/framework.ts +44 -0
- package/src/types/module.ts +52 -0
- package/src/types/trace.ts +67 -0
package/dist/src/framework.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { join } from 'node:path';
|
|
2
|
+
import { appendFileSync, mkdirSync } from 'node:fs';
|
|
2
3
|
import { JsStore } from '@animalabs/chronicle';
|
|
3
4
|
import { MembraneError } from '@animalabs/membrane';
|
|
4
5
|
import { ContextManager, PassthroughStrategy } from '@animalabs/context-manager';
|
|
5
6
|
import { ProcessQueueImpl } from './queue.js';
|
|
6
7
|
import { Agent } from './agent.js';
|
|
7
|
-
import { ModuleRegistry } from './module-registry.js';
|
|
8
|
+
import { ModuleRegistry, isStateExistsError } from './module-registry.js';
|
|
8
9
|
import { McplServerRegistry } from './mcpl/server-registry.js';
|
|
9
10
|
import { FeatureSetManager } from './mcpl/feature-set-manager.js';
|
|
10
11
|
import { ScopeManager } from './mcpl/scope-manager.js';
|
|
@@ -24,7 +25,8 @@ const FRAMEWORK_STATE_ID = 'framework/state';
|
|
|
24
25
|
const CONVERSATION_ROUTER_STATE_ID = 'framework/conversation-router';
|
|
25
26
|
const INFERENCE_LOG_ID = 'framework/inference-log';
|
|
26
27
|
const PROCESS_LOG_ID = 'framework/process-log';
|
|
27
|
-
const TURN_CHECKPOINTS_ID = 'framework/turn-checkpoints';
|
|
28
|
+
const TURN_CHECKPOINTS_ID = 'framework/turn-checkpoints'; // legacy single-map layout, read-only fallback
|
|
29
|
+
const TURN_CHECKPOINTS_TREE_ID = 'framework/turn-checkpoints/tree';
|
|
28
30
|
/** Maximum number of turn checkpoints to keep per agent. */
|
|
29
31
|
const MAX_TURN_CHECKPOINTS = 20;
|
|
30
32
|
/**
|
|
@@ -64,6 +66,10 @@ class DefaultErrorPolicy {
|
|
|
64
66
|
}
|
|
65
67
|
/** Default sync interval in milliseconds */
|
|
66
68
|
const DEFAULT_SYNC_INTERVAL_MS = 1000;
|
|
69
|
+
/** Poll pending context-strategy maintenance independently of user activity. */
|
|
70
|
+
const DEFAULT_MAINTENANCE_INTERVAL_MS = 5000;
|
|
71
|
+
/** Bound one pass so a large backlog yields to inference and other agents. */
|
|
72
|
+
const MAINTENANCE_TICKS_PER_PASS = 8;
|
|
67
73
|
/**
|
|
68
74
|
* Extract fields the EventGate cares about from a ProcessEvent. The set of
|
|
69
75
|
* event variants that carry `content`/`mount`/`paths`/`metadata` is open-ended
|
|
@@ -95,6 +101,12 @@ function extractGateFields(event) {
|
|
|
95
101
|
: undefined;
|
|
96
102
|
return { content, metadata, mount, paths };
|
|
97
103
|
}
|
|
104
|
+
/** Cap an API error message for inline use in a rewind marker: keep enough to
|
|
105
|
+
* identify the failure class without pasting a wall of provider JSON into the
|
|
106
|
+
* agent's context. */
|
|
107
|
+
function truncateReason(reason, max = 160) {
|
|
108
|
+
return reason.length <= max ? reason : reason.slice(0, max) + '…';
|
|
109
|
+
}
|
|
98
110
|
export class AgentFramework {
|
|
99
111
|
store;
|
|
100
112
|
ownsStore;
|
|
@@ -120,6 +132,16 @@ export class AgentFramework {
|
|
|
120
132
|
traceListeners = [];
|
|
121
133
|
syncIntervalMs;
|
|
122
134
|
syncTimer = null;
|
|
135
|
+
maintenanceIntervalMs;
|
|
136
|
+
maintenanceTimer = null;
|
|
137
|
+
maintenancePass = null;
|
|
138
|
+
maintenanceRunId = 0;
|
|
139
|
+
currentMaintenanceRun = null;
|
|
140
|
+
maintenanceHistory = [];
|
|
141
|
+
/** Last time we console-warned about stale (busy-requeued) inference requests, per agent. */
|
|
142
|
+
staleWarnAt = new Map();
|
|
143
|
+
/** Per-agent last inference activity (epoch ms), for /healthz + doctor tooling. */
|
|
144
|
+
lastInferenceAt = new Map();
|
|
123
145
|
processLoggingPersist;
|
|
124
146
|
processLoggingBroadcast;
|
|
125
147
|
activeStreams = new Map();
|
|
@@ -130,6 +152,36 @@ export class AgentFramework {
|
|
|
130
152
|
/** N consecutive failed inferences ⇒ the agent is treated as hard-down and
|
|
131
153
|
* escalated loudly to stderr. */
|
|
132
154
|
inferenceFailureEscalationThreshold = 3;
|
|
155
|
+
/** Per-(agent,kind) timestamp of the last ops webhook post. Throttles
|
|
156
|
+
* opsAlert() so a persistent failure re-posts once per cooldown window
|
|
157
|
+
* instead of on every occurrence. */
|
|
158
|
+
opsAlertLastSent = new Map();
|
|
159
|
+
/** Cooldown between webhook posts for the same (agent, kind). */
|
|
160
|
+
opsAlertCooldownMs = 15 * 60_000;
|
|
161
|
+
/** Per-agent refusal bookkeeping for observability — exposed via
|
|
162
|
+
* healthSnapshot() and mirrored to failures.log / ops alerts. */
|
|
163
|
+
refusalStats = new Map();
|
|
164
|
+
/** Per-agent count of consecutive refusals (reset on any non-refusal
|
|
165
|
+
* completion). Distinct from refusalRewinds, which budgets the auto-rewind
|
|
166
|
+
* loop — this one drives ops alerting. */
|
|
167
|
+
refusalStreak = new Map();
|
|
168
|
+
/** Per-agent count of consecutive refusal-driven rewinds in the current turn
|
|
169
|
+
* chain (reset when a turn completes without a refusal). Bounds the auto
|
|
170
|
+
* rewind loop — see refusalHandling + rewindTriggeringTurn. */
|
|
171
|
+
refusalRewinds = new Map();
|
|
172
|
+
/** Per-agent count of poison-history rewinds performed by the hard-down
|
|
173
|
+
* breaker (noteInferenceExhausted). Reset on any successful inference.
|
|
174
|
+
* Bounds the automatic quarantine loop the same way refusalRewinds bounds
|
|
175
|
+
* the refusal loop, so the breaker can never shed the whole history. */
|
|
176
|
+
exhaustionRewinds = new Map();
|
|
177
|
+
/** Per-agent current rewind episode: the single consolidated marker's id and
|
|
178
|
+
* how many turns have been shed so far. One marker per episode, updated in
|
|
179
|
+
* place (see updateRewindMarker); cleared when the episode ends. */
|
|
180
|
+
rewindEpisode = new Map();
|
|
181
|
+
/** Active `/unstick` sessions: an admin-forced rewind-until-clean loop that
|
|
182
|
+
* runs even when the agent's autoRewind toggle is off. Tracks the remaining
|
|
183
|
+
* budget, what was shed (for the report), and where to post the outcome. */
|
|
184
|
+
forcedRewind = new Map();
|
|
133
185
|
/** Name of the primary (non-ephemeral) agent for routing framework messages. */
|
|
134
186
|
primaryAgentName = null;
|
|
135
187
|
// Messages deferred while an agent is waiting_for_tools (to preserve
|
|
@@ -171,17 +223,25 @@ export class AgentFramework {
|
|
|
171
223
|
mcplPrefixMap = new Map();
|
|
172
224
|
/** Maps serverId → McplServerConfig for prefix lookup. */
|
|
173
225
|
mcplServerConfigs = new Map();
|
|
226
|
+
/** Host capabilities advertised during the MCP handshake — stored so servers
|
|
227
|
+
* can be connected at runtime (connectMcplServer) after initialization. */
|
|
228
|
+
mcplHostCapabilities = null;
|
|
229
|
+
/** Inference routing policy from FrameworkConfig — stored so the MCPL
|
|
230
|
+
* subsystem can be lazily initialized by connectMcplServer when the
|
|
231
|
+
* framework started with zero configured servers. */
|
|
232
|
+
mcplInferenceRoutingConfig = null;
|
|
174
233
|
// EventGate (null when FrameworkConfig.gate is omitted)
|
|
175
234
|
eventGate = null;
|
|
176
235
|
// Session-level token usage tracking (always-on)
|
|
177
236
|
usageTracker;
|
|
178
|
-
constructor(store, ownsStore, membrane, inferencePolicy, errorPolicy, syncIntervalMs, processLoggingPersist, processLoggingBroadcast) {
|
|
237
|
+
constructor(store, ownsStore, membrane, inferencePolicy, errorPolicy, syncIntervalMs, maintenanceIntervalMs, processLoggingPersist, processLoggingBroadcast) {
|
|
179
238
|
this.store = store;
|
|
180
239
|
this.ownsStore = ownsStore;
|
|
181
240
|
this.membrane = membrane;
|
|
182
241
|
this.inferencePolicy = inferencePolicy;
|
|
183
242
|
this.errorPolicy = errorPolicy;
|
|
184
243
|
this.syncIntervalMs = syncIntervalMs;
|
|
244
|
+
this.maintenanceIntervalMs = maintenanceIntervalMs;
|
|
185
245
|
this.processLoggingPersist = processLoggingPersist;
|
|
186
246
|
this.processLoggingBroadcast = processLoggingBroadcast;
|
|
187
247
|
this.queue = new ProcessQueueImpl();
|
|
@@ -237,11 +297,14 @@ export class AgentFramework {
|
|
|
237
297
|
catch {
|
|
238
298
|
// Already registered
|
|
239
299
|
}
|
|
300
|
+
// The legacy single-map checkpoint state (TURN_CHECKPOINTS_ID) is no longer
|
|
301
|
+
// registered for new stores — it's read-only fallback data in old ones.
|
|
240
302
|
try {
|
|
241
|
-
store.registerState({ id:
|
|
303
|
+
store.registerState({ id: TURN_CHECKPOINTS_TREE_ID, strategy: 'tree' });
|
|
242
304
|
}
|
|
243
|
-
catch {
|
|
244
|
-
|
|
305
|
+
catch (error) {
|
|
306
|
+
if (!isStateExistsError(error))
|
|
307
|
+
throw error;
|
|
245
308
|
}
|
|
246
309
|
// Process logging config (default: disabled)
|
|
247
310
|
const processLoggingPersist = config.processLogging?.persist ?? false;
|
|
@@ -260,7 +323,7 @@ export class AgentFramework {
|
|
|
260
323
|
// Already registered
|
|
261
324
|
}
|
|
262
325
|
}
|
|
263
|
-
const framework = new AgentFramework(store, ownsStore, config.membrane, config.inferencePolicy ?? new DefaultInferencePolicy(), config.errorPolicy ?? new DefaultErrorPolicy(), config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS, processLoggingPersist, processLoggingBroadcast);
|
|
326
|
+
const framework = new AgentFramework(store, ownsStore, config.membrane, config.inferencePolicy ?? new DefaultInferencePolicy(), config.errorPolicy ?? new DefaultErrorPolicy(), config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS, config.maintenanceIntervalMs ?? DEFAULT_MAINTENANCE_INTERVAL_MS, processLoggingPersist, processLoggingBroadcast);
|
|
264
327
|
// Restore persisted usage data (if any) from prior session
|
|
265
328
|
framework.restoreUsageState();
|
|
266
329
|
// Create agents
|
|
@@ -327,6 +390,9 @@ export class AgentFramework {
|
|
|
327
390
|
});
|
|
328
391
|
framework.livenessWatchdog.start();
|
|
329
392
|
}
|
|
393
|
+
// Stored for lazy MCPL initialization (connectMcplServer on a framework
|
|
394
|
+
// that started with zero configured servers).
|
|
395
|
+
framework.mcplInferenceRoutingConfig = config.inferenceRouting ?? null;
|
|
330
396
|
// Initialize MCPL subsystems if configured
|
|
331
397
|
if (config.mcplServers && config.mcplServers.length > 0) {
|
|
332
398
|
// Validate tool prefixes: no collisions with module names or between servers
|
|
@@ -345,6 +411,30 @@ export class AgentFramework {
|
|
|
345
411
|
}
|
|
346
412
|
await framework.initializeMcpl(config.mcplServers, config.inferenceRouting);
|
|
347
413
|
}
|
|
414
|
+
// Diagnostics: `kill -USR2 <pid>` dumps live wake/inference state to stderr
|
|
415
|
+
// (journal) without a restart — for catching the wake-wedge on the running
|
|
416
|
+
// process. Shows the gate's `inferring` set + buffered-event count (the
|
|
417
|
+
// wedge signature), active streams, and pending inference requests.
|
|
418
|
+
try {
|
|
419
|
+
process.on('SIGUSR2', () => {
|
|
420
|
+
try {
|
|
421
|
+
const gate = framework.eventGate?.inferenceDiagnostics() ?? null;
|
|
422
|
+
console.error('[diagnostics] ' + JSON.stringify({
|
|
423
|
+
at: new Date().toISOString(),
|
|
424
|
+
gate,
|
|
425
|
+
activeStreams: [...framework.activeStreams.keys()],
|
|
426
|
+
pendingRequests: framework.pendingRequests.length,
|
|
427
|
+
agents: [...framework.agents.keys()],
|
|
428
|
+
}));
|
|
429
|
+
}
|
|
430
|
+
catch (err) {
|
|
431
|
+
console.error('[diagnostics] dump failed:', err instanceof Error ? err.message : err);
|
|
432
|
+
}
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
catch {
|
|
436
|
+
// SIGUSR2 not available on this platform — non-fatal.
|
|
437
|
+
}
|
|
348
438
|
return framework;
|
|
349
439
|
}
|
|
350
440
|
/**
|
|
@@ -367,6 +457,15 @@ export class AgentFramework {
|
|
|
367
457
|
}
|
|
368
458
|
}, this.syncIntervalMs);
|
|
369
459
|
}
|
|
460
|
+
if (this.maintenanceIntervalMs > 0) {
|
|
461
|
+
this.maintenanceTimer = setInterval(() => {
|
|
462
|
+
this.startQueuedMaintenance();
|
|
463
|
+
}, this.maintenanceIntervalMs);
|
|
464
|
+
this.maintenanceTimer.unref?.();
|
|
465
|
+
// Do not make a restored queue wait a full interval before its first
|
|
466
|
+
// attempt. MCPL/module initialization has completed before start().
|
|
467
|
+
this.startQueuedMaintenance();
|
|
468
|
+
}
|
|
370
469
|
}
|
|
371
470
|
/**
|
|
372
471
|
* Stop the event loop.
|
|
@@ -390,6 +489,13 @@ export class AgentFramework {
|
|
|
390
489
|
clearInterval(this.syncTimer);
|
|
391
490
|
this.syncTimer = null;
|
|
392
491
|
}
|
|
492
|
+
if (this.maintenanceTimer) {
|
|
493
|
+
clearInterval(this.maintenanceTimer);
|
|
494
|
+
this.maintenanceTimer = null;
|
|
495
|
+
}
|
|
496
|
+
if (this.maintenancePass) {
|
|
497
|
+
await this.maintenancePass;
|
|
498
|
+
}
|
|
393
499
|
if (this.loopPromise) {
|
|
394
500
|
await this.loopPromise;
|
|
395
501
|
}
|
|
@@ -415,6 +521,100 @@ export class AgentFramework {
|
|
|
415
521
|
this.store.close();
|
|
416
522
|
}
|
|
417
523
|
}
|
|
524
|
+
/**
|
|
525
|
+
* Advance queued ContextManager work without requiring a new message.
|
|
526
|
+
*
|
|
527
|
+
* Tool definitions are refreshed first because compression replays stored
|
|
528
|
+
* tool cycles and providers reject those requests when the corresponding
|
|
529
|
+
* schemas are absent. Passes never overlap; each agent gets a bounded drain
|
|
530
|
+
* so maintenance cannot monopolize the framework event loop.
|
|
531
|
+
*/
|
|
532
|
+
startQueuedMaintenance() {
|
|
533
|
+
if (this.maintenancePass || !this.running)
|
|
534
|
+
return;
|
|
535
|
+
const pass = this.runQueuedMaintenance().catch((error) => {
|
|
536
|
+
console.error('[context-maintenance] pass failed:', error instanceof Error ? error.message : error);
|
|
537
|
+
});
|
|
538
|
+
this.maintenancePass = pass;
|
|
539
|
+
void pass.finally(() => {
|
|
540
|
+
if (this.maintenancePass === pass)
|
|
541
|
+
this.maintenancePass = null;
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
async runQueuedMaintenance() {
|
|
545
|
+
const allTools = this.getAllTools();
|
|
546
|
+
const queued = [...this.agents.values()].flatMap((agent) => {
|
|
547
|
+
const cm = agent.getContextManager();
|
|
548
|
+
const tools = allTools.filter((tool) => agent.canUseTool(tool.name));
|
|
549
|
+
cm.setToolDefinitions(tools);
|
|
550
|
+
if (cm.isReady())
|
|
551
|
+
return [];
|
|
552
|
+
const pending = cm.getPendingWork()?.description;
|
|
553
|
+
const progress = this.contextProgress(cm);
|
|
554
|
+
const record = {
|
|
555
|
+
agentName: agent.name,
|
|
556
|
+
startedAt: Date.now(),
|
|
557
|
+
ticks: 0,
|
|
558
|
+
readyBefore: false,
|
|
559
|
+
...(pending ? { pendingBefore: pending } : {}),
|
|
560
|
+
...(progress ? { progressBefore: progress } : {}),
|
|
561
|
+
};
|
|
562
|
+
return [{
|
|
563
|
+
agent,
|
|
564
|
+
cm,
|
|
565
|
+
record,
|
|
566
|
+
}];
|
|
567
|
+
});
|
|
568
|
+
if (queued.length === 0)
|
|
569
|
+
return;
|
|
570
|
+
const run = {
|
|
571
|
+
id: ++this.maintenanceRunId,
|
|
572
|
+
startedAt: Date.now(),
|
|
573
|
+
agents: queued.map(item => item.record),
|
|
574
|
+
};
|
|
575
|
+
this.currentMaintenanceRun = run;
|
|
576
|
+
try {
|
|
577
|
+
await Promise.all(queued.map(async ({ agent, cm, record }) => {
|
|
578
|
+
try {
|
|
579
|
+
for (let i = 0; i < MAINTENANCE_TICKS_PER_PASS && this.running && !cm.isReady(); i++) {
|
|
580
|
+
await cm.tick();
|
|
581
|
+
record.ticks++;
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
catch (error) {
|
|
585
|
+
record.error = error instanceof Error ? error.message : String(error);
|
|
586
|
+
console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
|
|
587
|
+
}
|
|
588
|
+
finally {
|
|
589
|
+
const pending = cm.getPendingWork()?.description;
|
|
590
|
+
record.finishedAt = Date.now();
|
|
591
|
+
record.readyAfter = cm.isReady();
|
|
592
|
+
if (pending)
|
|
593
|
+
record.pendingAfter = pending;
|
|
594
|
+
const progress = this.contextProgress(cm);
|
|
595
|
+
if (progress)
|
|
596
|
+
record.progressAfter = progress;
|
|
597
|
+
}
|
|
598
|
+
}));
|
|
599
|
+
}
|
|
600
|
+
finally {
|
|
601
|
+
run.finishedAt = Date.now();
|
|
602
|
+
run.agents.sort((a, b) => a.agentName.localeCompare(b.agentName));
|
|
603
|
+
this.currentMaintenanceRun = null;
|
|
604
|
+
this.maintenanceHistory.unshift(run);
|
|
605
|
+
if (this.maintenanceHistory.length > 50)
|
|
606
|
+
this.maintenanceHistory.length = 50;
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
contextProgress(cm) {
|
|
610
|
+
const strategy = cm.getStrategy();
|
|
611
|
+
if (typeof strategy.getProgressSnapshot !== 'function')
|
|
612
|
+
return undefined;
|
|
613
|
+
const progress = strategy.getProgressSnapshot();
|
|
614
|
+
return progress && typeof progress === 'object'
|
|
615
|
+
? progress
|
|
616
|
+
: undefined;
|
|
617
|
+
}
|
|
418
618
|
/**
|
|
419
619
|
* Push a process event to the queue.
|
|
420
620
|
*/
|
|
@@ -526,6 +726,27 @@ export class AgentFramework {
|
|
|
526
726
|
}
|
|
527
727
|
return [...moduleTools, ...this.mcplTools, ...channelTools, ...gateTools];
|
|
528
728
|
}
|
|
729
|
+
/** Counts-only context-maintenance diagnostics for authenticated debug UIs. */
|
|
730
|
+
getContextMaintenanceSnapshot() {
|
|
731
|
+
const agents = [...this.agents.values()].map((agent) => {
|
|
732
|
+
const cm = agent.getContextManager();
|
|
733
|
+
const pending = cm.getPendingWork()?.description;
|
|
734
|
+
const progress = this.contextProgress(cm);
|
|
735
|
+
return {
|
|
736
|
+
agentName: agent.name,
|
|
737
|
+
ready: cm.isReady(),
|
|
738
|
+
...(pending ? { pending } : {}),
|
|
739
|
+
...(progress ? { progress } : {}),
|
|
740
|
+
};
|
|
741
|
+
});
|
|
742
|
+
return structuredClone({
|
|
743
|
+
intervalMs: this.maintenanceIntervalMs,
|
|
744
|
+
ticksPerPass: MAINTENANCE_TICKS_PER_PASS,
|
|
745
|
+
current: this.currentMaintenanceRun,
|
|
746
|
+
history: this.maintenanceHistory,
|
|
747
|
+
agents,
|
|
748
|
+
});
|
|
749
|
+
}
|
|
529
750
|
/**
|
|
530
751
|
* Build the membrane-normalized request that WOULD be emitted if `agentName`
|
|
531
752
|
* were activated right now — WITHOUT running inference, opening a stream, or
|
|
@@ -568,8 +789,9 @@ export class AgentFramework {
|
|
|
568
789
|
// Module gatherContext (fail-open, matches startAgentStream)
|
|
569
790
|
try {
|
|
570
791
|
const moduleInjections = await this.moduleRegistry.gatherContext(agentName);
|
|
571
|
-
|
|
572
|
-
|
|
792
|
+
const scopedModuleInjections = this.scopeInjectionsForAgent(agentName, moduleInjections);
|
|
793
|
+
if (scopedModuleInjections.length > 0) {
|
|
794
|
+
injections = scopedModuleInjections;
|
|
573
795
|
}
|
|
574
796
|
}
|
|
575
797
|
catch (error) {
|
|
@@ -580,7 +802,7 @@ export class AgentFramework {
|
|
|
580
802
|
if (this.hookOrchestrator) {
|
|
581
803
|
try {
|
|
582
804
|
const hookParams = this.buildBeforeInferenceParams(agent);
|
|
583
|
-
const hookInjections = await this.hookOrchestrator.beforeInference(hookParams);
|
|
805
|
+
const hookInjections = this.scopeInjectionsForAgent(agentName, await this.hookOrchestrator.beforeInference(hookParams));
|
|
584
806
|
if (hookInjections.length > 0) {
|
|
585
807
|
injections = injections ? [...injections, ...hookInjections] : hookInjections;
|
|
586
808
|
}
|
|
@@ -743,6 +965,7 @@ export class AgentFramework {
|
|
|
743
965
|
clearInterval(completionWatchdog);
|
|
744
966
|
this.offTrace(traceListener);
|
|
745
967
|
this.agents.delete(agent.name);
|
|
968
|
+
this.evictTurnCheckpoints(agent.name);
|
|
746
969
|
};
|
|
747
970
|
// Watchdog: if inference never starts within 30s, the agent is a zombie.
|
|
748
971
|
// This catches cases where the event loop stalled, the agent was
|
|
@@ -1080,6 +1303,181 @@ export class AgentFramework {
|
|
|
1080
1303
|
console.error('[inference-refusal] reaction failed:', err instanceof Error ? err.message : err);
|
|
1081
1304
|
}
|
|
1082
1305
|
}
|
|
1306
|
+
/**
|
|
1307
|
+
* Rewind the single turn that fed a refused generation: redact the newest
|
|
1308
|
+
* message the agent did NOT author (a tool result, or an incoming message)
|
|
1309
|
+
* and inject a metadata-only marker in its place. The marker carries a
|
|
1310
|
+
* *description* of what was withheld (kind + size), never the content — so it
|
|
1311
|
+
* cannot itself re-trip the classifier — while the raw record survives in the
|
|
1312
|
+
* chronicle log for forensics. Returns a record describing the removal, or
|
|
1313
|
+
* null if there is nothing eligible to rewind (e.g. only the agent's own
|
|
1314
|
+
* turns, or a prior system marker, remain).
|
|
1315
|
+
*
|
|
1316
|
+
* Shared by the auto path (on `stop_reason: refusal`) and the `/rewind`
|
|
1317
|
+
* host command.
|
|
1318
|
+
*/
|
|
1319
|
+
shedNewestTurn(agent) {
|
|
1320
|
+
const cm = agent.getContextManager();
|
|
1321
|
+
const all = cm.getAllMessages();
|
|
1322
|
+
const typeOf = (b) => b.type;
|
|
1323
|
+
const hasBlock = (m, t) => Array.isArray(m?.content) && m.content.some((b) => typeOf(b) === t);
|
|
1324
|
+
// Newest message that is not our own episode marker. We shed strictly
|
|
1325
|
+
// newest-first, in sequence — INCLUDING the agent's own turns (poison lives
|
|
1326
|
+
// in tool_use/narration turns too), because newest-first reaches whatever is
|
|
1327
|
+
// poisoning the context if allowed to run deep enough. The one invariant:
|
|
1328
|
+
// shed COMPLETE exchanges. Removing a `tool_result` also removes its paired
|
|
1329
|
+
// `tool_use` assistant turn, so we never leave an orphaned tool_use / signed
|
|
1330
|
+
// `thinking` block — which the API rejects with a 400 ("thinking blocks in
|
|
1331
|
+
// the latest assistant message cannot be modified"). We do NOT add a marker
|
|
1332
|
+
// here; the caller keeps one consolidated marker (updateRewindMarker).
|
|
1333
|
+
let idx = -1;
|
|
1334
|
+
for (let i = all.length - 1; i >= 0; i--) {
|
|
1335
|
+
const meta = (all[i].metadata ?? {});
|
|
1336
|
+
if (meta.system)
|
|
1337
|
+
continue;
|
|
1338
|
+
idx = i;
|
|
1339
|
+
break;
|
|
1340
|
+
}
|
|
1341
|
+
if (idx < 0)
|
|
1342
|
+
return null;
|
|
1343
|
+
const msg = all[idx];
|
|
1344
|
+
const md = (msg.metadata ?? {});
|
|
1345
|
+
const content = Array.isArray(msg.content) ? msg.content : [];
|
|
1346
|
+
const toolBlocks = content.filter((b) => typeOf(b) === 'tool_result');
|
|
1347
|
+
const images = content.filter((b) => typeOf(b) === 'image').length;
|
|
1348
|
+
const textLen = content
|
|
1349
|
+
.filter((b) => typeOf(b) === 'text')
|
|
1350
|
+
.reduce((n, b) => n + String(b.text ?? '').length, 0);
|
|
1351
|
+
// Determine the complete exchange to remove.
|
|
1352
|
+
const removedIds = [msg.id];
|
|
1353
|
+
let kind;
|
|
1354
|
+
let descriptor;
|
|
1355
|
+
if (toolBlocks.length > 0) {
|
|
1356
|
+
kind = 'tool';
|
|
1357
|
+
// Pair the tool_result with the tool_use assistant turn right before it.
|
|
1358
|
+
if (idx - 1 >= 0 && hasBlock(all[idx - 1], 'tool_use')) {
|
|
1359
|
+
removedIds.push(all[idx - 1].id);
|
|
1360
|
+
}
|
|
1361
|
+
const sz = toolBlocks.reduce((n, b) => n + String(b.content ?? '').length, 0);
|
|
1362
|
+
descriptor = `a tool exchange (~${Math.max(1, Math.round(sz / 1024))}KB` +
|
|
1363
|
+
`${images ? `, ${images} image(s)` : ''})`;
|
|
1364
|
+
}
|
|
1365
|
+
else if (md.messageId) {
|
|
1366
|
+
kind = 'human';
|
|
1367
|
+
descriptor = `an incoming message from ${msg.participant} ` +
|
|
1368
|
+
`(${textLen} chars${images ? `, ${images} image(s)` : ''})`;
|
|
1369
|
+
}
|
|
1370
|
+
else {
|
|
1371
|
+
kind = 'other';
|
|
1372
|
+
descriptor = `a ${msg.participant} turn ` +
|
|
1373
|
+
`(${textLen} chars${images ? `, ${images} image(s)` : ''})`;
|
|
1374
|
+
}
|
|
1375
|
+
const discordRef = md.messageId && md.channelId
|
|
1376
|
+
? { channelId: String(md.channelId), messageId: String(md.messageId) }
|
|
1377
|
+
: undefined;
|
|
1378
|
+
for (const id of removedIds)
|
|
1379
|
+
cm.removeMessage(id);
|
|
1380
|
+
return { kind, descriptor, removedIds, discordRef };
|
|
1381
|
+
}
|
|
1382
|
+
/**
|
|
1383
|
+
* Maintain exactly ONE consolidated marker for the current rewind episode,
|
|
1384
|
+
* updated in place as more turns are shed. Six rewinds ⇒ one message that
|
|
1385
|
+
* says "the 6 most recent turns were set aside", not six separate notes — so
|
|
1386
|
+
* the context converges (shed N, add 1) instead of growing, and the marker
|
|
1387
|
+
* sits at the tail giving the model something actionable to answer once the
|
|
1388
|
+
* refusal clears. Returns the running count.
|
|
1389
|
+
*/
|
|
1390
|
+
updateRewindMarker(agent, category, cause = 'refusal') {
|
|
1391
|
+
const cm = agent.getContextManager();
|
|
1392
|
+
const ep = this.rewindEpisode.get(agent.name);
|
|
1393
|
+
const count = (ep?.count ?? 0) + 1;
|
|
1394
|
+
const why = cause === 'refusal'
|
|
1395
|
+
? `the model refused on them (content filter: ${category}). Their content is ` +
|
|
1396
|
+
`not reproduced here (so this note can't re-trigger the filter); the ` +
|
|
1397
|
+
`originals remain in the raw record`
|
|
1398
|
+
: `the model API kept rejecting the conversation on them (${category}). ` +
|
|
1399
|
+
`Their content is not reproduced here (so this note can't re-trigger the ` +
|
|
1400
|
+
`rejection); the originals remain in the raw record`;
|
|
1401
|
+
const text = `[refusal-rewind] The ${count} most recent turn(s) were set aside because ` +
|
|
1402
|
+
`${why}. You are clear to continue — if you ` +
|
|
1403
|
+
`were mid-task, take a different approach; otherwise carry on with whatever ` +
|
|
1404
|
+
`is now in front of you, or briefly acknowledge the gap and ask what's next.`;
|
|
1405
|
+
const blocks = [{ type: 'text', text }];
|
|
1406
|
+
if (ep) {
|
|
1407
|
+
cm.editMessage(ep.markerId, blocks);
|
|
1408
|
+
this.rewindEpisode.set(agent.name, { markerId: ep.markerId, count, category });
|
|
1409
|
+
}
|
|
1410
|
+
else {
|
|
1411
|
+
const id = cm.addMessage('user', blocks, {
|
|
1412
|
+
system: true, kind: 'refusal-rewind', category, count, cause,
|
|
1413
|
+
});
|
|
1414
|
+
this.rewindEpisode.set(agent.name, { markerId: id, count, category });
|
|
1415
|
+
}
|
|
1416
|
+
return count;
|
|
1417
|
+
}
|
|
1418
|
+
/**
|
|
1419
|
+
* Announce a refusal-rewind on the conversational surface (Discord), used
|
|
1420
|
+
* when the withheld turn was a *human* message so it isn't dropped silently.
|
|
1421
|
+
* Best-effort; mirrors reactToRefusal's locus resolution.
|
|
1422
|
+
*/
|
|
1423
|
+
async announceRewind(agentName, rec, category) {
|
|
1424
|
+
try {
|
|
1425
|
+
const incoming = this.channelRegistry?.buildChannelContext()?.incoming;
|
|
1426
|
+
if (!incoming)
|
|
1427
|
+
return;
|
|
1428
|
+
const parts = incoming.channelId.split(':');
|
|
1429
|
+
if (parts[0] !== 'discord')
|
|
1430
|
+
return;
|
|
1431
|
+
const channelId = parts[parts.length - 1];
|
|
1432
|
+
const serverId = this.channelRegistry?.getChannelServerId(incoming.channelId);
|
|
1433
|
+
const server = serverId ? this.mcplServerRegistry?.getServer(serverId) : null;
|
|
1434
|
+
if (!server)
|
|
1435
|
+
return;
|
|
1436
|
+
await server.sendToolsCall('send_message', {
|
|
1437
|
+
channelId,
|
|
1438
|
+
content: `⚠️ I had to set aside ${rec.descriptor} — it tripped a content ` +
|
|
1439
|
+
`filter (${category}), so it's withheld from my context and I'm ` +
|
|
1440
|
+
`continuing without it. If it was important, please rephrase or re-send.`,
|
|
1441
|
+
});
|
|
1442
|
+
}
|
|
1443
|
+
catch (err) {
|
|
1444
|
+
console.error('[refusal-rewind] announce failed:', err instanceof Error ? err.message : err);
|
|
1445
|
+
}
|
|
1446
|
+
}
|
|
1447
|
+
/**
|
|
1448
|
+
* Conclude an active `/unstick` session: post the outcome (what was shed and
|
|
1449
|
+
* whether the model stopped refusing) to the channel the command came from,
|
|
1450
|
+
* then clear the session. Idempotent — a no-op if there's no session.
|
|
1451
|
+
*/
|
|
1452
|
+
finishUnstick(agentName, success, category) {
|
|
1453
|
+
const s = this.forcedRewind.get(agentName);
|
|
1454
|
+
if (!s)
|
|
1455
|
+
return;
|
|
1456
|
+
this.forcedRewind.delete(agentName);
|
|
1457
|
+
const n = s.removed.length;
|
|
1458
|
+
const list = n
|
|
1459
|
+
? '\n' + s.removed.map((r) => `• ${r.descriptor}`).join('\n')
|
|
1460
|
+
: '';
|
|
1461
|
+
const content = success
|
|
1462
|
+
? `🔧 Unstuck **${agentName}** — shed ${n} turn(s); the model responded.${list}`
|
|
1463
|
+
: `⚠️ Couldn't unstick **${agentName}** — still refusing after ${n} rewind(s)` +
|
|
1464
|
+
`${category ? ` (category=${category})` : ''}.${list}`;
|
|
1465
|
+
console.error(`[unstick] agent=${agentName} ${success ? 'succeeded' : 'gave up'} after ${n} rewind(s)`);
|
|
1466
|
+
if (s.channelId)
|
|
1467
|
+
void this.postToChannel(s.serverId, s.channelId, content);
|
|
1468
|
+
}
|
|
1469
|
+
/** Best-effort send_message to a raw channel via a named MCPL server. */
|
|
1470
|
+
async postToChannel(serverId, channelId, content) {
|
|
1471
|
+
try {
|
|
1472
|
+
const server = this.mcplServerRegistry?.getServer(serverId);
|
|
1473
|
+
if (!server)
|
|
1474
|
+
return;
|
|
1475
|
+
await server.sendToolsCall('send_message', { channelId, content });
|
|
1476
|
+
}
|
|
1477
|
+
catch (err) {
|
|
1478
|
+
console.error('[unstick] report post failed:', err instanceof Error ? err.message : err);
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1083
1481
|
/**
|
|
1084
1482
|
* Handle a `host/command` request from an MCPL surface server (e.g. a
|
|
1085
1483
|
* Discord slash command). Currently supports:
|
|
@@ -1090,13 +1488,41 @@ export class AgentFramework {
|
|
|
1090
1488
|
* `previewActivation` render (no inference, no Chronicle writes).
|
|
1091
1489
|
*/
|
|
1092
1490
|
async handleHostCommand(serverId, params) {
|
|
1093
|
-
if (params.command !== 'undo' && params.command !== 'hide') {
|
|
1491
|
+
if (params.command !== 'undo' && params.command !== 'hide' && params.command !== 'unstick') {
|
|
1094
1492
|
return { ok: false, error: `Unknown host command: ${String(params.command)}` };
|
|
1095
1493
|
}
|
|
1096
1494
|
const agentName = params.agentName ?? [...this.agents.keys()][0];
|
|
1097
1495
|
if (!agentName || !this.agents.has(agentName)) {
|
|
1098
1496
|
return { ok: false, error: `Unknown agent: ${String(agentName)}` };
|
|
1099
1497
|
}
|
|
1498
|
+
// unstick: force the refusal-rewind loop on demand (even if the agent's
|
|
1499
|
+
// autoRewind toggle is off). Redacts the turn that fed the refusal and
|
|
1500
|
+
// re-runs, up to `cap` times, until the model stops refusing. Kicks the
|
|
1501
|
+
// loop and returns immediately; the outcome (what was shed + whether it
|
|
1502
|
+
// cleared) is posted to the channel when the chain resolves.
|
|
1503
|
+
if (params.command === 'unstick') {
|
|
1504
|
+
const agent = this.agents.get(agentName);
|
|
1505
|
+
if (agent.state.status !== 'idle') {
|
|
1506
|
+
return { ok: false, error: `Cannot unstick while agent is ${agent.state.status}` };
|
|
1507
|
+
}
|
|
1508
|
+
const cap = Math.max(1, Math.min(10, Math.floor(params.maxRewinds ?? agent.refusalHandling?.maxRewinds ?? 3)));
|
|
1509
|
+
this.forcedRewind.set(agentName, {
|
|
1510
|
+
remaining: cap,
|
|
1511
|
+
removed: [],
|
|
1512
|
+
serverId,
|
|
1513
|
+
channelId: params.channelId ?? '',
|
|
1514
|
+
});
|
|
1515
|
+
this.refusalRewinds.set(agentName, 0);
|
|
1516
|
+
this.pendingRequests.push({
|
|
1517
|
+
agentName,
|
|
1518
|
+
reason: 'unstick',
|
|
1519
|
+
source: 'framework',
|
|
1520
|
+
timestamp: Date.now(),
|
|
1521
|
+
});
|
|
1522
|
+
console.error(`[unstick] agent=${agentName} started cap=${cap} ` +
|
|
1523
|
+
`by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`);
|
|
1524
|
+
return { ok: true, started: true, cap };
|
|
1525
|
+
}
|
|
1100
1526
|
// hide: redact a single message (or an inclusive range) from the active
|
|
1101
1527
|
// branch, addressed by Discord message id. Unlike undo this is a
|
|
1102
1528
|
// removal-in-place (chronicle redact), not a branch rewind.
|
|
@@ -1615,18 +2041,86 @@ export class AgentFramework {
|
|
|
1615
2041
|
}
|
|
1616
2042
|
this.saveTurnCheckpoints(agentName, checkpoints);
|
|
1617
2043
|
}
|
|
2044
|
+
/**
|
|
2045
|
+
* Checkpoints live in one Tree state (TURN_CHECKPOINTS_TREE_ID) keyed by agent
|
|
2046
|
+
* name, each key pointing at a JSON blob of that agent's ≤MAX_TURN_CHECKPOINTS
|
|
2047
|
+
* list. This kills two unbounded-in-agents-ever dimensions at once:
|
|
2048
|
+
*
|
|
2049
|
+
* - the original single Record<agentName, list> map was rewritten whole on
|
|
2050
|
+
* every turn of every agent and never evicted a departed subagent's key;
|
|
2051
|
+
* - the intermediate design (one state slot per agent) fixed the writes but
|
|
2052
|
+
* leaked a permanent chronicle state *registration* per spawn — chronicle
|
|
2053
|
+
* has no deregistration, and the state index is rewritten and fsynced on
|
|
2054
|
+
* every sync tick, so per-tick cost grew with fleet history anyway.
|
|
2055
|
+
*
|
|
2056
|
+
* A tree gives per-key O(entry) writes, real key removal (treeRemove), and
|
|
2057
|
+
* exactly one registration for the life of the store. The legacy map state
|
|
2058
|
+
* (TURN_CHECKPOINTS_ID) is kept as a read-only fallback for stores written
|
|
2059
|
+
* before the split; eviction tombstones legacy keys so a reused agent name
|
|
2060
|
+
* can't inherit a dead agent's checkpoints through the fallback.
|
|
2061
|
+
*/
|
|
2062
|
+
legacyCheckpointKeys = null;
|
|
2063
|
+
hasLegacyCheckpoints(agentName) {
|
|
2064
|
+
if (!this.legacyCheckpointKeys) {
|
|
2065
|
+
const legacy = this.store.getStateJson(TURN_CHECKPOINTS_ID);
|
|
2066
|
+
this.legacyCheckpointKeys = new Set(legacy && typeof legacy === 'object' ? Object.keys(legacy) : []);
|
|
2067
|
+
}
|
|
2068
|
+
return this.legacyCheckpointKeys.has(agentName);
|
|
2069
|
+
}
|
|
1618
2070
|
getTurnCheckpoints(agentName) {
|
|
1619
|
-
const
|
|
1620
|
-
if (
|
|
2071
|
+
const entry = this.store.treeGet(TURN_CHECKPOINTS_TREE_ID, agentName);
|
|
2072
|
+
if (entry) {
|
|
2073
|
+
const blob = this.store.getBlob(entry.blobHash);
|
|
2074
|
+
if (!blob)
|
|
2075
|
+
return [];
|
|
2076
|
+
try {
|
|
2077
|
+
const parsed = JSON.parse(blob.toString());
|
|
2078
|
+
return Array.isArray(parsed) ? parsed : [];
|
|
2079
|
+
}
|
|
2080
|
+
catch {
|
|
2081
|
+
return [];
|
|
2082
|
+
}
|
|
2083
|
+
}
|
|
2084
|
+
if (!this.hasLegacyCheckpoints(agentName))
|
|
1621
2085
|
return [];
|
|
1622
|
-
const
|
|
1623
|
-
|
|
2086
|
+
const legacy = this.store.getStateJson(TURN_CHECKPOINTS_ID);
|
|
2087
|
+
const list = legacy?.[agentName];
|
|
2088
|
+
return Array.isArray(list) ? [...list] : [];
|
|
1624
2089
|
}
|
|
1625
2090
|
saveTurnCheckpoints(agentName, checkpoints) {
|
|
1626
|
-
const
|
|
1627
|
-
const
|
|
1628
|
-
|
|
1629
|
-
|
|
2091
|
+
const bytes = Buffer.from(JSON.stringify(checkpoints));
|
|
2092
|
+
const blobHash = this.store.storeBlob(bytes, 'application/json');
|
|
2093
|
+
this.store.treeSet(TURN_CHECKPOINTS_TREE_ID, agentName, {
|
|
2094
|
+
blobHash,
|
|
2095
|
+
size: bytes.length,
|
|
2096
|
+
mode: 0o644,
|
|
2097
|
+
});
|
|
2098
|
+
}
|
|
2099
|
+
/**
|
|
2100
|
+
* Drop a departed agent's checkpoint tree key and turn/redo bookkeeping.
|
|
2101
|
+
* Without this, spawn-and-dispose agents leave a key (and in-memory map
|
|
2102
|
+
* entries) behind for the life of the store/session.
|
|
2103
|
+
*/
|
|
2104
|
+
evictTurnCheckpoints(agentName) {
|
|
2105
|
+
this.turnCounters.delete(agentName);
|
|
2106
|
+
this.redoStacks.delete(agentName);
|
|
2107
|
+
// Diagnostics maps are also keyed by agent name and never evicted —
|
|
2108
|
+
// spawn-and-dispose fleets would grow them for the session's life.
|
|
2109
|
+
this.staleWarnAt.delete(agentName);
|
|
2110
|
+
this.lastInferenceAt.delete(agentName);
|
|
2111
|
+
const entry = this.store.treeGet(TURN_CHECKPOINTS_TREE_ID, agentName);
|
|
2112
|
+
if (this.hasLegacyCheckpoints(agentName)) {
|
|
2113
|
+
// A bare treeRemove would resurrect the legacy map's list through the
|
|
2114
|
+
// fallback next time this name is reused — shadow it with an empty
|
|
2115
|
+
// tombstone instead. size <= 2 ⇔ the blob is already "[]".
|
|
2116
|
+
if (!entry || entry.size > 2) {
|
|
2117
|
+
this.saveTurnCheckpoints(agentName, []);
|
|
2118
|
+
}
|
|
2119
|
+
return;
|
|
2120
|
+
}
|
|
2121
|
+
if (entry) {
|
|
2122
|
+
this.store.treeRemove(TURN_CHECKPOINTS_TREE_ID, agentName);
|
|
2123
|
+
}
|
|
1630
2124
|
}
|
|
1631
2125
|
/**
|
|
1632
2126
|
* Run until the queue is empty and all agents are idle.
|
|
@@ -1634,6 +2128,9 @@ export class AgentFramework {
|
|
|
1634
2128
|
*/
|
|
1635
2129
|
async runUntilIdle() {
|
|
1636
2130
|
while (!this.queue.isEmpty ||
|
|
2131
|
+
// Direct inference requests (e.g. runEphemeralToCompletion) bypass the
|
|
2132
|
+
// event queue — without this the loop can exit before they're drained.
|
|
2133
|
+
this.pendingRequests.length > 0 ||
|
|
1637
2134
|
this.activeStreams.size > 0 ||
|
|
1638
2135
|
Array.from(this.agents.values()).some((a) => a.state.status !== 'idle')) {
|
|
1639
2136
|
await this.processNextEvent();
|
|
@@ -2083,6 +2580,7 @@ export class AgentFramework {
|
|
|
2083
2580
|
this.agents.delete(agentName);
|
|
2084
2581
|
this.agentConfigs.delete(agentName);
|
|
2085
2582
|
this.conversationAgentHomes.delete(agentName);
|
|
2583
|
+
this.evictTurnCheckpoints(agentName);
|
|
2086
2584
|
this.emitTrace({
|
|
2087
2585
|
type: 'mcpl:conversation-disposed',
|
|
2088
2586
|
agentName,
|
|
@@ -2257,6 +2755,7 @@ export class AgentFramework {
|
|
|
2257
2755
|
requestCount: requests.length,
|
|
2258
2756
|
oldestRequestAge: now - oldest,
|
|
2259
2757
|
});
|
|
2758
|
+
console.error(`[inference-dropped] agent=${agentName} reason=agent_not_found requests=${requests.length}`);
|
|
2260
2759
|
continue;
|
|
2261
2760
|
}
|
|
2262
2761
|
// Skip if agent is busy (inferring, streaming, or waiting for tools)
|
|
@@ -2271,12 +2770,23 @@ export class AgentFramework {
|
|
|
2271
2770
|
requestCount: requests.length,
|
|
2272
2771
|
oldestRequestAge: now - oldest,
|
|
2273
2772
|
});
|
|
2773
|
+
// Loud (but throttled) note that requests are waiting on a busy agent.
|
|
2774
|
+
if ((this.staleWarnAt.get(agentName) ?? 0) < now - 60_000) {
|
|
2775
|
+
this.staleWarnAt.set(agentName, now);
|
|
2776
|
+
console.error(`[inference-stale] agent=${agentName} busy (${agent.state.status}) — ` +
|
|
2777
|
+
`${requests.length} request(s) waiting ${Math.round((now - oldest) / 1000)}s`);
|
|
2778
|
+
}
|
|
2274
2779
|
}
|
|
2275
2780
|
this.pendingRequests.push(...requests);
|
|
2276
2781
|
continue;
|
|
2277
2782
|
}
|
|
2278
2783
|
// Check policy
|
|
2279
2784
|
if (!this.inferencePolicy.shouldInfer(agentName, requests, state)) {
|
|
2785
|
+
// Loud drop: a queued request that dies here is otherwise invisible —
|
|
2786
|
+
// the 2026-07-09 mythos "not responding" diagnosis burned hours on
|
|
2787
|
+
// exactly this class of silent drop. One stderr line per drop.
|
|
2788
|
+
console.error(`[inference-dropped] agent=${agentName} reason=policy-skip ` +
|
|
2789
|
+
`requests=${requests.length} triggers=${requests.map((r) => r.reason).join(',')}`);
|
|
2280
2790
|
continue;
|
|
2281
2791
|
}
|
|
2282
2792
|
// Start streaming inference (non-blocking — driveStream runs in background)
|
|
@@ -2313,20 +2823,20 @@ export class AgentFramework {
|
|
|
2313
2823
|
}
|
|
2314
2824
|
this.emitTrace({ type: 'inference:started', agentName: agent.name });
|
|
2315
2825
|
this.eventGate?.onInferenceStarted(agent.name);
|
|
2826
|
+
this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), startedAt: Date.now() });
|
|
2316
2827
|
try {
|
|
2317
2828
|
const allTools = this.getAllTools();
|
|
2318
2829
|
const tools = allTools.filter((t) => agent.canUseTool(t.name));
|
|
2319
2830
|
// Gather context from modules (pull-based) and MCPL hooks (push-based)
|
|
2320
2831
|
// Both produce ContextInjection[] that get merged before inference.
|
|
2321
2832
|
let injections;
|
|
2322
|
-
// Module gatherContext (fail-open,
|
|
2323
|
-
//
|
|
2324
|
-
//
|
|
2325
|
-
//
|
|
2326
|
-
//
|
|
2327
|
-
// agent itself, or conversation forks will see cross-channel content.
|
|
2833
|
+
// Module gatherContext (fail-open, per-module timeout — the module's
|
|
2834
|
+
// contextTimeoutMs, else the registry default). Injections are
|
|
2835
|
+
// channel-scoped via scopeInjectionsForAgent, matching the MCPL hook
|
|
2836
|
+
// injections below, so conversation forks don't see cross-channel
|
|
2837
|
+
// content.
|
|
2328
2838
|
try {
|
|
2329
|
-
const moduleInjections = await this.moduleRegistry.gatherContext(agent.name);
|
|
2839
|
+
const moduleInjections = this.scopeInjectionsForAgent(agent.name, await this.moduleRegistry.gatherContext(agent.name));
|
|
2330
2840
|
if (moduleInjections.length > 0) {
|
|
2331
2841
|
injections = moduleInjections;
|
|
2332
2842
|
}
|
|
@@ -2370,6 +2880,13 @@ export class AgentFramework {
|
|
|
2370
2880
|
type: 'inference:exhausted',
|
|
2371
2881
|
agentName: agent.name,
|
|
2372
2882
|
error: err.message,
|
|
2883
|
+
// Drives the poison-history breaker. `retryable` is kept for
|
|
2884
|
+
// observability, but the breaker gates on `errorType` — only an
|
|
2885
|
+
// `invalid_request` (a 400-class rejection of the history itself)
|
|
2886
|
+
// means retrying the same context can never succeed. auth/abort/
|
|
2887
|
+
// context_length are also non-retryable but must NOT cost history.
|
|
2888
|
+
retryable: err instanceof MembraneError ? err.retryable : undefined,
|
|
2889
|
+
errorType: err instanceof MembraneError ? err.type : undefined,
|
|
2373
2890
|
});
|
|
2374
2891
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
2375
2892
|
if (action.emit) {
|
|
@@ -2608,7 +3125,75 @@ export class AgentFramework {
|
|
|
2608
3125
|
const category = stopDetails?.category ?? 'unknown';
|
|
2609
3126
|
console.error(`[inference-refusal] agent=${agent.name} category=${category}` +
|
|
2610
3127
|
(stopDetails?.explanation ? ` explanation=${stopDetails.explanation}` : ''));
|
|
2611
|
-
|
|
3128
|
+
this.noteRefusal(agent.name, category, tokenUsage);
|
|
3129
|
+
// Rewind: excise the turn that fed the refusal, drop a
|
|
3130
|
+
// metadata-only marker, and retry — keeping the agent on its own
|
|
3131
|
+
// model instead of substituting a fallback. Driven either by an
|
|
3132
|
+
// admin `/unstick` (forced session) or the agent's autoRewind
|
|
3133
|
+
// config; both are bounded.
|
|
3134
|
+
let handledByRewind = false;
|
|
3135
|
+
const rh = agent.refusalHandling;
|
|
3136
|
+
const forced = this.forcedRewind.get(agent.name);
|
|
3137
|
+
// Shed exactly one more (newest, in-sequence) turn and keep the
|
|
3138
|
+
// single episode marker current. `budgetLeft` bounds the loop.
|
|
3139
|
+
const doRewind = (budgetLeft, onStep, onGiveUp) => {
|
|
3140
|
+
if (!budgetLeft) {
|
|
3141
|
+
onGiveUp();
|
|
3142
|
+
return;
|
|
3143
|
+
}
|
|
3144
|
+
const rec = this.shedNewestTurn(agent);
|
|
3145
|
+
if (!rec) {
|
|
3146
|
+
onGiveUp();
|
|
3147
|
+
return;
|
|
3148
|
+
}
|
|
3149
|
+
const count = this.updateRewindMarker(agent, category);
|
|
3150
|
+
onStep(count);
|
|
3151
|
+
if (forced)
|
|
3152
|
+
forced.removed.push(rec);
|
|
3153
|
+
this.pendingRequests.push({
|
|
3154
|
+
agentName: agent.name,
|
|
3155
|
+
reason: forced ? 'unstick-retry' : 'refusal-rewind-retry',
|
|
3156
|
+
source: 'framework',
|
|
3157
|
+
timestamp: Date.now(),
|
|
3158
|
+
});
|
|
3159
|
+
handledByRewind = true;
|
|
3160
|
+
};
|
|
3161
|
+
if (forced) {
|
|
3162
|
+
doRewind(forced.remaining > 0, (count) => {
|
|
3163
|
+
forced.remaining -= 1;
|
|
3164
|
+
console.error(`[unstick] agent=${agent.name} shed ${count} turn(s) ` +
|
|
3165
|
+
`(remaining ${forced.remaining})`);
|
|
3166
|
+
}, () => this.finishUnstick(agent.name, false, category));
|
|
3167
|
+
}
|
|
3168
|
+
else if (rh?.autoRewind) {
|
|
3169
|
+
const cap = Math.max(1, rh.maxRewinds ?? 3);
|
|
3170
|
+
const used = this.refusalRewinds.get(agent.name) ?? 0;
|
|
3171
|
+
doRewind(used < cap, (count) => {
|
|
3172
|
+
this.refusalRewinds.set(agent.name, used + 1);
|
|
3173
|
+
console.error(`[refusal-rewind] agent=${agent.name} shed ${count} turn(s) so far ` +
|
|
3174
|
+
`(cap ${cap})`);
|
|
3175
|
+
}, () => {
|
|
3176
|
+
console.error(`[refusal-rewind] agent=${agent.name} gave up: ` +
|
|
3177
|
+
`${used >= cap ? `cap ${cap} reached` : 'nothing left to shed'}`);
|
|
3178
|
+
this.refusalRewinds.set(agent.name, 0);
|
|
3179
|
+
this.rewindEpisode.delete(agent.name);
|
|
3180
|
+
});
|
|
3181
|
+
}
|
|
3182
|
+
if (!handledByRewind)
|
|
3183
|
+
void this.reactToRefusal(agent.name, category);
|
|
3184
|
+
}
|
|
3185
|
+
else {
|
|
3186
|
+
// A turn that completed WITHOUT a refusal ends the rewind episode:
|
|
3187
|
+
// the model responded. Leave the consolidated marker in place as
|
|
3188
|
+
// the durable record; just clear the per-episode counters.
|
|
3189
|
+
if (this.forcedRewind.has(agent.name)) {
|
|
3190
|
+
this.finishUnstick(agent.name, true);
|
|
3191
|
+
}
|
|
3192
|
+
if (this.refusalRewinds.get(agent.name)) {
|
|
3193
|
+
this.refusalRewinds.set(agent.name, 0);
|
|
3194
|
+
}
|
|
3195
|
+
this.rewindEpisode.delete(agent.name);
|
|
3196
|
+
this.refusalStreak.delete(agent.name);
|
|
2612
3197
|
}
|
|
2613
3198
|
// Dispatch speech (and thoughts if any)
|
|
2614
3199
|
if (speechContent.length > 0 || thoughts.length > 0) {
|
|
@@ -2692,11 +3277,19 @@ export class AgentFramework {
|
|
|
2692
3277
|
`(${silenced ? 'silencing tool / explicit send' : 'no prose'})`);
|
|
2693
3278
|
}
|
|
2694
3279
|
else {
|
|
2695
|
-
|
|
3280
|
+
// Snapshot the locus ONCE. This loop dispatches AFTER the agent
|
|
3281
|
+
// is idle (see PR #32 note below), so a queued inbound from
|
|
3282
|
+
// another channel can start a NEW turn and overwrite the per-agent
|
|
3283
|
+
// triggering channel (`activeTriggerChannels`) BETWEEN segments —
|
|
3284
|
+
// splitting one turn's reply across channels (the residual
|
|
3285
|
+
// multi-segment race left by 83dedf8's per-agent locus). Pin every
|
|
3286
|
+
// segment of THIS turn to the channel resolved right now.
|
|
3287
|
+
const turnLocus = this.channelRegistry.resolveLocus(agent.name);
|
|
3288
|
+
console.error(`[routing] ${agent.name}: tool-call turn [${toolNames.join(', ')}] -> routing ${segments.length} prose segment(s) in order -> ${turnLocus ?? '(default)'}`);
|
|
2696
3289
|
// Deliver sequentially (await each) so the segments land in order.
|
|
2697
3290
|
for (const seg of segments) {
|
|
2698
3291
|
try {
|
|
2699
|
-
await this.channelRegistry.routeSpeech(agent.name, seg);
|
|
3292
|
+
await this.channelRegistry.routeSpeech(agent.name, seg, turnLocus);
|
|
2700
3293
|
}
|
|
2701
3294
|
catch (err) {
|
|
2702
3295
|
console.error('speech routing failed:', err);
|
|
@@ -2742,6 +3335,12 @@ export class AgentFramework {
|
|
|
2742
3335
|
type: 'inference:exhausted',
|
|
2743
3336
|
agentName: agent.name,
|
|
2744
3337
|
error: err.message,
|
|
3338
|
+
// Drives the poison-history breaker. `retryable` is kept for
|
|
3339
|
+
// observability, but the breaker gates on `errorType` — only an
|
|
3340
|
+
// `invalid_request` (a 400-class rejection of the history
|
|
3341
|
+
// itself) means retrying the same context can never succeed.
|
|
3342
|
+
retryable: err instanceof MembraneError ? err.retryable : undefined,
|
|
3343
|
+
errorType: err instanceof MembraneError ? err.type : undefined,
|
|
2745
3344
|
});
|
|
2746
3345
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
2747
3346
|
if (action.emit) {
|
|
@@ -2755,18 +3354,51 @@ export class AgentFramework {
|
|
|
2755
3354
|
// Only reset if this is still the active stream (a budget restart
|
|
2756
3355
|
// may have already started a new stream, bumping streamId)
|
|
2757
3356
|
if (agent.streamId === myStreamId) {
|
|
3357
|
+
const durationMs = Date.now() - startTime;
|
|
2758
3358
|
agent.reset();
|
|
2759
3359
|
this.emitTrace({
|
|
2760
3360
|
type: 'inference:exhausted',
|
|
2761
3361
|
agentName: agent.name,
|
|
2762
3362
|
error: `Stream aborted: ${reason}`,
|
|
2763
3363
|
});
|
|
3364
|
+
// Postmortem 2026-05-28 P2 #7: persist the abort to the
|
|
3365
|
+
// inference log so future investigations can attribute the
|
|
3366
|
+
// terminal cause without relying on live in-memory reducer
|
|
3367
|
+
// state. Without this, abort-terminated inferences are
|
|
3368
|
+
// invisible to forensic queries (only request-side telemetry
|
|
3369
|
+
// via llm-calls.jsonl shows them, and only by absence).
|
|
3370
|
+
this.logInference({
|
|
3371
|
+
timestamp: startTime,
|
|
3372
|
+
agentName: agent.name,
|
|
3373
|
+
requestId,
|
|
3374
|
+
success: false,
|
|
3375
|
+
error: `Stream aborted: ${reason}`,
|
|
3376
|
+
request: compiledRequest ?? { note: 'streaming request aborted' },
|
|
3377
|
+
durationMs,
|
|
3378
|
+
});
|
|
2764
3379
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
2765
3380
|
}
|
|
2766
3381
|
break;
|
|
2767
3382
|
}
|
|
2768
|
-
case 'usage':
|
|
3383
|
+
case 'usage': {
|
|
2769
3384
|
agent.lastStreamInputTokens = event.usage.inputTokens;
|
|
3385
|
+
// Closed-loop estimator calibration (2026-07-12). Sample the REAL
|
|
3386
|
+
// prefix size of THIS API call (fresh + cache write + cache read)
|
|
3387
|
+
// and hand it to the context strategy, which accepts exactly one
|
|
3388
|
+
// sample per compile (its arm-once gate) and rejects out-of-band
|
|
3389
|
+
// ratios. It must be sampled HERE, per call: `response.details.
|
|
3390
|
+
// usage` at turn completion is CUMULATIVE across the tool-use
|
|
3391
|
+
// loop (5 calls x ~160k reported as 884k), which is not a
|
|
3392
|
+
// window-shaped number and drove the multiplier to 2.37 before
|
|
3393
|
+
// the guards caught it.
|
|
3394
|
+
try {
|
|
3395
|
+
const realTotal = (event.usage.inputTokens ?? 0) +
|
|
3396
|
+
(event.usage.cacheCreationTokens ?? 0) +
|
|
3397
|
+
(event.usage.cacheReadTokens ?? 0);
|
|
3398
|
+
const strat = agent.getContextManager?.()?.getStrategy?.();
|
|
3399
|
+
strat?.reportRealInputTokens?.(realTotal);
|
|
3400
|
+
}
|
|
3401
|
+
catch { /* calibration is best-effort */ }
|
|
2770
3402
|
this.emitTrace({
|
|
2771
3403
|
type: 'inference:usage',
|
|
2772
3404
|
agentName: agent.name,
|
|
@@ -2778,6 +3410,7 @@ export class AgentFramework {
|
|
|
2778
3410
|
},
|
|
2779
3411
|
});
|
|
2780
3412
|
break;
|
|
3413
|
+
}
|
|
2781
3414
|
}
|
|
2782
3415
|
}
|
|
2783
3416
|
}
|
|
@@ -2785,6 +3418,7 @@ export class AgentFramework {
|
|
|
2785
3418
|
// Stream itself threw (unexpected) — no retry path here, so also emit
|
|
2786
3419
|
// inference:exhausted so ephemeral agent promises can settle.
|
|
2787
3420
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
3421
|
+
const durationMs = Date.now() - startTime;
|
|
2788
3422
|
this.emitTrace({
|
|
2789
3423
|
type: 'inference:failed',
|
|
2790
3424
|
agentName: agent.name,
|
|
@@ -2796,10 +3430,31 @@ export class AgentFramework {
|
|
|
2796
3430
|
agentName: agent.name,
|
|
2797
3431
|
error: err.message,
|
|
2798
3432
|
});
|
|
3433
|
+
// Postmortem 2026-05-28 P2 #7: catch-path failures (stream itself
|
|
3434
|
+
// threw) were previously only visible via in-memory trace listeners.
|
|
3435
|
+
// Persist to inference log for forensic attribution.
|
|
3436
|
+
this.logInference({
|
|
3437
|
+
timestamp: startTime,
|
|
3438
|
+
agentName: agent.name,
|
|
3439
|
+
requestId,
|
|
3440
|
+
success: false,
|
|
3441
|
+
error: `Stream threw: ${err.message}`,
|
|
3442
|
+
request: compiledRequest ?? { note: 'streaming request threw' },
|
|
3443
|
+
durationMs,
|
|
3444
|
+
});
|
|
2799
3445
|
agent.reset();
|
|
2800
3446
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
2801
3447
|
}
|
|
2802
3448
|
finally {
|
|
3449
|
+
// Clear the gate's inference flag on EVERY exit path (paired with the
|
|
3450
|
+
// onInferenceStarted in startAgentStream). The branch-level calls above
|
|
3451
|
+
// are kept but are best-effort; if any exit path bypassed them the agent
|
|
3452
|
+
// stayed stuck in the gate's `inferring` set, which permanently BUFFERS
|
|
3453
|
+
// all incoming events → the agent silently stops waking on messages
|
|
3454
|
+
// (typing still stops, compression still runs — matching the observed
|
|
3455
|
+
// wedge). onInferenceEnded is idempotent, so a redundant call is safe.
|
|
3456
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
3457
|
+
this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), endedAt: Date.now() });
|
|
2803
3458
|
// Stop the typing indicator on every exit path (complete, error,
|
|
2804
3459
|
// exhausted, abort) so it never sticks after the turn ends.
|
|
2805
3460
|
this.channelRegistry?.stopTyping();
|
|
@@ -2884,7 +3539,10 @@ export class AgentFramework {
|
|
|
2884
3539
|
if (blocks) {
|
|
2885
3540
|
return { toolUseId: callId, content: blocks, isError: false };
|
|
2886
3541
|
}
|
|
2887
|
-
|
|
3542
|
+
// JSON.stringify returns the VALUE undefined (not a string) for undefined
|
|
3543
|
+
// input — a module tool returning `{ success: true }` with no data would
|
|
3544
|
+
// otherwise make `content.length` below throw a TypeError mid-turn.
|
|
3545
|
+
let content = JSON.stringify(afResult.data) ?? '';
|
|
2888
3546
|
if (maxChars && content.length > maxChars) {
|
|
2889
3547
|
content = safeSlice(content, 0, maxChars)
|
|
2890
3548
|
+ '\n\n[truncated — original was ' + content.length + ' chars]';
|
|
@@ -2954,11 +3612,11 @@ export class AgentFramework {
|
|
|
2954
3612
|
entryToStore.response = { blobId };
|
|
2955
3613
|
}
|
|
2956
3614
|
}
|
|
2957
|
-
// Append
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
this.store.
|
|
3615
|
+
// Append one record per inference. A read-modify-write via setStateJson
|
|
3616
|
+
// would emit a whole-array Set — record size then grows with accumulated
|
|
3617
|
+
// history (O(n²) aggregate disk) and the append_log strategy registered
|
|
3618
|
+
// for this state never sees an append.
|
|
3619
|
+
this.store.appendToStateJson(INFERENCE_LOG_ID, entryToStore);
|
|
2962
3620
|
}
|
|
2963
3621
|
logProcessEvent(event, responses) {
|
|
2964
3622
|
const entry = {
|
|
@@ -2974,11 +3632,8 @@ export class AgentFramework {
|
|
|
2974
3632
|
const blobId = this.store.storeBlob(Buffer.from(responsesJson), 'application/json');
|
|
2975
3633
|
entryToStore.responses = { blobId };
|
|
2976
3634
|
}
|
|
2977
|
-
// Append
|
|
2978
|
-
|
|
2979
|
-
const entries = Array.isArray(data) ? data : [];
|
|
2980
|
-
entries.push(entryToStore);
|
|
2981
|
-
this.store.setStateJson(PROCESS_LOG_ID, entries);
|
|
3635
|
+
// Append one record per event — same rationale as logInference.
|
|
3636
|
+
this.store.appendToStateJson(PROCESS_LOG_ID, entryToStore);
|
|
2982
3637
|
}
|
|
2983
3638
|
/**
|
|
2984
3639
|
* Find the MCPL server for a tool call by checking against the prefix map.
|
|
@@ -3175,15 +3830,19 @@ export class AgentFramework {
|
|
|
3175
3830
|
// without this the only durable record of a failed inference is a field in
|
|
3176
3831
|
// llm-calls.jsonl — invisible to operator, agent, and monitoring.
|
|
3177
3832
|
if (event.type === 'inference:exhausted') {
|
|
3178
|
-
this.noteInferenceExhausted(event.agentName ?? 'unknown', event.error ?? 'unknown error');
|
|
3833
|
+
this.noteInferenceExhausted(event.agentName ?? 'unknown', event.error ?? 'unknown error', event.retryable, event.errorType);
|
|
3179
3834
|
}
|
|
3180
3835
|
else if (event.type === 'inference:completed') {
|
|
3181
3836
|
// A successful response — even mid-turn between tool calls — proves the
|
|
3182
|
-
// agent isn't hard-down; clear its consecutive-failure streak
|
|
3837
|
+
// agent isn't hard-down; clear its consecutive-failure streak (and the
|
|
3838
|
+
// poison-history breaker's rewind budget).
|
|
3183
3839
|
const name = event.agentName;
|
|
3184
3840
|
if (name && this.consecutiveInferenceFailures.get(name)) {
|
|
3185
3841
|
this.consecutiveInferenceFailures.set(name, 0);
|
|
3186
3842
|
}
|
|
3843
|
+
if (name && this.exhaustionRewinds.get(name)) {
|
|
3844
|
+
this.exhaustionRewinds.set(name, 0);
|
|
3845
|
+
}
|
|
3187
3846
|
}
|
|
3188
3847
|
const traceEvent = {
|
|
3189
3848
|
...event,
|
|
@@ -3212,12 +3871,120 @@ export class AgentFramework {
|
|
|
3212
3871
|
* inference, so this never causes a retry/wake loop.
|
|
3213
3872
|
* 3. escalation — after N consecutive failures the agent is hard-down;
|
|
3214
3873
|
* log that loudly (a repeated identical failure is the textbook signal).
|
|
3874
|
+
* 4. breaker — when the failures are an `invalid_request` (a 400-class
|
|
3875
|
+
* rejection of the history itself, e.g. corrupted tool_use/tool_result
|
|
3876
|
+
* pairing or an oversized attachment), retrying the same context can
|
|
3877
|
+
* never succeed: every new push event wakes the agent onto the same
|
|
3878
|
+
* poisoned history forever. At the hard-down threshold, automatically
|
|
3879
|
+
* quarantine: shed the newest complete exchange (the same forced-rewind
|
|
3880
|
+
* primitive `/unstick` uses — shedNewestTurn never orphans a
|
|
3881
|
+
* tool_use/thinking block) and retry, bounded by the same rewind cap so
|
|
3882
|
+
* the breaker can never eat the whole history.
|
|
3215
3883
|
*/
|
|
3216
|
-
|
|
3884
|
+
/**
|
|
3885
|
+
* Live health snapshot for /healthz and doctor tooling: gate state, queued
|
|
3886
|
+
* work, per-agent status + last inference activity. Cheap and read-only.
|
|
3887
|
+
*/
|
|
3888
|
+
healthSnapshot() {
|
|
3889
|
+
return {
|
|
3890
|
+
at: new Date().toISOString(),
|
|
3891
|
+
uptimeSec: Math.round(process.uptime()),
|
|
3892
|
+
gate: this.eventGate?.inferenceDiagnostics() ?? null,
|
|
3893
|
+
pendingRequests: this.pendingRequests.length,
|
|
3894
|
+
activeStreams: [...this.activeStreams.keys()],
|
|
3895
|
+
agents: [...this.agents.entries()].map(([name, agent]) => ({
|
|
3896
|
+
name,
|
|
3897
|
+
status: agent.state.status,
|
|
3898
|
+
consecutiveInferenceFailures: this.consecutiveInferenceFailures.get(name) ?? 0,
|
|
3899
|
+
lastInference: this.lastInferenceAt.get(name) ?? null,
|
|
3900
|
+
refusalStats: this.refusalStats.get(name) ?? null,
|
|
3901
|
+
})),
|
|
3902
|
+
};
|
|
3903
|
+
}
|
|
3904
|
+
/** Append a structured JSONL record to logs/failures.log (best-effort).
|
|
3905
|
+
* Durable and independent of journald/unit log redirects — this is what
|
|
3906
|
+
* connectome-doctor and fleet tooling read. Always stamps `at`. Legacy
|
|
3907
|
+
* records carry {at, agent, consecutive, reason}; new records add `kind`
|
|
3908
|
+
* and kind-specific fields (additive only — doctor parses by regex). */
|
|
3909
|
+
logFailure(record) {
|
|
3910
|
+
try {
|
|
3911
|
+
mkdirSync('logs', { recursive: true });
|
|
3912
|
+
appendFileSync('logs/failures.log', JSON.stringify({ at: new Date().toISOString(), ...record }) + '\n');
|
|
3913
|
+
}
|
|
3914
|
+
catch { /* best-effort */ }
|
|
3915
|
+
}
|
|
3916
|
+
/**
|
|
3917
|
+
* Ops alert — the one escalation path for "a human should hear about this":
|
|
3918
|
+
* (1) durable failures.log record (unless the caller already wrote one),
|
|
3919
|
+
* (2) an `ops:alert` trace so authorized observers get it on the wire,
|
|
3920
|
+
* (3) if CONNECTOME_OPS_WEBHOOK is set, a Discord post throttled to one per
|
|
3921
|
+
* (agent, kind) per cooldown window — a persistent failure re-posts
|
|
3922
|
+
* every ~15 min instead of flooding the channel on every occurrence.
|
|
3923
|
+
* See connectome docs/observability.md.
|
|
3924
|
+
*/
|
|
3925
|
+
opsAlert(kind, agentName, message, opts) {
|
|
3926
|
+
if (!opts?.skipLog) {
|
|
3927
|
+
this.logFailure({ agent: agentName, kind, reason: message, ...opts?.data });
|
|
3928
|
+
}
|
|
3929
|
+
this.emitTrace({ type: 'ops:alert', kind, agentName, message, data: opts?.data });
|
|
3930
|
+
const hook = process.env.CONNECTOME_OPS_WEBHOOK;
|
|
3931
|
+
if (!hook)
|
|
3932
|
+
return;
|
|
3933
|
+
const key = `${agentName}:${kind}`;
|
|
3934
|
+
const now = Date.now();
|
|
3935
|
+
if (now - (this.opsAlertLastSent.get(key) ?? 0) < this.opsAlertCooldownMs)
|
|
3936
|
+
return;
|
|
3937
|
+
// Stamp BEFORE the async post so a burst can't race past the cooldown.
|
|
3938
|
+
this.opsAlertLastSent.set(key, now);
|
|
3939
|
+
fetch(hook, {
|
|
3940
|
+
method: 'POST',
|
|
3941
|
+
headers: { 'content-type': 'application/json' },
|
|
3942
|
+
body: JSON.stringify({
|
|
3943
|
+
content: `\u{1F6A8} **${agentName}** ${kind}: ${message.slice(0, 500)}`,
|
|
3944
|
+
}),
|
|
3945
|
+
}).catch((err) => console.error(`[ops-alert] webhook post failed (${key}):`, err instanceof Error ? err.message : err));
|
|
3946
|
+
}
|
|
3947
|
+
/**
|
|
3948
|
+
* Record a model refusal for observability: per-agent stats + a
|
|
3949
|
+
* consecutive-refusal streak (exposed via healthSnapshot), a durable
|
|
3950
|
+
* failures.log record for EVERY refusal (previously invisible outside
|
|
3951
|
+
* stderr), and a throttled ops alert once refusals repeat — one refusal is
|
|
3952
|
+
* often survivable (auto-rewind may clear it); two in a row means stuck.
|
|
3953
|
+
* The streak is reset by any non-refusal completion in the stream driver.
|
|
3954
|
+
*/
|
|
3955
|
+
noteRefusal(agentName, category, tokens) {
|
|
3956
|
+
const stats = this.refusalStats.get(agentName)
|
|
3957
|
+
?? { total: 0, byCategory: {}, lastAt: 0, lastCategory: '' };
|
|
3958
|
+
stats.total += 1;
|
|
3959
|
+
stats.byCategory[category] = (stats.byCategory[category] ?? 0) + 1;
|
|
3960
|
+
stats.lastAt = Date.now();
|
|
3961
|
+
stats.lastCategory = category;
|
|
3962
|
+
this.refusalStats.set(agentName, stats);
|
|
3963
|
+
const streak = (this.refusalStreak.get(agentName) ?? 0) + 1;
|
|
3964
|
+
this.refusalStreak.set(agentName, streak);
|
|
3965
|
+
this.logFailure({
|
|
3966
|
+
agent: agentName,
|
|
3967
|
+
kind: 'refusal',
|
|
3968
|
+
category,
|
|
3969
|
+
streak,
|
|
3970
|
+
locus: this.channelRegistry?.buildChannelContext()?.incoming?.channelId ?? null,
|
|
3971
|
+
tokens: tokens ?? null,
|
|
3972
|
+
});
|
|
3973
|
+
if (streak >= 2) {
|
|
3974
|
+
this.opsAlert('refusal', agentName, `refusal streak ${streak}, category=${category}`, { skipLog: true, data: { category, streak } });
|
|
3975
|
+
}
|
|
3976
|
+
return streak;
|
|
3977
|
+
}
|
|
3978
|
+
noteInferenceExhausted(agentName, reason, retryable, errorType) {
|
|
3217
3979
|
const streak = (this.consecutiveInferenceFailures.get(agentName) ?? 0) + 1;
|
|
3218
3980
|
this.consecutiveInferenceFailures.set(agentName, streak);
|
|
3981
|
+
this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), failedAt: Date.now(), lastError: reason.slice(0, 300) });
|
|
3219
3982
|
// (1) Durable stderr line — works in headless/daemon mode with no client.
|
|
3220
3983
|
console.error(`[inference-failed] agent=${agentName} consecutive=${streak}: ${reason}`);
|
|
3984
|
+
// (1b) Machine-greppable durable record, independent of journald/unit log
|
|
3985
|
+
// redirects: logs/failures.log under the host's working directory. This is
|
|
3986
|
+
// what connectome-doctor reads. Legacy fields kept; `kind` is additive.
|
|
3987
|
+
this.logFailure({ agent: agentName, consecutive: streak, reason, kind: 'inference-exhausted' });
|
|
3221
3988
|
// (2) Agent-facing chronicle marker (no inference triggered → no loop).
|
|
3222
3989
|
const agent = this.agents.get(agentName);
|
|
3223
3990
|
if (agent && process.env.SUPPRESS_INFERENCE_FAILED_MARKER !== '1') {
|
|
@@ -3239,8 +4006,64 @@ export class AgentFramework {
|
|
|
3239
4006
|
if (streak >= this.inferenceFailureEscalationThreshold) {
|
|
3240
4007
|
console.error(`[inference-hard-down] agent=${agentName} has FAILED ${streak} consecutive ` +
|
|
3241
4008
|
`inferences — it cannot complete a turn. Last reason: ${reason}`);
|
|
4009
|
+
// Ops alert (webhook + ops:alert trace), throttled per (agent, kind) —
|
|
4010
|
+
// a hard-down agent re-posts every ~15 min, not on every failed retry.
|
|
4011
|
+
// failures.log already got the per-exhaustion record above.
|
|
4012
|
+
this.opsAlert('hard-down', agentName, `${streak} consecutive inference failures. Last reason: ${reason}`, { skipLog: true });
|
|
4013
|
+
// (4) Poison-history breaker: ONLY for `invalid_request` — a 400-class
|
|
4014
|
+
// rejection of the history itself. Deliberately NOT keyed on
|
|
4015
|
+
// `retryable === false`, which membrane also returns for auth (expired
|
|
4016
|
+
// key), abort (deliberate cancel), context_length (compression's job,
|
|
4017
|
+
// and shedding newest is the wrong direction), safety and unsupported —
|
|
4018
|
+
// none of which mean the history is poisoned, and all of which would
|
|
4019
|
+
// otherwise shed good exchanges and stamp a false "the API kept
|
|
4020
|
+
// rejecting your history" marker. `retryable` is kept only for the trace.
|
|
4021
|
+
void retryable;
|
|
4022
|
+
if (errorType === 'invalid_request' && agent) {
|
|
4023
|
+
this.quarantinePoisonedHistory(agent, reason);
|
|
4024
|
+
}
|
|
3242
4025
|
}
|
|
3243
4026
|
}
|
|
4027
|
+
/**
|
|
4028
|
+
* Automatic poison-history quarantine (the actual "breaker"). Sheds the
|
|
4029
|
+
* newest complete exchange from the agent's history — reusing the same
|
|
4030
|
+
* primitives as the refusal auto-rewind / `/unstick` (shedNewestTurn +
|
|
4031
|
+
* the single consolidated episode marker) — and queues a retry so the
|
|
4032
|
+
* rewound history is verified immediately instead of waiting for the next
|
|
4033
|
+
* push event to wake the agent onto the same poisoned context.
|
|
4034
|
+
*
|
|
4035
|
+
* Bounded: at most `refusalHandling.maxRewinds` (default 3, hard cap 10)
|
|
4036
|
+
* sheds per failure episode; the budget resets on any successful inference.
|
|
4037
|
+
* At the cap (or with nothing left to shed) it stops — the agent stays up
|
|
4038
|
+
* and hard-down logging continues, but no further history is consumed.
|
|
4039
|
+
*/
|
|
4040
|
+
quarantinePoisonedHistory(agent, reason) {
|
|
4041
|
+
const cap = Math.max(1, Math.min(10, agent.refusalHandling?.maxRewinds ?? 3));
|
|
4042
|
+
const used = this.exhaustionRewinds.get(agent.name) ?? 0;
|
|
4043
|
+
if (used >= cap) {
|
|
4044
|
+
console.error(`[inference-rewind] agent=${agent.name} rewind cap ${cap} reached — ` +
|
|
4045
|
+
`not shedding further history. Manual repair (/unstick or /undo) needed.`);
|
|
4046
|
+
return;
|
|
4047
|
+
}
|
|
4048
|
+
const rec = this.shedNewestTurn(agent);
|
|
4049
|
+
if (!rec) {
|
|
4050
|
+
console.error(`[inference-rewind] agent=${agent.name} has nothing left to shed — ` +
|
|
4051
|
+
`history is only system markers. Manual repair needed.`);
|
|
4052
|
+
return;
|
|
4053
|
+
}
|
|
4054
|
+
this.exhaustionRewinds.set(agent.name, used + 1);
|
|
4055
|
+
const count = this.updateRewindMarker(agent, truncateReason(reason), 'inference-failure');
|
|
4056
|
+
console.error(`[inference-rewind] agent=${agent.name} auto-quarantined ${rec.descriptor} ` +
|
|
4057
|
+
`(${count} turn(s) shed this episode, cap ${cap}) after non-retryable ` +
|
|
4058
|
+
`inference failures — retrying on the rewound history.`);
|
|
4059
|
+
// Retry immediately on the repaired history (bounded by the cap above).
|
|
4060
|
+
this.pendingRequests.push({
|
|
4061
|
+
agentName: agent.name,
|
|
4062
|
+
reason: 'inference-failure-rewind-retry',
|
|
4063
|
+
source: 'framework',
|
|
4064
|
+
timestamp: Date.now(),
|
|
4065
|
+
});
|
|
4066
|
+
}
|
|
3244
4067
|
// ==========================================================================
|
|
3245
4068
|
// MCPL subsystem wiring
|
|
3246
4069
|
// ==========================================================================
|
|
@@ -3314,8 +4137,9 @@ export class AgentFramework {
|
|
|
3314
4137
|
}
|
|
3315
4138
|
},
|
|
3316
4139
|
});
|
|
3317
|
-
// Host capabilities advertised during the MCP handshake
|
|
3318
|
-
|
|
4140
|
+
// Host capabilities advertised during the MCP handshake — stored so
|
|
4141
|
+
// servers can also be connected later at runtime (connectMcplServer).
|
|
4142
|
+
this.mcplHostCapabilities = {
|
|
3319
4143
|
version: '0.4',
|
|
3320
4144
|
pushEvents: true,
|
|
3321
4145
|
contextHooks: {
|
|
@@ -3326,56 +4150,193 @@ export class AgentFramework {
|
|
|
3326
4150
|
};
|
|
3327
4151
|
for (const config of serverConfigs) {
|
|
3328
4152
|
try {
|
|
3329
|
-
|
|
3330
|
-
// registers channels — handleRegister fires during the handshake.
|
|
3331
|
-
if (config.channelSubscription !== undefined && this.channelRegistry) {
|
|
3332
|
-
this.channelRegistry.setSubscriptionPolicy(config.id, config.channelSubscription);
|
|
3333
|
-
}
|
|
3334
|
-
const connection = await this.mcplServerRegistry.addServer(config, hostCapabilities);
|
|
3335
|
-
// Wire event listeners then flush any events that arrived during the
|
|
3336
|
-
// handshake window (between setupMessageRouting and now).
|
|
3337
|
-
this.wireMcplEvents(connection);
|
|
3338
|
-
connection.ready();
|
|
3339
|
-
// Initialize feature sets if server advertises MCPL capabilities
|
|
3340
|
-
if (connection.capabilities) {
|
|
3341
|
-
const updateParams = this.featureSetManager.initializeServer(config.id, connection.capabilities, {
|
|
3342
|
-
enabledFeatureSets: config.enabledFeatureSets,
|
|
3343
|
-
disabledFeatureSets: config.disabledFeatureSets,
|
|
3344
|
-
});
|
|
3345
|
-
// Inform server which feature sets are enabled/disabled
|
|
3346
|
-
if (updateParams.enabled?.length || updateParams.disabled?.length) {
|
|
3347
|
-
connection.sendFeatureSetsUpdate(updateParams);
|
|
3348
|
-
}
|
|
3349
|
-
// Configure scope whitelist/blacklist patterns
|
|
3350
|
-
if (config.scopes) {
|
|
3351
|
-
this.scopeManager.configureAll(config.scopes);
|
|
3352
|
-
}
|
|
3353
|
-
// Register stateful feature sets with checkpoint manager (Step 8)
|
|
3354
|
-
if (this.checkpointManager) {
|
|
3355
|
-
const declared = this.featureSetManager.getDeclaredFeatureSets(config.id);
|
|
3356
|
-
if (declared) {
|
|
3357
|
-
for (const [fsName, fsDecl] of Object.entries(declared)) {
|
|
3358
|
-
if (fsDecl.rollback || fsDecl.hostState) {
|
|
3359
|
-
this.checkpointManager.registerFeatureSet(config.id, fsName, {
|
|
3360
|
-
hostState: fsDecl.hostState ?? false,
|
|
3361
|
-
rollback: fsDecl.rollback ?? false,
|
|
3362
|
-
});
|
|
3363
|
-
}
|
|
3364
|
-
}
|
|
3365
|
-
}
|
|
3366
|
-
}
|
|
3367
|
-
}
|
|
3368
|
-
this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` });
|
|
4153
|
+
await this.connectMcplServerInternal(config);
|
|
3369
4154
|
}
|
|
3370
4155
|
catch (error) {
|
|
3371
4156
|
// Fail-open: log and continue with remaining servers
|
|
3372
4157
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
3373
4158
|
console.error(`Failed to connect MCPL server "${config.id}":`, err.message);
|
|
4159
|
+
// Reaching this catch means no stub was created, so nothing will
|
|
4160
|
+
// retry — except in the edge case where addServer succeeded (with a
|
|
4161
|
+
// reconnecting stub) and a later setup step threw; config.reconnect
|
|
4162
|
+
// is the best cheap approximation of that.
|
|
4163
|
+
this.emitTrace({
|
|
4164
|
+
type: 'mcpl:server-connect-failed',
|
|
4165
|
+
serverId: config.id,
|
|
4166
|
+
error: err.message,
|
|
4167
|
+
attempt: 0,
|
|
4168
|
+
willRetry: config.reconnect === true,
|
|
4169
|
+
});
|
|
3374
4170
|
}
|
|
3375
4171
|
}
|
|
3376
4172
|
// Discover tools from all connected servers
|
|
3377
4173
|
await this.refreshMcplTools();
|
|
3378
4174
|
}
|
|
4175
|
+
/**
|
|
4176
|
+
* Connect a single MCPL server: register routing entries, open the
|
|
4177
|
+
* connection, wire events, and initialize feature sets / scopes /
|
|
4178
|
+
* checkpoints. Shared by startup (initializeMcpl) and the runtime
|
|
4179
|
+
* lifecycle API (connectMcplServer). Throws on connection failure.
|
|
4180
|
+
*/
|
|
4181
|
+
async connectMcplServerInternal(config) {
|
|
4182
|
+
if (!this.mcplServerRegistry || !this.mcplHostCapabilities) {
|
|
4183
|
+
throw new Error('MCPL subsystem is not initialized');
|
|
4184
|
+
}
|
|
4185
|
+
// Register prefix + config for tool dispatch routing (idempotent with
|
|
4186
|
+
// the pre-registration pass in initializeMcpl).
|
|
4187
|
+
const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
|
|
4188
|
+
this.mcplPrefixMap.set(prefix, config.id);
|
|
4189
|
+
this.mcplServerConfigs.set(config.id, config);
|
|
4190
|
+
// Record per-server channel subscription policy before the server
|
|
4191
|
+
// registers channels — handleRegister fires during the handshake.
|
|
4192
|
+
if (config.channelSubscription !== undefined && this.channelRegistry) {
|
|
4193
|
+
this.channelRegistry.setSubscriptionPolicy(config.id, config.channelSubscription);
|
|
4194
|
+
}
|
|
4195
|
+
const connection = await this.mcplServerRegistry.addServer(config, this.mcplHostCapabilities);
|
|
4196
|
+
// Wire event listeners then flush any events that arrived during the
|
|
4197
|
+
// handshake window (between setupMessageRouting and now).
|
|
4198
|
+
this.wireMcplEvents(connection);
|
|
4199
|
+
connection.ready();
|
|
4200
|
+
// Initialize feature sets if server advertises MCPL capabilities
|
|
4201
|
+
this.registerMcplServerFeatures(config, connection);
|
|
4202
|
+
this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` });
|
|
4203
|
+
}
|
|
4204
|
+
/**
|
|
4205
|
+
* (Re-)establish a server's host-side MCPL registration: feature-set state
|
|
4206
|
+
* (plus the featureSets/update sent to the server), scope patterns, and
|
|
4207
|
+
* stateful-feature-set checkpoint registration.
|
|
4208
|
+
*
|
|
4209
|
+
* Called on initial connect (connectMcplServerInternal) AND after every
|
|
4210
|
+
* auto-reconnect. The 'close' handler drops the feature-set registration, so
|
|
4211
|
+
* without the reconnect re-run FeatureSetManager.validateInbound would throw
|
|
4212
|
+
* "Unknown server" forever — every push event and inference request from the
|
|
4213
|
+
* revived server silently rejected until a full host restart.
|
|
4214
|
+
*
|
|
4215
|
+
* CheckpointManager.registerFeatureSet is idempotent (no-op for a key that
|
|
4216
|
+
* already has a tree), so checkpoint trees preserved across a transient
|
|
4217
|
+
* disconnect are resumed, not reset.
|
|
4218
|
+
*/
|
|
4219
|
+
registerMcplServerFeatures(config, connection) {
|
|
4220
|
+
if (!connection.capabilities)
|
|
4221
|
+
return;
|
|
4222
|
+
const updateParams = this.featureSetManager.initializeServer(config.id, connection.capabilities, {
|
|
4223
|
+
enabledFeatureSets: config.enabledFeatureSets,
|
|
4224
|
+
disabledFeatureSets: config.disabledFeatureSets,
|
|
4225
|
+
});
|
|
4226
|
+
// Inform server which feature sets are enabled/disabled
|
|
4227
|
+
if (updateParams.enabled?.length || updateParams.disabled?.length) {
|
|
4228
|
+
connection.sendFeatureSetsUpdate(updateParams);
|
|
4229
|
+
}
|
|
4230
|
+
// Configure scope whitelist/blacklist patterns
|
|
4231
|
+
if (config.scopes) {
|
|
4232
|
+
this.scopeManager.configureAll(config.scopes);
|
|
4233
|
+
}
|
|
4234
|
+
// Register stateful feature sets with checkpoint manager (Step 8)
|
|
4235
|
+
if (this.checkpointManager) {
|
|
4236
|
+
const declared = this.featureSetManager.getDeclaredFeatureSets(config.id);
|
|
4237
|
+
if (declared) {
|
|
4238
|
+
for (const [fsName, fsDecl] of Object.entries(declared)) {
|
|
4239
|
+
if (fsDecl.rollback || fsDecl.hostState) {
|
|
4240
|
+
this.checkpointManager.registerFeatureSet(config.id, fsName, {
|
|
4241
|
+
hostState: fsDecl.hostState ?? false,
|
|
4242
|
+
rollback: fsDecl.rollback ?? false,
|
|
4243
|
+
});
|
|
4244
|
+
}
|
|
4245
|
+
}
|
|
4246
|
+
}
|
|
4247
|
+
}
|
|
4248
|
+
}
|
|
4249
|
+
// ==========================================================================
|
|
4250
|
+
// Runtime MCPL server lifecycle (agent-facing hot deploy/restart/unload)
|
|
4251
|
+
// ==========================================================================
|
|
4252
|
+
/**
|
|
4253
|
+
* Connect a new MCPL server at runtime. Refreshes the tool list and
|
|
4254
|
+
* notifies the agent of newly available tools. Throws if the MCPL
|
|
4255
|
+
* subsystem is not initialized, the id is already connected, or the
|
|
4256
|
+
* connection/handshake fails.
|
|
4257
|
+
*/
|
|
4258
|
+
async connectMcplServer(config) {
|
|
4259
|
+
// Lazily bring up the MCPL subsystem — a framework that started with zero
|
|
4260
|
+
// configured servers can still deploy its first one at runtime.
|
|
4261
|
+
if (!this.mcplServerRegistry || !this.mcplHostCapabilities) {
|
|
4262
|
+
await this.initializeMcpl([], this.mcplInferenceRoutingConfig ?? undefined);
|
|
4263
|
+
}
|
|
4264
|
+
// Same collision rules create() enforces at startup: the tool prefix must
|
|
4265
|
+
// not shadow a module name or another server's prefix.
|
|
4266
|
+
const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
|
|
4267
|
+
if (this.moduleRegistry.getAllModules().some(m => m.name === prefix)) {
|
|
4268
|
+
throw new Error(`MCPL server "${config.id}" toolPrefix "${prefix}" collides with module "${prefix}"`);
|
|
4269
|
+
}
|
|
4270
|
+
const prefixOwner = this.mcplPrefixMap.get(prefix);
|
|
4271
|
+
if (prefixOwner && prefixOwner !== config.id) {
|
|
4272
|
+
throw new Error(`MCPL server "${config.id}" toolPrefix "${prefix}" collides with server "${prefixOwner}"`);
|
|
4273
|
+
}
|
|
4274
|
+
const oldToolNames = new Set(this.mcplTools.map(t => t.name));
|
|
4275
|
+
await this.connectMcplServerInternal(config);
|
|
4276
|
+
await this.refreshMcplTools();
|
|
4277
|
+
this.emitMcplToolDiff(oldToolNames, config.id);
|
|
4278
|
+
}
|
|
4279
|
+
/**
|
|
4280
|
+
* Disconnect an MCPL server at runtime: close the connection, destroy its
|
|
4281
|
+
* feature-set and checkpoint state (permanent removal — unlike a transient
|
|
4282
|
+
* transport close, which preserves checkpoints for the reconnect), remove
|
|
4283
|
+
* its channels from the registry, drop routing entries, and refresh tools.
|
|
4284
|
+
* No-op-ish if the server is not connected (still clears routing state).
|
|
4285
|
+
*/
|
|
4286
|
+
async disconnectMcplServer(id) {
|
|
4287
|
+
if (!this.mcplServerRegistry) {
|
|
4288
|
+
throw new Error('MCPL subsystem is not initialized');
|
|
4289
|
+
}
|
|
4290
|
+
const config = this.mcplServerConfigs.get(id);
|
|
4291
|
+
const oldToolNames = new Set(this.mcplTools.map(t => t.name));
|
|
4292
|
+
await this.mcplServerRegistry.removeServer(id);
|
|
4293
|
+
this.channelRegistry?.removeServer(id);
|
|
4294
|
+
// Permanent removal: also destroy feature-set and checkpoint state
|
|
4295
|
+
// explicitly. The 'close' handler usually does this, but a connection that
|
|
4296
|
+
// already transiently closed (reconnect pending) emits no second 'close'
|
|
4297
|
+
// from close(), and the transient path deliberately preserves checkpoints.
|
|
4298
|
+
this.featureSetManager?.removeServer(id);
|
|
4299
|
+
this.checkpointManager?.removeServer(id);
|
|
4300
|
+
const prefix = config?.toolPrefix ?? `mcpl--${id}`;
|
|
4301
|
+
this.mcplPrefixMap.delete(prefix);
|
|
4302
|
+
this.mcplServerConfigs.delete(id);
|
|
4303
|
+
await this.refreshMcplTools();
|
|
4304
|
+
this.emitMcplToolDiff(oldToolNames, id);
|
|
4305
|
+
}
|
|
4306
|
+
/**
|
|
4307
|
+
* Restart an MCPL server at runtime: disconnect, then reconnect with the
|
|
4308
|
+
* same (or an updated) config. This actually respawns a stdio child —
|
|
4309
|
+
* unlike `reconnect: true`, which only retries after transport-level
|
|
4310
|
+
* failures. Throws if the server has no stored config and none is given.
|
|
4311
|
+
*/
|
|
4312
|
+
async restartMcplServer(id, newConfig) {
|
|
4313
|
+
const config = newConfig ?? this.mcplServerConfigs.get(id);
|
|
4314
|
+
if (!config) {
|
|
4315
|
+
throw new Error(`MCPL server "${id}" is not configured`);
|
|
4316
|
+
}
|
|
4317
|
+
await this.disconnectMcplServer(id);
|
|
4318
|
+
await this.connectMcplServer(config);
|
|
4319
|
+
}
|
|
4320
|
+
/**
|
|
4321
|
+
* List configured MCPL servers with live connection status and the number
|
|
4322
|
+
* of tools each currently contributes.
|
|
4323
|
+
*/
|
|
4324
|
+
listMcplServers() {
|
|
4325
|
+
const result = [];
|
|
4326
|
+
for (const [id, config] of this.mcplServerConfigs) {
|
|
4327
|
+
const prefix = config.toolPrefix ?? `mcpl--${id}`;
|
|
4328
|
+
const connection = this.mcplServerRegistry?.getServer(id) ?? null;
|
|
4329
|
+
result.push({
|
|
4330
|
+
id,
|
|
4331
|
+
connected: connection?.isConnected ?? false,
|
|
4332
|
+
toolPrefix: prefix,
|
|
4333
|
+
toolCount: this.mcplTools.filter(t => t.name.startsWith(`${prefix}--`)).length,
|
|
4334
|
+
command: config.command,
|
|
4335
|
+
url: config.url,
|
|
4336
|
+
});
|
|
4337
|
+
}
|
|
4338
|
+
return result;
|
|
4339
|
+
}
|
|
3379
4340
|
/**
|
|
3380
4341
|
* Wire event listeners on an MCPL server connection.
|
|
3381
4342
|
* Push events and inference requests are deferred to Steps 6/7.
|
|
@@ -3446,14 +4407,102 @@ export class AgentFramework {
|
|
|
3446
4407
|
connection.on('tools-list-changed', () => {
|
|
3447
4408
|
this.handleToolsListChanged(connection.id);
|
|
3448
4409
|
});
|
|
3449
|
-
//
|
|
3450
|
-
|
|
4410
|
+
// Re-establish full server registration on reconnect. The 'close' handler
|
|
4411
|
+
// removed the feature-set state, so pushes / inference requests from the
|
|
4412
|
+
// revived server would otherwise be rejected with "Unknown server" until a
|
|
4413
|
+
// host restart. Re-runs the same init as the initial connect path (the
|
|
4414
|
+
// fresh handshake refreshed connection.capabilities); checkpoint trees
|
|
4415
|
+
// preserved across the transient close are resumed by idempotent
|
|
4416
|
+
// registration. Then refresh tools (server may have different tools).
|
|
4417
|
+
connection.on('reconnect', (info) => {
|
|
4418
|
+
try {
|
|
4419
|
+
const config = this.mcplServerConfigs.get(connection.id);
|
|
4420
|
+
if (config) {
|
|
4421
|
+
this.registerMcplServerFeatures(config, connection);
|
|
4422
|
+
}
|
|
4423
|
+
}
|
|
4424
|
+
catch (error) {
|
|
4425
|
+
console.error(`MCPL server "${connection.id}" re-registration after reconnect failed:`, error instanceof Error ? error.message : error);
|
|
4426
|
+
}
|
|
3451
4427
|
this.handleToolsListChanged(connection.id);
|
|
4428
|
+
this.emitTrace({
|
|
4429
|
+
type: 'mcpl:server-reconnected',
|
|
4430
|
+
serverId: connection.id,
|
|
4431
|
+
attempts: info?.attempts ?? 0,
|
|
4432
|
+
});
|
|
4433
|
+
// Mirror the module:removed emitted on 'close' so module-lifecycle
|
|
4434
|
+
// consumers see the server come back, not just vanish.
|
|
4435
|
+
this.emitTrace({ type: 'module:added', moduleName: `mcpl:${connection.id}` });
|
|
3452
4436
|
});
|
|
3453
|
-
//
|
|
3454
|
-
|
|
4437
|
+
// Surface connect/reconnect failures. Before these traces existed the
|
|
4438
|
+
// only receipt was a console.error on the host's own stderr — invisible
|
|
4439
|
+
// unless someone ssh'd in and read the process log.
|
|
4440
|
+
connection.on('connect-failed', (params) => {
|
|
4441
|
+
this.emitTrace({
|
|
4442
|
+
type: 'mcpl:server-connect-failed',
|
|
4443
|
+
serverId: connection.id,
|
|
4444
|
+
error: params.error,
|
|
4445
|
+
attempt: params.attempt,
|
|
4446
|
+
willRetry: connection.willReconnect,
|
|
4447
|
+
});
|
|
4448
|
+
});
|
|
4449
|
+
connection.on('reconnect-failed', (params) => {
|
|
4450
|
+
this.emitTrace({
|
|
4451
|
+
type: 'mcpl:server-connect-failed',
|
|
4452
|
+
serverId: connection.id,
|
|
4453
|
+
error: params.error,
|
|
4454
|
+
attempt: params.attempt,
|
|
4455
|
+
willRetry: connection.willReconnect,
|
|
4456
|
+
});
|
|
4457
|
+
// The reconnect loop never gives up (backoff caps at ~300s), so
|
|
4458
|
+
// "the server is effectively down" is an attempt-count judgment:
|
|
4459
|
+
// 5 failed attempts ≈ a few minutes of outage. Throttled per
|
|
4460
|
+
// (serverId, kind), so a long outage re-posts every ~15 min.
|
|
4461
|
+
if (params.attempt >= 5) {
|
|
4462
|
+
this.opsAlert('mcpl-down', connection.id, `MCPL server unreachable (attempt ${params.attempt}): ${params.error}`);
|
|
4463
|
+
}
|
|
4464
|
+
});
|
|
4465
|
+
// A late response to a tools/call that already timed out, carrying
|
|
4466
|
+
// stateful data the server advanced to. The host can't re-inject it (the
|
|
4467
|
+
// dispatch context is gone), but surfacing it makes the checkpoint-tree
|
|
4468
|
+
// divergence greppable instead of a silent drift into stale state.
|
|
4469
|
+
connection.on('orphaned-response', (info) => {
|
|
4470
|
+
this.emitTrace({
|
|
4471
|
+
type: 'mcpl:orphaned-response',
|
|
4472
|
+
serverId: connection.id,
|
|
4473
|
+
responseId: info.id,
|
|
4474
|
+
hadState: info.hadState,
|
|
4475
|
+
hadCheckpoint: info.hadCheckpoint,
|
|
4476
|
+
});
|
|
4477
|
+
});
|
|
4478
|
+
// Connection-level errors (e.g. child process 'error' after spawn).
|
|
4479
|
+
// Attaching this listener also keeps an unhandled EventEmitter 'error'
|
|
4480
|
+
// from crashing the host process.
|
|
4481
|
+
connection.on('error', (err) => {
|
|
4482
|
+
this.emitTrace({
|
|
4483
|
+
type: 'mcpl:server-error',
|
|
4484
|
+
serverId: connection.id,
|
|
4485
|
+
error: err.message,
|
|
4486
|
+
});
|
|
4487
|
+
});
|
|
4488
|
+
// Cleanup on disconnect. Feature-set state is always dropped (the server
|
|
4489
|
+
// can't be validated while down; reconnect re-registers it). Checkpoint
|
|
4490
|
+
// trees, however, are durable state: CheckpointManager.removeServer deletes
|
|
4491
|
+
// them AND persists the deletion to Chronicle. The close handler must
|
|
4492
|
+
// therefore NEVER destroy checkpoints — `willReconnect` is false on a clean
|
|
4493
|
+
// stop() too (reconnectEnabled is cleared before 'close' fires), so gating
|
|
4494
|
+
// on it would erase every checkpoint tree on an ordinary host restart while
|
|
4495
|
+
// a SIGKILL preserves them. Permanent removal is owned solely by
|
|
4496
|
+
// disconnectMcplServer, which deletes the trees explicitly.
|
|
4497
|
+
connection.on('close', (code, signal) => {
|
|
3455
4498
|
this.featureSetManager?.removeServer(connection.id);
|
|
3456
|
-
this.
|
|
4499
|
+
this.emitTrace({
|
|
4500
|
+
type: 'mcpl:server-closed',
|
|
4501
|
+
serverId: connection.id,
|
|
4502
|
+
code: code ?? null,
|
|
4503
|
+
signal: signal ?? null,
|
|
4504
|
+
willReconnect: connection.willReconnect,
|
|
4505
|
+
});
|
|
3457
4506
|
this.emitTrace({ type: 'module:removed', moduleName: `mcpl:${connection.id}` });
|
|
3458
4507
|
});
|
|
3459
4508
|
}
|
|
@@ -3822,7 +4871,7 @@ export class AgentFramework {
|
|
|
3822
4871
|
finish({ success: false, error: 'sleep: `seconds` must be a positive number', isError: true });
|
|
3823
4872
|
return;
|
|
3824
4873
|
}
|
|
3825
|
-
const { until } = gate.setSleep(seconds, input.message);
|
|
4874
|
+
const { until } = gate.setSleep(seconds, input.message, agentName);
|
|
3826
4875
|
const announce = input.announce !== false; // default true
|
|
3827
4876
|
const human = seconds >= 3600 ? `${(seconds / 3600).toFixed(1)}h`
|
|
3828
4877
|
: seconds >= 60 ? `${Math.round(seconds / 60)}m`
|