@animalabs/agent-framework 0.6.0 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/agent.d.ts +4 -0
- package/dist/src/agent.d.ts.map +1 -1
- package/dist/src/agent.js +15 -0
- package/dist/src/agent.js.map +1 -1
- package/dist/src/framework.d.ts +235 -2
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +1141 -92
- package/dist/src/framework.js.map +1 -1
- package/dist/src/gate/event-gate.d.ts +20 -1
- package/dist/src/gate/event-gate.d.ts.map +1 -1
- package/dist/src/gate/event-gate.js +60 -4
- package/dist/src/gate/event-gate.js.map +1 -1
- package/dist/src/index.d.ts +3 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +4 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/mcpl/channel-registry.d.ts +17 -1
- package/dist/src/mcpl/channel-registry.d.ts.map +1 -1
- package/dist/src/mcpl/channel-registry.js +34 -3
- package/dist/src/mcpl/channel-registry.js.map +1 -1
- package/dist/src/mcpl/push-handler.d.ts.map +1 -1
- package/dist/src/mcpl/push-handler.js +4 -0
- package/dist/src/mcpl/push-handler.js.map +1 -1
- package/dist/src/mcpl/server-connection.d.ts +29 -0
- package/dist/src/mcpl/server-connection.d.ts.map +1 -1
- package/dist/src/mcpl/server-connection.js +102 -4
- package/dist/src/mcpl/server-connection.js.map +1 -1
- package/dist/src/mcpl/types.d.ts +17 -1
- package/dist/src/mcpl/types.d.ts.map +1 -1
- package/dist/src/mcpl/types.js.map +1 -1
- package/dist/src/module-registry.d.ts +15 -1
- package/dist/src/module-registry.d.ts.map +1 -1
- package/dist/src/module-registry.js +92 -4
- package/dist/src/module-registry.js.map +1 -1
- package/dist/src/types/agent.d.ts +37 -0
- package/dist/src/types/agent.d.ts.map +1 -1
- package/dist/src/types/framework.d.ts +40 -0
- package/dist/src/types/framework.d.ts.map +1 -1
- package/dist/src/types/module.d.ts +46 -0
- package/dist/src/types/module.d.ts.map +1 -1
- package/dist/src/types/trace.d.ts +51 -0
- package/dist/src/types/trace.d.ts.map +1 -1
- package/dist/test/agent-cache-ttl.test.d.ts +2 -0
- package/dist/test/agent-cache-ttl.test.d.ts.map +1 -0
- package/dist/test/agent-cache-ttl.test.js +25 -0
- package/dist/test/agent-cache-ttl.test.js.map +1 -0
- package/dist/test/conversation-fork-injection-scoping.test.d.ts +2 -0
- package/dist/test/conversation-fork-injection-scoping.test.d.ts.map +1 -0
- package/dist/test/conversation-fork-injection-scoping.test.js +184 -0
- package/dist/test/conversation-fork-injection-scoping.test.js.map +1 -0
- package/dist/test/gather-context-timeout.test.d.ts +2 -0
- package/dist/test/gather-context-timeout.test.d.ts.map +1 -0
- package/dist/test/gather-context-timeout.test.js +120 -0
- package/dist/test/gather-context-timeout.test.js.map +1 -0
- package/dist/test/inference-exhaustion-rewind.test.d.ts +2 -0
- package/dist/test/inference-exhaustion-rewind.test.d.ts.map +1 -0
- package/dist/test/inference-exhaustion-rewind.test.js +181 -0
- package/dist/test/inference-exhaustion-rewind.test.js.map +1 -0
- package/dist/test/inference-failure-observability.test.js +9 -0
- package/dist/test/inference-failure-observability.test.js.map +1 -1
- package/dist/test/maintenance-timer.test.d.ts +2 -0
- package/dist/test/maintenance-timer.test.d.ts.map +1 -0
- package/dist/test/maintenance-timer.test.js +114 -0
- package/dist/test/maintenance-timer.test.js.map +1 -0
- package/dist/test/mcpl-reconnect-events.test.d.ts +2 -0
- package/dist/test/mcpl-reconnect-events.test.d.ts.map +1 -0
- package/dist/test/mcpl-reconnect-events.test.js +119 -0
- package/dist/test/mcpl-reconnect-events.test.js.map +1 -0
- package/dist/test/mcpl-reconnect-reregistration.test.d.ts +2 -0
- package/dist/test/mcpl-reconnect-reregistration.test.d.ts.map +1 -0
- package/dist/test/mcpl-reconnect-reregistration.test.js +181 -0
- package/dist/test/mcpl-reconnect-reregistration.test.js.map +1 -0
- package/dist/test/mcpl-request-timeout.test.d.ts +2 -0
- package/dist/test/mcpl-request-timeout.test.d.ts.map +1 -0
- package/dist/test/mcpl-request-timeout.test.js +123 -0
- package/dist/test/mcpl-request-timeout.test.js.map +1 -0
- package/dist/test/mcpl-runtime-lifecycle.test.d.ts +7 -0
- package/dist/test/mcpl-runtime-lifecycle.test.d.ts.map +1 -0
- package/dist/test/mcpl-runtime-lifecycle.test.js +142 -0
- package/dist/test/mcpl-runtime-lifecycle.test.js.map +1 -0
- package/dist/test/ops-alert.test.d.ts +2 -0
- package/dist/test/ops-alert.test.d.ts.map +1 -0
- package/dist/test/ops-alert.test.js +152 -0
- package/dist/test/ops-alert.test.js.map +1 -0
- package/dist/test/refusal-rewind.test.d.ts +2 -0
- package/dist/test/refusal-rewind.test.d.ts.map +1 -0
- package/dist/test/refusal-rewind.test.js +108 -0
- package/dist/test/refusal-rewind.test.js.map +1 -0
- package/dist/test/state-scaling-e2e.test.d.ts +22 -0
- package/dist/test/state-scaling-e2e.test.d.ts.map +1 -0
- package/dist/test/state-scaling-e2e.test.js +200 -0
- package/dist/test/state-scaling-e2e.test.js.map +1 -0
- package/dist/test/state-scaling.test.d.ts +12 -0
- package/dist/test/state-scaling.test.d.ts.map +1 -0
- package/dist/test/state-scaling.test.js +243 -0
- package/dist/test/state-scaling.test.js.map +1 -0
- package/dist/test/tool-result-content.test.d.ts +2 -0
- package/dist/test/tool-result-content.test.d.ts.map +1 -0
- package/dist/test/tool-result-content.test.js +37 -0
- package/dist/test/tool-result-content.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +3 -2
- package/src/agent.ts +16 -0
- package/src/framework.ts +1339 -99
- package/src/gate/event-gate.ts +56 -4
- package/src/index.ts +10 -0
- package/src/mcpl/channel-registry.ts +34 -3
- package/src/mcpl/push-handler.ts +4 -0
- package/src/mcpl/server-connection.ts +110 -4
- package/src/mcpl/types.ts +19 -1
- package/src/module-registry.ts +113 -4
- package/src/types/agent.ts +38 -0
- package/src/types/framework.ts +44 -0
- package/src/types/module.ts +52 -0
- package/src/types/trace.ts +67 -0
package/src/framework.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { join } from 'node:path';
|
|
2
|
+
import { appendFileSync, mkdirSync } from 'node:fs';
|
|
2
3
|
import { JsStore } from '@animalabs/chronicle';
|
|
3
4
|
import type { Membrane, ContentBlock, NormalizedRequest, YieldingStream, ToolResult as MembraneToolResult, ToolResultContentBlock } from '@animalabs/membrane';
|
|
4
5
|
import { MembraneError } from '@animalabs/membrane';
|
|
@@ -39,10 +40,13 @@ import type {
|
|
|
39
40
|
AgentState,
|
|
40
41
|
Module,
|
|
41
42
|
SpeechContext,
|
|
43
|
+
ContextMaintenanceRun,
|
|
44
|
+
ContextMaintenanceAgentRun,
|
|
45
|
+
ContextMaintenanceSnapshot,
|
|
42
46
|
} from './types/index.js';
|
|
43
47
|
import { ProcessQueueImpl } from './queue.js';
|
|
44
48
|
import { Agent } from './agent.js';
|
|
45
|
-
import { ModuleRegistry } from './module-registry.js';
|
|
49
|
+
import { ModuleRegistry, isStateExistsError } from './module-registry.js';
|
|
46
50
|
import { McplServerRegistry } from './mcpl/server-registry.js';
|
|
47
51
|
import { FeatureSetManager } from './mcpl/feature-set-manager.js';
|
|
48
52
|
import { ScopeManager } from './mcpl/scope-manager.js';
|
|
@@ -80,7 +84,8 @@ const FRAMEWORK_STATE_ID = 'framework/state';
|
|
|
80
84
|
const CONVERSATION_ROUTER_STATE_ID = 'framework/conversation-router';
|
|
81
85
|
const INFERENCE_LOG_ID = 'framework/inference-log';
|
|
82
86
|
const PROCESS_LOG_ID = 'framework/process-log';
|
|
83
|
-
const TURN_CHECKPOINTS_ID = 'framework/turn-checkpoints';
|
|
87
|
+
const TURN_CHECKPOINTS_ID = 'framework/turn-checkpoints'; // legacy single-map layout, read-only fallback
|
|
88
|
+
const TURN_CHECKPOINTS_TREE_ID = 'framework/turn-checkpoints/tree';
|
|
84
89
|
|
|
85
90
|
/** Maximum number of turn checkpoints to keep per agent. */
|
|
86
91
|
const MAX_TURN_CHECKPOINTS = 20;
|
|
@@ -142,6 +147,10 @@ class DefaultErrorPolicy implements ErrorPolicy {
|
|
|
142
147
|
|
|
143
148
|
/** Default sync interval in milliseconds */
|
|
144
149
|
const DEFAULT_SYNC_INTERVAL_MS = 1000;
|
|
150
|
+
/** Poll pending context-strategy maintenance independently of user activity. */
|
|
151
|
+
const DEFAULT_MAINTENANCE_INTERVAL_MS = 5000;
|
|
152
|
+
/** Bound one pass so a large backlog yields to inference and other agents. */
|
|
153
|
+
const MAINTENANCE_TICKS_PER_PASS = 8;
|
|
145
154
|
|
|
146
155
|
/**
|
|
147
156
|
* Extract fields the EventGate cares about from a ProcessEvent. The set of
|
|
@@ -198,10 +207,34 @@ interface HostCommandParams {
|
|
|
198
207
|
fromMessageId?: string;
|
|
199
208
|
/** For the `hide` command: Discord message id ending an inclusive range. */
|
|
200
209
|
toMessageId?: string;
|
|
210
|
+
/** For the `unstick` command: max rewind/retry attempts (default = the
|
|
211
|
+
* agent's refusalHandling.maxRewinds, else 3). */
|
|
212
|
+
maxRewinds?: number;
|
|
213
|
+
/** For the `unstick` command: raw channel id to post the outcome report to. */
|
|
214
|
+
channelId?: string;
|
|
201
215
|
requesterId?: string;
|
|
202
216
|
requesterName?: string;
|
|
203
217
|
}
|
|
204
218
|
|
|
219
|
+
/** Descriptor of a single refusal-driven rewind (see rewindTriggeringTurn). */
|
|
220
|
+
interface RewindRecord {
|
|
221
|
+
/** tool = machine tool exchange; human = an ingested message; other = else. */
|
|
222
|
+
kind: 'tool' | 'human' | 'other';
|
|
223
|
+
/** Content-free, safe-to-replay description of what was withheld. */
|
|
224
|
+
descriptor: string;
|
|
225
|
+
/** All message ids removed in this shed (a tool exchange removes 2+). */
|
|
226
|
+
removedIds: MessageId[];
|
|
227
|
+
/** Discord (channelId, messageId) of the removed message, if it had one. */
|
|
228
|
+
discordRef?: { channelId: string; messageId: string };
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** Cap an API error message for inline use in a rewind marker: keep enough to
|
|
232
|
+
* identify the failure class without pasting a wall of provider JSON into the
|
|
233
|
+
* agent's context. */
|
|
234
|
+
function truncateReason(reason: string, max = 160): string {
|
|
235
|
+
return reason.length <= max ? reason : reason.slice(0, max) + '…';
|
|
236
|
+
}
|
|
237
|
+
|
|
205
238
|
export class AgentFramework {
|
|
206
239
|
private store: JsStore;
|
|
207
240
|
private ownsStore: boolean;
|
|
@@ -227,6 +260,16 @@ export class AgentFramework {
|
|
|
227
260
|
private traceListeners: TraceEventListener[] = [];
|
|
228
261
|
private syncIntervalMs: number;
|
|
229
262
|
private syncTimer: ReturnType<typeof setInterval> | null = null;
|
|
263
|
+
private maintenanceIntervalMs: number;
|
|
264
|
+
private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
|
|
265
|
+
private maintenancePass: Promise<void> | null = null;
|
|
266
|
+
private maintenanceRunId = 0;
|
|
267
|
+
private currentMaintenanceRun: ContextMaintenanceRun | null = null;
|
|
268
|
+
private maintenanceHistory: ContextMaintenanceRun[] = [];
|
|
269
|
+
/** Last time we console-warned about stale (busy-requeued) inference requests, per agent. */
|
|
270
|
+
private staleWarnAt = new Map<string, number>();
|
|
271
|
+
/** Per-agent last inference activity (epoch ms), for /healthz + doctor tooling. */
|
|
272
|
+
private lastInferenceAt = new Map<string, { startedAt?: number; endedAt?: number; failedAt?: number; lastError?: string }>();
|
|
230
273
|
private processLoggingPersist: boolean;
|
|
231
274
|
private processLoggingBroadcast: boolean;
|
|
232
275
|
private activeStreams: Map<string, Promise<void>> = new Map();
|
|
@@ -237,6 +280,41 @@ export class AgentFramework {
|
|
|
237
280
|
/** N consecutive failed inferences ⇒ the agent is treated as hard-down and
|
|
238
281
|
* escalated loudly to stderr. */
|
|
239
282
|
private readonly inferenceFailureEscalationThreshold = 3;
|
|
283
|
+
/** Per-(agent,kind) timestamp of the last ops webhook post. Throttles
|
|
284
|
+
* opsAlert() so a persistent failure re-posts once per cooldown window
|
|
285
|
+
* instead of on every occurrence. */
|
|
286
|
+
private opsAlertLastSent: Map<string, number> = new Map();
|
|
287
|
+
/** Cooldown between webhook posts for the same (agent, kind). */
|
|
288
|
+
private readonly opsAlertCooldownMs = 15 * 60_000;
|
|
289
|
+
/** Per-agent refusal bookkeeping for observability — exposed via
|
|
290
|
+
* healthSnapshot() and mirrored to failures.log / ops alerts. */
|
|
291
|
+
private refusalStats: Map<string, { total: number; byCategory: Record<string, number>; lastAt: number; lastCategory: string }> = new Map();
|
|
292
|
+
/** Per-agent count of consecutive refusals (reset on any non-refusal
|
|
293
|
+
* completion). Distinct from refusalRewinds, which budgets the auto-rewind
|
|
294
|
+
* loop — this one drives ops alerting. */
|
|
295
|
+
private refusalStreak: Map<string, number> = new Map();
|
|
296
|
+
/** Per-agent count of consecutive refusal-driven rewinds in the current turn
|
|
297
|
+
* chain (reset when a turn completes without a refusal). Bounds the auto
|
|
298
|
+
* rewind loop — see refusalHandling + rewindTriggeringTurn. */
|
|
299
|
+
private refusalRewinds: Map<string, number> = new Map();
|
|
300
|
+
/** Per-agent count of poison-history rewinds performed by the hard-down
|
|
301
|
+
* breaker (noteInferenceExhausted). Reset on any successful inference.
|
|
302
|
+
* Bounds the automatic quarantine loop the same way refusalRewinds bounds
|
|
303
|
+
* the refusal loop, so the breaker can never shed the whole history. */
|
|
304
|
+
private exhaustionRewinds: Map<string, number> = new Map();
|
|
305
|
+
/** Per-agent current rewind episode: the single consolidated marker's id and
|
|
306
|
+
* how many turns have been shed so far. One marker per episode, updated in
|
|
307
|
+
* place (see updateRewindMarker); cleared when the episode ends. */
|
|
308
|
+
private rewindEpisode: Map<string, { markerId: MessageId; count: number; category: string }> = new Map();
|
|
309
|
+
/** Active `/unstick` sessions: an admin-forced rewind-until-clean loop that
|
|
310
|
+
* runs even when the agent's autoRewind toggle is off. Tracks the remaining
|
|
311
|
+
* budget, what was shed (for the report), and where to post the outcome. */
|
|
312
|
+
private forcedRewind: Map<string, {
|
|
313
|
+
remaining: number;
|
|
314
|
+
removed: RewindRecord[];
|
|
315
|
+
serverId: string;
|
|
316
|
+
channelId: string;
|
|
317
|
+
}> = new Map();
|
|
240
318
|
/** Name of the primary (non-ephemeral) agent for routing framework messages. */
|
|
241
319
|
private primaryAgentName: string | null = null;
|
|
242
320
|
|
|
@@ -283,6 +361,13 @@ export class AgentFramework {
|
|
|
283
361
|
private mcplPrefixMap: Map<string, string> = new Map();
|
|
284
362
|
/** Maps serverId → McplServerConfig for prefix lookup. */
|
|
285
363
|
private mcplServerConfigs: Map<string, import('./mcpl/types.js').McplServerConfig> = new Map();
|
|
364
|
+
/** Host capabilities advertised during the MCP handshake — stored so servers
|
|
365
|
+
* can be connected at runtime (connectMcplServer) after initialization. */
|
|
366
|
+
private mcplHostCapabilities: McplHostCapabilities | null = null;
|
|
367
|
+
/** Inference routing policy from FrameworkConfig — stored so the MCPL
|
|
368
|
+
* subsystem can be lazily initialized by connectMcplServer when the
|
|
369
|
+
* framework started with zero configured servers. */
|
|
370
|
+
private mcplInferenceRoutingConfig: import('./mcpl/types.js').InferenceRoutingPolicy | null = null;
|
|
286
371
|
|
|
287
372
|
// EventGate (null when FrameworkConfig.gate is omitted)
|
|
288
373
|
private eventGate: EventGate | null = null;
|
|
@@ -297,6 +382,7 @@ export class AgentFramework {
|
|
|
297
382
|
inferencePolicy: InferencePolicy,
|
|
298
383
|
errorPolicy: ErrorPolicy,
|
|
299
384
|
syncIntervalMs: number,
|
|
385
|
+
maintenanceIntervalMs: number,
|
|
300
386
|
processLoggingPersist: boolean,
|
|
301
387
|
processLoggingBroadcast: boolean
|
|
302
388
|
) {
|
|
@@ -306,6 +392,7 @@ export class AgentFramework {
|
|
|
306
392
|
this.inferencePolicy = inferencePolicy;
|
|
307
393
|
this.errorPolicy = errorPolicy;
|
|
308
394
|
this.syncIntervalMs = syncIntervalMs;
|
|
395
|
+
this.maintenanceIntervalMs = maintenanceIntervalMs;
|
|
309
396
|
this.processLoggingPersist = processLoggingPersist;
|
|
310
397
|
this.processLoggingBroadcast = processLoggingBroadcast;
|
|
311
398
|
this.queue = new ProcessQueueImpl();
|
|
@@ -363,10 +450,12 @@ export class AgentFramework {
|
|
|
363
450
|
// Already registered
|
|
364
451
|
}
|
|
365
452
|
|
|
453
|
+
// The legacy single-map checkpoint state (TURN_CHECKPOINTS_ID) is no longer
|
|
454
|
+
// registered for new stores — it's read-only fallback data in old ones.
|
|
366
455
|
try {
|
|
367
|
-
store.registerState({ id:
|
|
368
|
-
} catch {
|
|
369
|
-
|
|
456
|
+
store.registerState({ id: TURN_CHECKPOINTS_TREE_ID, strategy: 'tree' });
|
|
457
|
+
} catch (error) {
|
|
458
|
+
if (!isStateExistsError(error)) throw error;
|
|
370
459
|
}
|
|
371
460
|
|
|
372
461
|
// Process logging config (default: disabled)
|
|
@@ -394,6 +483,7 @@ export class AgentFramework {
|
|
|
394
483
|
config.inferencePolicy ?? new DefaultInferencePolicy(),
|
|
395
484
|
config.errorPolicy ?? new DefaultErrorPolicy(),
|
|
396
485
|
config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS,
|
|
486
|
+
config.maintenanceIntervalMs ?? DEFAULT_MAINTENANCE_INTERVAL_MS,
|
|
397
487
|
processLoggingPersist,
|
|
398
488
|
processLoggingBroadcast
|
|
399
489
|
);
|
|
@@ -472,6 +562,10 @@ export class AgentFramework {
|
|
|
472
562
|
framework.livenessWatchdog.start();
|
|
473
563
|
}
|
|
474
564
|
|
|
565
|
+
// Stored for lazy MCPL initialization (connectMcplServer on a framework
|
|
566
|
+
// that started with zero configured servers).
|
|
567
|
+
framework.mcplInferenceRoutingConfig = config.inferenceRouting ?? null;
|
|
568
|
+
|
|
475
569
|
// Initialize MCPL subsystems if configured
|
|
476
570
|
if (config.mcplServers && config.mcplServers.length > 0) {
|
|
477
571
|
// Validate tool prefixes: no collisions with module names or between servers
|
|
@@ -496,6 +590,31 @@ export class AgentFramework {
|
|
|
496
590
|
await framework.initializeMcpl(config.mcplServers, config.inferenceRouting);
|
|
497
591
|
}
|
|
498
592
|
|
|
593
|
+
// Diagnostics: `kill -USR2 <pid>` dumps live wake/inference state to stderr
|
|
594
|
+
// (journal) without a restart — for catching the wake-wedge on the running
|
|
595
|
+
// process. Shows the gate's `inferring` set + buffered-event count (the
|
|
596
|
+
// wedge signature), active streams, and pending inference requests.
|
|
597
|
+
try {
|
|
598
|
+
process.on('SIGUSR2', () => {
|
|
599
|
+
try {
|
|
600
|
+
const gate = framework.eventGate?.inferenceDiagnostics() ?? null;
|
|
601
|
+
console.error(
|
|
602
|
+
'[diagnostics] ' + JSON.stringify({
|
|
603
|
+
at: new Date().toISOString(),
|
|
604
|
+
gate,
|
|
605
|
+
activeStreams: [...framework.activeStreams.keys()],
|
|
606
|
+
pendingRequests: framework.pendingRequests.length,
|
|
607
|
+
agents: [...framework.agents.keys()],
|
|
608
|
+
}),
|
|
609
|
+
);
|
|
610
|
+
} catch (err) {
|
|
611
|
+
console.error('[diagnostics] dump failed:', err instanceof Error ? err.message : err);
|
|
612
|
+
}
|
|
613
|
+
});
|
|
614
|
+
} catch {
|
|
615
|
+
// SIGUSR2 not available on this platform — non-fatal.
|
|
616
|
+
}
|
|
617
|
+
|
|
499
618
|
return framework;
|
|
500
619
|
}
|
|
501
620
|
|
|
@@ -520,6 +639,16 @@ export class AgentFramework {
|
|
|
520
639
|
}
|
|
521
640
|
}, this.syncIntervalMs);
|
|
522
641
|
}
|
|
642
|
+
|
|
643
|
+
if (this.maintenanceIntervalMs > 0) {
|
|
644
|
+
this.maintenanceTimer = setInterval(() => {
|
|
645
|
+
this.startQueuedMaintenance();
|
|
646
|
+
}, this.maintenanceIntervalMs);
|
|
647
|
+
this.maintenanceTimer.unref?.();
|
|
648
|
+
// Do not make a restored queue wait a full interval before its first
|
|
649
|
+
// attempt. MCPL/module initialization has completed before start().
|
|
650
|
+
this.startQueuedMaintenance();
|
|
651
|
+
}
|
|
523
652
|
}
|
|
524
653
|
|
|
525
654
|
/**
|
|
@@ -548,6 +677,14 @@ export class AgentFramework {
|
|
|
548
677
|
this.syncTimer = null;
|
|
549
678
|
}
|
|
550
679
|
|
|
680
|
+
if (this.maintenanceTimer) {
|
|
681
|
+
clearInterval(this.maintenanceTimer);
|
|
682
|
+
this.maintenanceTimer = null;
|
|
683
|
+
}
|
|
684
|
+
if (this.maintenancePass) {
|
|
685
|
+
await this.maintenancePass;
|
|
686
|
+
}
|
|
687
|
+
|
|
551
688
|
if (this.loopPromise) {
|
|
552
689
|
await this.loopPromise;
|
|
553
690
|
}
|
|
@@ -578,6 +715,103 @@ export class AgentFramework {
|
|
|
578
715
|
}
|
|
579
716
|
}
|
|
580
717
|
|
|
718
|
+
/**
|
|
719
|
+
* Advance queued ContextManager work without requiring a new message.
|
|
720
|
+
*
|
|
721
|
+
* Tool definitions are refreshed first because compression replays stored
|
|
722
|
+
* tool cycles and providers reject those requests when the corresponding
|
|
723
|
+
* schemas are absent. Passes never overlap; each agent gets a bounded drain
|
|
724
|
+
* so maintenance cannot monopolize the framework event loop.
|
|
725
|
+
*/
|
|
726
|
+
private startQueuedMaintenance(): void {
|
|
727
|
+
if (this.maintenancePass || !this.running) return;
|
|
728
|
+
const pass = this.runQueuedMaintenance().catch((error) => {
|
|
729
|
+
console.error(
|
|
730
|
+
'[context-maintenance] pass failed:',
|
|
731
|
+
error instanceof Error ? error.message : error,
|
|
732
|
+
);
|
|
733
|
+
});
|
|
734
|
+
this.maintenancePass = pass;
|
|
735
|
+
void pass.finally(() => {
|
|
736
|
+
if (this.maintenancePass === pass) this.maintenancePass = null;
|
|
737
|
+
});
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
private async runQueuedMaintenance(): Promise<void> {
|
|
741
|
+
const allTools = this.getAllTools();
|
|
742
|
+
const queued = [...this.agents.values()].flatMap((agent) => {
|
|
743
|
+
const cm = agent.getContextManager();
|
|
744
|
+
const tools = allTools.filter((tool) => agent.canUseTool(tool.name));
|
|
745
|
+
cm.setToolDefinitions(tools);
|
|
746
|
+
if (cm.isReady()) return [];
|
|
747
|
+
const pending = cm.getPendingWork()?.description;
|
|
748
|
+
const progress = this.contextProgress(cm);
|
|
749
|
+
const record: ContextMaintenanceAgentRun = {
|
|
750
|
+
agentName: agent.name,
|
|
751
|
+
startedAt: Date.now(),
|
|
752
|
+
ticks: 0,
|
|
753
|
+
readyBefore: false,
|
|
754
|
+
...(pending ? { pendingBefore: pending } : {}),
|
|
755
|
+
...(progress ? { progressBefore: progress } : {}),
|
|
756
|
+
};
|
|
757
|
+
return [{
|
|
758
|
+
agent,
|
|
759
|
+
cm,
|
|
760
|
+
record,
|
|
761
|
+
}];
|
|
762
|
+
});
|
|
763
|
+
if (queued.length === 0) return;
|
|
764
|
+
|
|
765
|
+
const run: ContextMaintenanceRun = {
|
|
766
|
+
id: ++this.maintenanceRunId,
|
|
767
|
+
startedAt: Date.now(),
|
|
768
|
+
agents: queued.map(item => item.record),
|
|
769
|
+
};
|
|
770
|
+
this.currentMaintenanceRun = run;
|
|
771
|
+
|
|
772
|
+
try {
|
|
773
|
+
await Promise.all(queued.map(async ({ agent, cm, record }) => {
|
|
774
|
+
try {
|
|
775
|
+
for (
|
|
776
|
+
let i = 0;
|
|
777
|
+
i < MAINTENANCE_TICKS_PER_PASS && this.running && !cm.isReady();
|
|
778
|
+
i++
|
|
779
|
+
) {
|
|
780
|
+
await cm.tick();
|
|
781
|
+
record.ticks++;
|
|
782
|
+
}
|
|
783
|
+
} catch (error) {
|
|
784
|
+
record.error = error instanceof Error ? error.message : String(error);
|
|
785
|
+
console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
|
|
786
|
+
} finally {
|
|
787
|
+
const pending = cm.getPendingWork()?.description;
|
|
788
|
+
record.finishedAt = Date.now();
|
|
789
|
+
record.readyAfter = cm.isReady();
|
|
790
|
+
if (pending) record.pendingAfter = pending;
|
|
791
|
+
const progress = this.contextProgress(cm);
|
|
792
|
+
if (progress) record.progressAfter = progress;
|
|
793
|
+
}
|
|
794
|
+
}));
|
|
795
|
+
} finally {
|
|
796
|
+
run.finishedAt = Date.now();
|
|
797
|
+
run.agents.sort((a, b) => a.agentName.localeCompare(b.agentName));
|
|
798
|
+
this.currentMaintenanceRun = null;
|
|
799
|
+
this.maintenanceHistory.unshift(run);
|
|
800
|
+
if (this.maintenanceHistory.length > 50) this.maintenanceHistory.length = 50;
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
private contextProgress(cm: ContextManager): Record<string, unknown> | undefined {
|
|
805
|
+
const strategy = cm.getStrategy() as {
|
|
806
|
+
getProgressSnapshot?: () => unknown;
|
|
807
|
+
};
|
|
808
|
+
if (typeof strategy.getProgressSnapshot !== 'function') return undefined;
|
|
809
|
+
const progress = strategy.getProgressSnapshot();
|
|
810
|
+
return progress && typeof progress === 'object'
|
|
811
|
+
? progress as Record<string, unknown>
|
|
812
|
+
: undefined;
|
|
813
|
+
}
|
|
814
|
+
|
|
581
815
|
/**
|
|
582
816
|
* Push a process event to the queue.
|
|
583
817
|
*/
|
|
@@ -700,6 +934,28 @@ export class AgentFramework {
|
|
|
700
934
|
return [...moduleTools, ...this.mcplTools, ...channelTools, ...gateTools];
|
|
701
935
|
}
|
|
702
936
|
|
|
937
|
+
/** Counts-only context-maintenance diagnostics for authenticated debug UIs. */
|
|
938
|
+
getContextMaintenanceSnapshot(): ContextMaintenanceSnapshot {
|
|
939
|
+
const agents = [...this.agents.values()].map((agent) => {
|
|
940
|
+
const cm = agent.getContextManager();
|
|
941
|
+
const pending = cm.getPendingWork()?.description;
|
|
942
|
+
const progress = this.contextProgress(cm);
|
|
943
|
+
return {
|
|
944
|
+
agentName: agent.name,
|
|
945
|
+
ready: cm.isReady(),
|
|
946
|
+
...(pending ? { pending } : {}),
|
|
947
|
+
...(progress ? { progress } : {}),
|
|
948
|
+
};
|
|
949
|
+
});
|
|
950
|
+
return structuredClone({
|
|
951
|
+
intervalMs: this.maintenanceIntervalMs,
|
|
952
|
+
ticksPerPass: MAINTENANCE_TICKS_PER_PASS,
|
|
953
|
+
current: this.currentMaintenanceRun,
|
|
954
|
+
history: this.maintenanceHistory,
|
|
955
|
+
agents,
|
|
956
|
+
});
|
|
957
|
+
}
|
|
958
|
+
|
|
703
959
|
/**
|
|
704
960
|
* Build the membrane-normalized request that WOULD be emitted if `agentName`
|
|
705
961
|
* were activated right now — WITHOUT running inference, opening a stream, or
|
|
@@ -749,8 +1005,9 @@ export class AgentFramework {
|
|
|
749
1005
|
// Module gatherContext (fail-open, matches startAgentStream)
|
|
750
1006
|
try {
|
|
751
1007
|
const moduleInjections = await this.moduleRegistry.gatherContext(agentName);
|
|
752
|
-
|
|
753
|
-
|
|
1008
|
+
const scopedModuleInjections = this.scopeInjectionsForAgent(agentName, moduleInjections);
|
|
1009
|
+
if (scopedModuleInjections.length > 0) {
|
|
1010
|
+
injections = scopedModuleInjections;
|
|
754
1011
|
}
|
|
755
1012
|
} catch (error) {
|
|
756
1013
|
console.error('Module gatherContext error (preview):', error);
|
|
@@ -761,7 +1018,10 @@ export class AgentFramework {
|
|
|
761
1018
|
if (this.hookOrchestrator) {
|
|
762
1019
|
try {
|
|
763
1020
|
const hookParams = this.buildBeforeInferenceParams(agent);
|
|
764
|
-
const hookInjections =
|
|
1021
|
+
const hookInjections = this.scopeInjectionsForAgent(
|
|
1022
|
+
agentName,
|
|
1023
|
+
await this.hookOrchestrator.beforeInference(hookParams),
|
|
1024
|
+
);
|
|
765
1025
|
if (hookInjections.length > 0) {
|
|
766
1026
|
injections = injections ? [...injections, ...hookInjections] : hookInjections;
|
|
767
1027
|
}
|
|
@@ -951,6 +1211,7 @@ export class AgentFramework {
|
|
|
951
1211
|
clearInterval(completionWatchdog);
|
|
952
1212
|
this.offTrace(traceListener);
|
|
953
1213
|
this.agents.delete(agent.name);
|
|
1214
|
+
this.evictTurnCheckpoints(agent.name);
|
|
954
1215
|
};
|
|
955
1216
|
|
|
956
1217
|
// Watchdog: if inference never starts within 30s, the agent is a zombie.
|
|
@@ -1309,6 +1570,194 @@ export class AgentFramework {
|
|
|
1309
1570
|
}
|
|
1310
1571
|
}
|
|
1311
1572
|
|
|
1573
|
+
/**
|
|
1574
|
+
* Rewind the single turn that fed a refused generation: redact the newest
|
|
1575
|
+
* message the agent did NOT author (a tool result, or an incoming message)
|
|
1576
|
+
* and inject a metadata-only marker in its place. The marker carries a
|
|
1577
|
+
* *description* of what was withheld (kind + size), never the content — so it
|
|
1578
|
+
* cannot itself re-trip the classifier — while the raw record survives in the
|
|
1579
|
+
* chronicle log for forensics. Returns a record describing the removal, or
|
|
1580
|
+
* null if there is nothing eligible to rewind (e.g. only the agent's own
|
|
1581
|
+
* turns, or a prior system marker, remain).
|
|
1582
|
+
*
|
|
1583
|
+
* Shared by the auto path (on `stop_reason: refusal`) and the `/rewind`
|
|
1584
|
+
* host command.
|
|
1585
|
+
*/
|
|
1586
|
+
private shedNewestTurn(agent: Agent): RewindRecord | null {
|
|
1587
|
+
const cm = agent.getContextManager();
|
|
1588
|
+
const all = cm.getAllMessages();
|
|
1589
|
+
const typeOf = (b: unknown) => (b as { type?: string }).type;
|
|
1590
|
+
const hasBlock = (m: { content?: unknown } | undefined, t: string) =>
|
|
1591
|
+
Array.isArray(m?.content) && (m!.content as unknown[]).some((b) => typeOf(b) === t);
|
|
1592
|
+
|
|
1593
|
+
// Newest message that is not our own episode marker. We shed strictly
|
|
1594
|
+
// newest-first, in sequence — INCLUDING the agent's own turns (poison lives
|
|
1595
|
+
// in tool_use/narration turns too), because newest-first reaches whatever is
|
|
1596
|
+
// poisoning the context if allowed to run deep enough. The one invariant:
|
|
1597
|
+
// shed COMPLETE exchanges. Removing a `tool_result` also removes its paired
|
|
1598
|
+
// `tool_use` assistant turn, so we never leave an orphaned tool_use / signed
|
|
1599
|
+
// `thinking` block — which the API rejects with a 400 ("thinking blocks in
|
|
1600
|
+
// the latest assistant message cannot be modified"). We do NOT add a marker
|
|
1601
|
+
// here; the caller keeps one consolidated marker (updateRewindMarker).
|
|
1602
|
+
let idx = -1;
|
|
1603
|
+
for (let i = all.length - 1; i >= 0; i--) {
|
|
1604
|
+
const meta = (all[i].metadata ?? {}) as { system?: unknown };
|
|
1605
|
+
if (meta.system) continue;
|
|
1606
|
+
idx = i; break;
|
|
1607
|
+
}
|
|
1608
|
+
if (idx < 0) return null;
|
|
1609
|
+
const msg = all[idx];
|
|
1610
|
+
const md = (msg.metadata ?? {}) as {
|
|
1611
|
+
messageId?: unknown; channelId?: unknown; system?: unknown;
|
|
1612
|
+
};
|
|
1613
|
+
|
|
1614
|
+
const content = Array.isArray(msg.content) ? msg.content : [];
|
|
1615
|
+
const toolBlocks = content.filter((b) => typeOf(b) === 'tool_result');
|
|
1616
|
+
const images = content.filter((b) => typeOf(b) === 'image').length;
|
|
1617
|
+
const textLen = content
|
|
1618
|
+
.filter((b) => typeOf(b) === 'text')
|
|
1619
|
+
.reduce((n, b) => n + String((b as { text?: string }).text ?? '').length, 0);
|
|
1620
|
+
|
|
1621
|
+
// Determine the complete exchange to remove.
|
|
1622
|
+
const removedIds: MessageId[] = [msg.id];
|
|
1623
|
+
let kind: RewindRecord['kind'];
|
|
1624
|
+
let descriptor: string;
|
|
1625
|
+
if (toolBlocks.length > 0) {
|
|
1626
|
+
kind = 'tool';
|
|
1627
|
+
// Pair the tool_result with the tool_use assistant turn right before it.
|
|
1628
|
+
if (idx - 1 >= 0 && hasBlock(all[idx - 1], 'tool_use')) {
|
|
1629
|
+
removedIds.push(all[idx - 1].id);
|
|
1630
|
+
}
|
|
1631
|
+
const sz = toolBlocks.reduce(
|
|
1632
|
+
(n, b) => n + String((b as { content?: unknown }).content ?? '').length, 0);
|
|
1633
|
+
descriptor = `a tool exchange (~${Math.max(1, Math.round(sz / 1024))}KB` +
|
|
1634
|
+
`${images ? `, ${images} image(s)` : ''})`;
|
|
1635
|
+
} else if (md.messageId) {
|
|
1636
|
+
kind = 'human';
|
|
1637
|
+
descriptor = `an incoming message from ${msg.participant} ` +
|
|
1638
|
+
`(${textLen} chars${images ? `, ${images} image(s)` : ''})`;
|
|
1639
|
+
} else {
|
|
1640
|
+
kind = 'other';
|
|
1641
|
+
descriptor = `a ${msg.participant} turn ` +
|
|
1642
|
+
`(${textLen} chars${images ? `, ${images} image(s)` : ''})`;
|
|
1643
|
+
}
|
|
1644
|
+
const discordRef = md.messageId && md.channelId
|
|
1645
|
+
? { channelId: String(md.channelId), messageId: String(md.messageId) }
|
|
1646
|
+
: undefined;
|
|
1647
|
+
|
|
1648
|
+
for (const id of removedIds) cm.removeMessage(id);
|
|
1649
|
+
return { kind, descriptor, removedIds, discordRef };
|
|
1650
|
+
}
|
|
1651
|
+
|
|
1652
|
+
/**
|
|
1653
|
+
* Maintain exactly ONE consolidated marker for the current rewind episode,
|
|
1654
|
+
* updated in place as more turns are shed. Six rewinds ⇒ one message that
|
|
1655
|
+
* says "the 6 most recent turns were set aside", not six separate notes — so
|
|
1656
|
+
* the context converges (shed N, add 1) instead of growing, and the marker
|
|
1657
|
+
* sits at the tail giving the model something actionable to answer once the
|
|
1658
|
+
* refusal clears. Returns the running count.
|
|
1659
|
+
*/
|
|
1660
|
+
private updateRewindMarker(
|
|
1661
|
+
agent: Agent,
|
|
1662
|
+
category: string,
|
|
1663
|
+
cause: 'refusal' | 'inference-failure' = 'refusal',
|
|
1664
|
+
): number {
|
|
1665
|
+
const cm = agent.getContextManager();
|
|
1666
|
+
const ep = this.rewindEpisode.get(agent.name);
|
|
1667
|
+
const count = (ep?.count ?? 0) + 1;
|
|
1668
|
+
const why = cause === 'refusal'
|
|
1669
|
+
? `the model refused on them (content filter: ${category}). Their content is ` +
|
|
1670
|
+
`not reproduced here (so this note can't re-trigger the filter); the ` +
|
|
1671
|
+
`originals remain in the raw record`
|
|
1672
|
+
: `the model API kept rejecting the conversation on them (${category}). ` +
|
|
1673
|
+
`Their content is not reproduced here (so this note can't re-trigger the ` +
|
|
1674
|
+
`rejection); the originals remain in the raw record`;
|
|
1675
|
+
const text =
|
|
1676
|
+
`[refusal-rewind] The ${count} most recent turn(s) were set aside because ` +
|
|
1677
|
+
`${why}. You are clear to continue — if you ` +
|
|
1678
|
+
`were mid-task, take a different approach; otherwise carry on with whatever ` +
|
|
1679
|
+
`is now in front of you, or briefly acknowledge the gap and ask what's next.`;
|
|
1680
|
+
const blocks: ContentBlock[] = [{ type: 'text', text }];
|
|
1681
|
+
if (ep) {
|
|
1682
|
+
cm.editMessage(ep.markerId, blocks);
|
|
1683
|
+
this.rewindEpisode.set(agent.name, { markerId: ep.markerId, count, category });
|
|
1684
|
+
} else {
|
|
1685
|
+
const id = cm.addMessage('user', blocks, {
|
|
1686
|
+
system: true, kind: 'refusal-rewind', category, count, cause,
|
|
1687
|
+
});
|
|
1688
|
+
this.rewindEpisode.set(agent.name, { markerId: id, count, category });
|
|
1689
|
+
}
|
|
1690
|
+
return count;
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1693
|
+
/**
|
|
1694
|
+
* Announce a refusal-rewind on the conversational surface (Discord), used
|
|
1695
|
+
* when the withheld turn was a *human* message so it isn't dropped silently.
|
|
1696
|
+
* Best-effort; mirrors reactToRefusal's locus resolution.
|
|
1697
|
+
*/
|
|
1698
|
+
private async announceRewind(
|
|
1699
|
+
agentName: string,
|
|
1700
|
+
rec: RewindRecord,
|
|
1701
|
+
category: string,
|
|
1702
|
+
): Promise<void> {
|
|
1703
|
+
try {
|
|
1704
|
+
const incoming = this.channelRegistry?.buildChannelContext()?.incoming;
|
|
1705
|
+
if (!incoming) return;
|
|
1706
|
+
const parts = incoming.channelId.split(':');
|
|
1707
|
+
if (parts[0] !== 'discord') return;
|
|
1708
|
+
const channelId = parts[parts.length - 1];
|
|
1709
|
+
const serverId = this.channelRegistry?.getChannelServerId(incoming.channelId);
|
|
1710
|
+
const server = serverId ? this.mcplServerRegistry?.getServer(serverId) : null;
|
|
1711
|
+
if (!server) return;
|
|
1712
|
+
await server.sendToolsCall('send_message', {
|
|
1713
|
+
channelId,
|
|
1714
|
+
content:
|
|
1715
|
+
`⚠️ I had to set aside ${rec.descriptor} — it tripped a content ` +
|
|
1716
|
+
`filter (${category}), so it's withheld from my context and I'm ` +
|
|
1717
|
+
`continuing without it. If it was important, please rephrase or re-send.`,
|
|
1718
|
+
});
|
|
1719
|
+
} catch (err) {
|
|
1720
|
+
console.error(
|
|
1721
|
+
'[refusal-rewind] announce failed:',
|
|
1722
|
+
err instanceof Error ? err.message : err,
|
|
1723
|
+
);
|
|
1724
|
+
}
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1727
|
+
/**
|
|
1728
|
+
* Conclude an active `/unstick` session: post the outcome (what was shed and
|
|
1729
|
+
* whether the model stopped refusing) to the channel the command came from,
|
|
1730
|
+
* then clear the session. Idempotent — a no-op if there's no session.
|
|
1731
|
+
*/
|
|
1732
|
+
private finishUnstick(agentName: string, success: boolean, category?: string): void {
|
|
1733
|
+
const s = this.forcedRewind.get(agentName);
|
|
1734
|
+
if (!s) return;
|
|
1735
|
+
this.forcedRewind.delete(agentName);
|
|
1736
|
+
const n = s.removed.length;
|
|
1737
|
+
const list = n
|
|
1738
|
+
? '\n' + s.removed.map((r) => `• ${r.descriptor}`).join('\n')
|
|
1739
|
+
: '';
|
|
1740
|
+
const content = success
|
|
1741
|
+
? `🔧 Unstuck **${agentName}** — shed ${n} turn(s); the model responded.${list}`
|
|
1742
|
+
: `⚠️ Couldn't unstick **${agentName}** — still refusing after ${n} rewind(s)` +
|
|
1743
|
+
`${category ? ` (category=${category})` : ''}.${list}`;
|
|
1744
|
+
console.error(
|
|
1745
|
+
`[unstick] agent=${agentName} ${success ? 'succeeded' : 'gave up'} after ${n} rewind(s)`,
|
|
1746
|
+
);
|
|
1747
|
+
if (s.channelId) void this.postToChannel(s.serverId, s.channelId, content);
|
|
1748
|
+
}
|
|
1749
|
+
|
|
1750
|
+
/** Best-effort send_message to a raw channel via a named MCPL server. */
|
|
1751
|
+
private async postToChannel(serverId: string, channelId: string, content: string): Promise<void> {
|
|
1752
|
+
try {
|
|
1753
|
+
const server = this.mcplServerRegistry?.getServer(serverId);
|
|
1754
|
+
if (!server) return;
|
|
1755
|
+
await server.sendToolsCall('send_message', { channelId, content });
|
|
1756
|
+
} catch (err) {
|
|
1757
|
+
console.error('[unstick] report post failed:', err instanceof Error ? err.message : err);
|
|
1758
|
+
}
|
|
1759
|
+
}
|
|
1760
|
+
|
|
1312
1761
|
/**
|
|
1313
1762
|
* Handle a `host/command` request from an MCPL surface server (e.g. a
|
|
1314
1763
|
* Discord slash command). Currently supports:
|
|
@@ -1332,8 +1781,12 @@ export class AgentFramework {
|
|
|
1332
1781
|
* that carried one — so the surface can mark them with a reaction. */
|
|
1333
1782
|
hiddenRefs?: Array<{ channelId: string; messageId: string }>;
|
|
1334
1783
|
lastVisible?: { participant?: string; role?: string; preview?: string } | null;
|
|
1784
|
+
/** For `unstick`: acknowledges the forced-rewind loop has started; the
|
|
1785
|
+
* outcome report is posted to the channel asynchronously. */
|
|
1786
|
+
started?: boolean;
|
|
1787
|
+
cap?: number;
|
|
1335
1788
|
}> {
|
|
1336
|
-
if (params.command !== 'undo' && params.command !== 'hide') {
|
|
1789
|
+
if (params.command !== 'undo' && params.command !== 'hide' && params.command !== 'unstick') {
|
|
1337
1790
|
return { ok: false, error: `Unknown host command: ${String(params.command)}` };
|
|
1338
1791
|
}
|
|
1339
1792
|
|
|
@@ -1342,6 +1795,38 @@ export class AgentFramework {
|
|
|
1342
1795
|
return { ok: false, error: `Unknown agent: ${String(agentName)}` };
|
|
1343
1796
|
}
|
|
1344
1797
|
|
|
1798
|
+
// unstick: force the refusal-rewind loop on demand (even if the agent's
|
|
1799
|
+
// autoRewind toggle is off). Redacts the turn that fed the refusal and
|
|
1800
|
+
// re-runs, up to `cap` times, until the model stops refusing. Kicks the
|
|
1801
|
+
// loop and returns immediately; the outcome (what was shed + whether it
|
|
1802
|
+
// cleared) is posted to the channel when the chain resolves.
|
|
1803
|
+
if (params.command === 'unstick') {
|
|
1804
|
+
const agent = this.agents.get(agentName)!;
|
|
1805
|
+
if (agent.state.status !== 'idle') {
|
|
1806
|
+
return { ok: false, error: `Cannot unstick while agent is ${agent.state.status}` };
|
|
1807
|
+
}
|
|
1808
|
+
const cap = Math.max(1, Math.min(10,
|
|
1809
|
+
Math.floor(params.maxRewinds ?? agent.refusalHandling?.maxRewinds ?? 3)));
|
|
1810
|
+
this.forcedRewind.set(agentName, {
|
|
1811
|
+
remaining: cap,
|
|
1812
|
+
removed: [],
|
|
1813
|
+
serverId,
|
|
1814
|
+
channelId: params.channelId ?? '',
|
|
1815
|
+
});
|
|
1816
|
+
this.refusalRewinds.set(agentName, 0);
|
|
1817
|
+
this.pendingRequests.push({
|
|
1818
|
+
agentName,
|
|
1819
|
+
reason: 'unstick',
|
|
1820
|
+
source: 'framework',
|
|
1821
|
+
timestamp: Date.now(),
|
|
1822
|
+
});
|
|
1823
|
+
console.error(
|
|
1824
|
+
`[unstick] agent=${agentName} started cap=${cap} ` +
|
|
1825
|
+
`by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`,
|
|
1826
|
+
);
|
|
1827
|
+
return { ok: true, started: true, cap };
|
|
1828
|
+
}
|
|
1829
|
+
|
|
1345
1830
|
// hide: redact a single message (or an inclusive range) from the active
|
|
1346
1831
|
// branch, addressed by Discord message id. Unlike undo this is a
|
|
1347
1832
|
// removal-in-place (chronicle redact), not a branch rewind.
|
|
@@ -1959,18 +2444,91 @@ export class AgentFramework {
|
|
|
1959
2444
|
this.saveTurnCheckpoints(agentName, checkpoints);
|
|
1960
2445
|
}
|
|
1961
2446
|
|
|
2447
|
+
/**
|
|
2448
|
+
* Checkpoints live in one Tree state (TURN_CHECKPOINTS_TREE_ID) keyed by agent
|
|
2449
|
+
* name, each key pointing at a JSON blob of that agent's ≤MAX_TURN_CHECKPOINTS
|
|
2450
|
+
* list. This kills two unbounded-in-agents-ever dimensions at once:
|
|
2451
|
+
*
|
|
2452
|
+
* - the original single Record<agentName, list> map was rewritten whole on
|
|
2453
|
+
* every turn of every agent and never evicted a departed subagent's key;
|
|
2454
|
+
* - the intermediate design (one state slot per agent) fixed the writes but
|
|
2455
|
+
* leaked a permanent chronicle state *registration* per spawn — chronicle
|
|
2456
|
+
* has no deregistration, and the state index is rewritten and fsynced on
|
|
2457
|
+
* every sync tick, so per-tick cost grew with fleet history anyway.
|
|
2458
|
+
*
|
|
2459
|
+
* A tree gives per-key O(entry) writes, real key removal (treeRemove), and
|
|
2460
|
+
* exactly one registration for the life of the store. The legacy map state
|
|
2461
|
+
* (TURN_CHECKPOINTS_ID) is kept as a read-only fallback for stores written
|
|
2462
|
+
* before the split; eviction tombstones legacy keys so a reused agent name
|
|
2463
|
+
* can't inherit a dead agent's checkpoints through the fallback.
|
|
2464
|
+
*/
|
|
2465
|
+
private legacyCheckpointKeys: Set<string> | null = null;
|
|
2466
|
+
|
|
2467
|
+
private hasLegacyCheckpoints(agentName: string): boolean {
|
|
2468
|
+
if (!this.legacyCheckpointKeys) {
|
|
2469
|
+
const legacy = this.store.getStateJson(TURN_CHECKPOINTS_ID);
|
|
2470
|
+
this.legacyCheckpointKeys = new Set(
|
|
2471
|
+
legacy && typeof legacy === 'object' ? Object.keys(legacy) : []
|
|
2472
|
+
);
|
|
2473
|
+
}
|
|
2474
|
+
return this.legacyCheckpointKeys.has(agentName);
|
|
2475
|
+
}
|
|
2476
|
+
|
|
1962
2477
|
private getTurnCheckpoints(agentName: string): TurnCheckpoint[] {
|
|
1963
|
-
const
|
|
1964
|
-
if (
|
|
1965
|
-
|
|
1966
|
-
|
|
2478
|
+
const entry = this.store.treeGet(TURN_CHECKPOINTS_TREE_ID, agentName);
|
|
2479
|
+
if (entry) {
|
|
2480
|
+
const blob = this.store.getBlob(entry.blobHash);
|
|
2481
|
+
if (!blob) return [];
|
|
2482
|
+
try {
|
|
2483
|
+
const parsed = JSON.parse(blob.toString());
|
|
2484
|
+
return Array.isArray(parsed) ? parsed : [];
|
|
2485
|
+
} catch {
|
|
2486
|
+
return [];
|
|
2487
|
+
}
|
|
2488
|
+
}
|
|
2489
|
+
if (!this.hasLegacyCheckpoints(agentName)) return [];
|
|
2490
|
+
const legacy = this.store.getStateJson(TURN_CHECKPOINTS_ID) as
|
|
2491
|
+
| Record<string, TurnCheckpoint[]>
|
|
2492
|
+
| null;
|
|
2493
|
+
const list = legacy?.[agentName];
|
|
2494
|
+
return Array.isArray(list) ? [...list] : [];
|
|
1967
2495
|
}
|
|
1968
2496
|
|
|
1969
2497
|
private saveTurnCheckpoints(agentName: string, checkpoints: TurnCheckpoint[]): void {
|
|
1970
|
-
const
|
|
1971
|
-
const
|
|
1972
|
-
|
|
1973
|
-
|
|
2498
|
+
const bytes = Buffer.from(JSON.stringify(checkpoints));
|
|
2499
|
+
const blobHash = this.store.storeBlob(bytes, 'application/json');
|
|
2500
|
+
this.store.treeSet(TURN_CHECKPOINTS_TREE_ID, agentName, {
|
|
2501
|
+
blobHash,
|
|
2502
|
+
size: bytes.length,
|
|
2503
|
+
mode: 0o644,
|
|
2504
|
+
});
|
|
2505
|
+
}
|
|
2506
|
+
|
|
2507
|
+
/**
|
|
2508
|
+
* Drop a departed agent's checkpoint tree key and turn/redo bookkeeping.
|
|
2509
|
+
* Without this, spawn-and-dispose agents leave a key (and in-memory map
|
|
2510
|
+
* entries) behind for the life of the store/session.
|
|
2511
|
+
*/
|
|
2512
|
+
private evictTurnCheckpoints(agentName: string): void {
|
|
2513
|
+
this.turnCounters.delete(agentName);
|
|
2514
|
+
this.redoStacks.delete(agentName);
|
|
2515
|
+
// Diagnostics maps are also keyed by agent name and never evicted —
|
|
2516
|
+
// spawn-and-dispose fleets would grow them for the session's life.
|
|
2517
|
+
this.staleWarnAt.delete(agentName);
|
|
2518
|
+
this.lastInferenceAt.delete(agentName);
|
|
2519
|
+
const entry = this.store.treeGet(TURN_CHECKPOINTS_TREE_ID, agentName);
|
|
2520
|
+
if (this.hasLegacyCheckpoints(agentName)) {
|
|
2521
|
+
// A bare treeRemove would resurrect the legacy map's list through the
|
|
2522
|
+
// fallback next time this name is reused — shadow it with an empty
|
|
2523
|
+
// tombstone instead. size <= 2 ⇔ the blob is already "[]".
|
|
2524
|
+
if (!entry || entry.size > 2) {
|
|
2525
|
+
this.saveTurnCheckpoints(agentName, []);
|
|
2526
|
+
}
|
|
2527
|
+
return;
|
|
2528
|
+
}
|
|
2529
|
+
if (entry) {
|
|
2530
|
+
this.store.treeRemove(TURN_CHECKPOINTS_TREE_ID, agentName);
|
|
2531
|
+
}
|
|
1974
2532
|
}
|
|
1975
2533
|
|
|
1976
2534
|
/**
|
|
@@ -1980,6 +2538,9 @@ export class AgentFramework {
|
|
|
1980
2538
|
async runUntilIdle(): Promise<void> {
|
|
1981
2539
|
while (
|
|
1982
2540
|
!this.queue.isEmpty ||
|
|
2541
|
+
// Direct inference requests (e.g. runEphemeralToCompletion) bypass the
|
|
2542
|
+
// event queue — without this the loop can exit before they're drained.
|
|
2543
|
+
this.pendingRequests.length > 0 ||
|
|
1983
2544
|
this.activeStreams.size > 0 ||
|
|
1984
2545
|
Array.from(this.agents.values()).some((a) => a.state.status !== 'idle')
|
|
1985
2546
|
) {
|
|
@@ -2512,6 +3073,7 @@ export class AgentFramework {
|
|
|
2512
3073
|
this.agents.delete(agentName);
|
|
2513
3074
|
this.agentConfigs.delete(agentName);
|
|
2514
3075
|
this.conversationAgentHomes.delete(agentName);
|
|
3076
|
+
this.evictTurnCheckpoints(agentName);
|
|
2515
3077
|
this.emitTrace({
|
|
2516
3078
|
type: 'mcpl:conversation-disposed',
|
|
2517
3079
|
agentName,
|
|
@@ -2718,6 +3280,7 @@ export class AgentFramework {
|
|
|
2718
3280
|
requestCount: requests.length,
|
|
2719
3281
|
oldestRequestAge: now - oldest,
|
|
2720
3282
|
});
|
|
3283
|
+
console.error(`[inference-dropped] agent=${agentName} reason=agent_not_found requests=${requests.length}`);
|
|
2721
3284
|
continue;
|
|
2722
3285
|
}
|
|
2723
3286
|
|
|
@@ -2733,6 +3296,14 @@ export class AgentFramework {
|
|
|
2733
3296
|
requestCount: requests.length,
|
|
2734
3297
|
oldestRequestAge: now - oldest,
|
|
2735
3298
|
});
|
|
3299
|
+
// Loud (but throttled) note that requests are waiting on a busy agent.
|
|
3300
|
+
if ((this.staleWarnAt.get(agentName) ?? 0) < now - 60_000) {
|
|
3301
|
+
this.staleWarnAt.set(agentName, now);
|
|
3302
|
+
console.error(
|
|
3303
|
+
`[inference-stale] agent=${agentName} busy (${agent.state.status}) — ` +
|
|
3304
|
+
`${requests.length} request(s) waiting ${Math.round((now - oldest) / 1000)}s`,
|
|
3305
|
+
);
|
|
3306
|
+
}
|
|
2736
3307
|
}
|
|
2737
3308
|
this.pendingRequests.push(...requests);
|
|
2738
3309
|
continue;
|
|
@@ -2740,6 +3311,13 @@ export class AgentFramework {
|
|
|
2740
3311
|
|
|
2741
3312
|
// Check policy
|
|
2742
3313
|
if (!this.inferencePolicy.shouldInfer(agentName, requests, state)) {
|
|
3314
|
+
// Loud drop: a queued request that dies here is otherwise invisible —
|
|
3315
|
+
// the 2026-07-09 mythos "not responding" diagnosis burned hours on
|
|
3316
|
+
// exactly this class of silent drop. One stderr line per drop.
|
|
3317
|
+
console.error(
|
|
3318
|
+
`[inference-dropped] agent=${agentName} reason=policy-skip ` +
|
|
3319
|
+
`requests=${requests.length} triggers=${requests.map((r) => r.reason).join(',')}`,
|
|
3320
|
+
);
|
|
2743
3321
|
continue;
|
|
2744
3322
|
}
|
|
2745
3323
|
|
|
@@ -2782,6 +3360,7 @@ export class AgentFramework {
|
|
|
2782
3360
|
|
|
2783
3361
|
this.emitTrace({ type: 'inference:started', agentName: agent.name });
|
|
2784
3362
|
this.eventGate?.onInferenceStarted(agent.name);
|
|
3363
|
+
this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), startedAt: Date.now() });
|
|
2785
3364
|
|
|
2786
3365
|
try {
|
|
2787
3366
|
const allTools = this.getAllTools();
|
|
@@ -2791,14 +3370,16 @@ export class AgentFramework {
|
|
|
2791
3370
|
// Both produce ContextInjection[] that get merged before inference.
|
|
2792
3371
|
let injections: ContextInjection[] | undefined;
|
|
2793
3372
|
|
|
2794
|
-
// Module gatherContext (fail-open,
|
|
2795
|
-
//
|
|
2796
|
-
//
|
|
2797
|
-
//
|
|
2798
|
-
//
|
|
2799
|
-
// agent itself, or conversation forks will see cross-channel content.
|
|
3373
|
+
// Module gatherContext (fail-open, per-module timeout — the module's
|
|
3374
|
+
// contextTimeoutMs, else the registry default). Injections are
|
|
3375
|
+
// channel-scoped via scopeInjectionsForAgent, matching the MCPL hook
|
|
3376
|
+
// injections below, so conversation forks don't see cross-channel
|
|
3377
|
+
// content.
|
|
2800
3378
|
try {
|
|
2801
|
-
const moduleInjections =
|
|
3379
|
+
const moduleInjections = this.scopeInjectionsForAgent(
|
|
3380
|
+
agent.name,
|
|
3381
|
+
await this.moduleRegistry.gatherContext(agent.name),
|
|
3382
|
+
);
|
|
2802
3383
|
if (moduleInjections.length > 0) {
|
|
2803
3384
|
injections = moduleInjections;
|
|
2804
3385
|
}
|
|
@@ -2845,6 +3426,13 @@ export class AgentFramework {
|
|
|
2845
3426
|
type: 'inference:exhausted',
|
|
2846
3427
|
agentName: agent.name,
|
|
2847
3428
|
error: err.message,
|
|
3429
|
+
// Drives the poison-history breaker. `retryable` is kept for
|
|
3430
|
+
// observability, but the breaker gates on `errorType` — only an
|
|
3431
|
+
// `invalid_request` (a 400-class rejection of the history itself)
|
|
3432
|
+
// means retrying the same context can never succeed. auth/abort/
|
|
3433
|
+
// context_length are also non-retryable but must NOT cost history.
|
|
3434
|
+
retryable: err instanceof MembraneError ? err.retryable : undefined,
|
|
3435
|
+
errorType: err instanceof MembraneError ? err.type : undefined,
|
|
2848
3436
|
});
|
|
2849
3437
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
2850
3438
|
if (action.emit) {
|
|
@@ -3121,7 +3709,87 @@ export class AgentFramework {
|
|
|
3121
3709
|
`[inference-refusal] agent=${agent.name} category=${category}` +
|
|
3122
3710
|
(stopDetails?.explanation ? ` explanation=${stopDetails.explanation}` : ''),
|
|
3123
3711
|
);
|
|
3124
|
-
|
|
3712
|
+
|
|
3713
|
+
this.noteRefusal(agent.name, category, tokenUsage);
|
|
3714
|
+
|
|
3715
|
+
// Rewind: excise the turn that fed the refusal, drop a
|
|
3716
|
+
// metadata-only marker, and retry — keeping the agent on its own
|
|
3717
|
+
// model instead of substituting a fallback. Driven either by an
|
|
3718
|
+
// admin `/unstick` (forced session) or the agent's autoRewind
|
|
3719
|
+
// config; both are bounded.
|
|
3720
|
+
let handledByRewind = false;
|
|
3721
|
+
const rh = agent.refusalHandling;
|
|
3722
|
+
const forced = this.forcedRewind.get(agent.name);
|
|
3723
|
+
// Shed exactly one more (newest, in-sequence) turn and keep the
|
|
3724
|
+
// single episode marker current. `budgetLeft` bounds the loop.
|
|
3725
|
+
const doRewind = (
|
|
3726
|
+
budgetLeft: boolean,
|
|
3727
|
+
onStep: (count: number) => void,
|
|
3728
|
+
onGiveUp: () => void,
|
|
3729
|
+
): void => {
|
|
3730
|
+
if (!budgetLeft) { onGiveUp(); return; }
|
|
3731
|
+
const rec = this.shedNewestTurn(agent);
|
|
3732
|
+
if (!rec) { onGiveUp(); return; }
|
|
3733
|
+
const count = this.updateRewindMarker(agent, category);
|
|
3734
|
+
onStep(count);
|
|
3735
|
+
if (forced) forced.removed.push(rec);
|
|
3736
|
+
this.pendingRequests.push({
|
|
3737
|
+
agentName: agent.name,
|
|
3738
|
+
reason: forced ? 'unstick-retry' : 'refusal-rewind-retry',
|
|
3739
|
+
source: 'framework',
|
|
3740
|
+
timestamp: Date.now(),
|
|
3741
|
+
});
|
|
3742
|
+
handledByRewind = true;
|
|
3743
|
+
};
|
|
3744
|
+
|
|
3745
|
+
if (forced) {
|
|
3746
|
+
doRewind(
|
|
3747
|
+
forced.remaining > 0,
|
|
3748
|
+
(count) => {
|
|
3749
|
+
forced.remaining -= 1;
|
|
3750
|
+
console.error(
|
|
3751
|
+
`[unstick] agent=${agent.name} shed ${count} turn(s) ` +
|
|
3752
|
+
`(remaining ${forced.remaining})`,
|
|
3753
|
+
);
|
|
3754
|
+
},
|
|
3755
|
+
() => this.finishUnstick(agent.name, false, category),
|
|
3756
|
+
);
|
|
3757
|
+
} else if (rh?.autoRewind) {
|
|
3758
|
+
const cap = Math.max(1, rh.maxRewinds ?? 3);
|
|
3759
|
+
const used = this.refusalRewinds.get(agent.name) ?? 0;
|
|
3760
|
+
doRewind(
|
|
3761
|
+
used < cap,
|
|
3762
|
+
(count) => {
|
|
3763
|
+
this.refusalRewinds.set(agent.name, used + 1);
|
|
3764
|
+
console.error(
|
|
3765
|
+
`[refusal-rewind] agent=${agent.name} shed ${count} turn(s) so far ` +
|
|
3766
|
+
`(cap ${cap})`,
|
|
3767
|
+
);
|
|
3768
|
+
},
|
|
3769
|
+
() => {
|
|
3770
|
+
console.error(
|
|
3771
|
+
`[refusal-rewind] agent=${agent.name} gave up: ` +
|
|
3772
|
+
`${used >= cap ? `cap ${cap} reached` : 'nothing left to shed'}`,
|
|
3773
|
+
);
|
|
3774
|
+
this.refusalRewinds.set(agent.name, 0);
|
|
3775
|
+
this.rewindEpisode.delete(agent.name);
|
|
3776
|
+
},
|
|
3777
|
+
);
|
|
3778
|
+
}
|
|
3779
|
+
|
|
3780
|
+
if (!handledByRewind) void this.reactToRefusal(agent.name, category);
|
|
3781
|
+
} else {
|
|
3782
|
+
// A turn that completed WITHOUT a refusal ends the rewind episode:
|
|
3783
|
+
// the model responded. Leave the consolidated marker in place as
|
|
3784
|
+
// the durable record; just clear the per-episode counters.
|
|
3785
|
+
if (this.forcedRewind.has(agent.name)) {
|
|
3786
|
+
this.finishUnstick(agent.name, true);
|
|
3787
|
+
}
|
|
3788
|
+
if (this.refusalRewinds.get(agent.name)) {
|
|
3789
|
+
this.refusalRewinds.set(agent.name, 0);
|
|
3790
|
+
}
|
|
3791
|
+
this.rewindEpisode.delete(agent.name);
|
|
3792
|
+
this.refusalStreak.delete(agent.name);
|
|
3125
3793
|
}
|
|
3126
3794
|
|
|
3127
3795
|
// Dispatch speech (and thoughts if any)
|
|
@@ -3212,13 +3880,21 @@ export class AgentFramework {
|
|
|
3212
3880
|
`(${silenced ? 'silencing tool / explicit send' : 'no prose'})`,
|
|
3213
3881
|
);
|
|
3214
3882
|
} else {
|
|
3883
|
+
// Snapshot the locus ONCE. This loop dispatches AFTER the agent
|
|
3884
|
+
// is idle (see PR #32 note below), so a queued inbound from
|
|
3885
|
+
// another channel can start a NEW turn and overwrite the per-agent
|
|
3886
|
+
// triggering channel (`activeTriggerChannels`) BETWEEN segments —
|
|
3887
|
+
// splitting one turn's reply across channels (the residual
|
|
3888
|
+
// multi-segment race left by 83dedf8's per-agent locus). Pin every
|
|
3889
|
+
// segment of THIS turn to the channel resolved right now.
|
|
3890
|
+
const turnLocus = this.channelRegistry.resolveLocus(agent.name);
|
|
3215
3891
|
console.error(
|
|
3216
|
-
`[routing] ${agent.name}: tool-call turn [${toolNames.join(', ')}] -> routing ${segments.length} prose segment(s) in order`,
|
|
3892
|
+
`[routing] ${agent.name}: tool-call turn [${toolNames.join(', ')}] -> routing ${segments.length} prose segment(s) in order -> ${turnLocus ?? '(default)'}`,
|
|
3217
3893
|
);
|
|
3218
3894
|
// Deliver sequentially (await each) so the segments land in order.
|
|
3219
3895
|
for (const seg of segments) {
|
|
3220
3896
|
try {
|
|
3221
|
-
await this.channelRegistry.routeSpeech(agent.name, seg);
|
|
3897
|
+
await this.channelRegistry.routeSpeech(agent.name, seg, turnLocus);
|
|
3222
3898
|
} catch (err) {
|
|
3223
3899
|
console.error('speech routing failed:', err);
|
|
3224
3900
|
}
|
|
@@ -3267,6 +3943,12 @@ export class AgentFramework {
|
|
|
3267
3943
|
type: 'inference:exhausted',
|
|
3268
3944
|
agentName: agent.name,
|
|
3269
3945
|
error: err.message,
|
|
3946
|
+
// Drives the poison-history breaker. `retryable` is kept for
|
|
3947
|
+
// observability, but the breaker gates on `errorType` — only an
|
|
3948
|
+
// `invalid_request` (a 400-class rejection of the history
|
|
3949
|
+
// itself) means retrying the same context can never succeed.
|
|
3950
|
+
retryable: err instanceof MembraneError ? err.retryable : undefined,
|
|
3951
|
+
errorType: err instanceof MembraneError ? err.type : undefined,
|
|
3270
3952
|
});
|
|
3271
3953
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
3272
3954
|
if (action.emit) {
|
|
@@ -3281,19 +3963,58 @@ export class AgentFramework {
|
|
|
3281
3963
|
// Only reset if this is still the active stream (a budget restart
|
|
3282
3964
|
// may have already started a new stream, bumping streamId)
|
|
3283
3965
|
if (agent.streamId === myStreamId) {
|
|
3966
|
+
const durationMs = Date.now() - startTime;
|
|
3284
3967
|
agent.reset();
|
|
3285
3968
|
this.emitTrace({
|
|
3286
3969
|
type: 'inference:exhausted',
|
|
3287
3970
|
agentName: agent.name,
|
|
3288
3971
|
error: `Stream aborted: ${reason}`,
|
|
3289
3972
|
});
|
|
3973
|
+
// Postmortem 2026-05-28 P2 #7: persist the abort to the
|
|
3974
|
+
// inference log so future investigations can attribute the
|
|
3975
|
+
// terminal cause without relying on live in-memory reducer
|
|
3976
|
+
// state. Without this, abort-terminated inferences are
|
|
3977
|
+
// invisible to forensic queries (only request-side telemetry
|
|
3978
|
+
// via llm-calls.jsonl shows them, and only by absence).
|
|
3979
|
+
this.logInference({
|
|
3980
|
+
timestamp: startTime,
|
|
3981
|
+
agentName: agent.name,
|
|
3982
|
+
requestId,
|
|
3983
|
+
success: false,
|
|
3984
|
+
error: `Stream aborted: ${reason}`,
|
|
3985
|
+
request: compiledRequest ?? { note: 'streaming request aborted' },
|
|
3986
|
+
durationMs,
|
|
3987
|
+
});
|
|
3290
3988
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
3291
3989
|
}
|
|
3292
3990
|
break;
|
|
3293
3991
|
}
|
|
3294
3992
|
|
|
3295
|
-
case 'usage':
|
|
3993
|
+
case 'usage': {
|
|
3296
3994
|
agent.lastStreamInputTokens = event.usage.inputTokens;
|
|
3995
|
+
|
|
3996
|
+
// Closed-loop estimator calibration (2026-07-12). Sample the REAL
|
|
3997
|
+
// prefix size of THIS API call (fresh + cache write + cache read)
|
|
3998
|
+
// and hand it to the context strategy, which accepts exactly one
|
|
3999
|
+
// sample per compile (its arm-once gate) and rejects out-of-band
|
|
4000
|
+
// ratios. It must be sampled HERE, per call: `response.details.
|
|
4001
|
+
// usage` at turn completion is CUMULATIVE across the tool-use
|
|
4002
|
+
// loop (5 calls x ~160k reported as 884k), which is not a
|
|
4003
|
+
// window-shaped number and drove the multiplier to 2.37 before
|
|
4004
|
+
// the guards caught it.
|
|
4005
|
+
try {
|
|
4006
|
+
const realTotal =
|
|
4007
|
+
(event.usage.inputTokens ?? 0) +
|
|
4008
|
+
(event.usage.cacheCreationTokens ?? 0) +
|
|
4009
|
+
(event.usage.cacheReadTokens ?? 0);
|
|
4010
|
+
const strat = (agent as unknown as {
|
|
4011
|
+
getContextManager?: () => { getStrategy?: () => unknown };
|
|
4012
|
+
}).getContextManager?.()?.getStrategy?.() as
|
|
4013
|
+
| { reportRealInputTokens?: (n: number) => void }
|
|
4014
|
+
| undefined;
|
|
4015
|
+
strat?.reportRealInputTokens?.(realTotal);
|
|
4016
|
+
} catch { /* calibration is best-effort */ }
|
|
4017
|
+
|
|
3297
4018
|
this.emitTrace({
|
|
3298
4019
|
type: 'inference:usage',
|
|
3299
4020
|
agentName: agent.name,
|
|
@@ -3305,12 +4026,14 @@ export class AgentFramework {
|
|
|
3305
4026
|
},
|
|
3306
4027
|
});
|
|
3307
4028
|
break;
|
|
4029
|
+
}
|
|
3308
4030
|
}
|
|
3309
4031
|
}
|
|
3310
4032
|
} catch (error) {
|
|
3311
4033
|
// Stream itself threw (unexpected) — no retry path here, so also emit
|
|
3312
4034
|
// inference:exhausted so ephemeral agent promises can settle.
|
|
3313
4035
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
4036
|
+
const durationMs = Date.now() - startTime;
|
|
3314
4037
|
this.emitTrace({
|
|
3315
4038
|
type: 'inference:failed',
|
|
3316
4039
|
agentName: agent.name,
|
|
@@ -3322,9 +4045,31 @@ export class AgentFramework {
|
|
|
3322
4045
|
agentName: agent.name,
|
|
3323
4046
|
error: err.message,
|
|
3324
4047
|
});
|
|
4048
|
+
// Postmortem 2026-05-28 P2 #7: catch-path failures (stream itself
|
|
4049
|
+
// threw) were previously only visible via in-memory trace listeners.
|
|
4050
|
+
// Persist to inference log for forensic attribution.
|
|
4051
|
+
this.logInference({
|
|
4052
|
+
timestamp: startTime,
|
|
4053
|
+
agentName: agent.name,
|
|
4054
|
+
requestId,
|
|
4055
|
+
success: false,
|
|
4056
|
+
error: `Stream threw: ${err.message}`,
|
|
4057
|
+
request: compiledRequest ?? { note: 'streaming request threw' },
|
|
4058
|
+
durationMs,
|
|
4059
|
+
});
|
|
3325
4060
|
agent.reset();
|
|
3326
4061
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
3327
4062
|
} finally {
|
|
4063
|
+
// Clear the gate's inference flag on EVERY exit path (paired with the
|
|
4064
|
+
// onInferenceStarted in startAgentStream). The branch-level calls above
|
|
4065
|
+
// are kept but are best-effort; if any exit path bypassed them the agent
|
|
4066
|
+
// stayed stuck in the gate's `inferring` set, which permanently BUFFERS
|
|
4067
|
+
// all incoming events → the agent silently stops waking on messages
|
|
4068
|
+
// (typing still stops, compression still runs — matching the observed
|
|
4069
|
+
// wedge). onInferenceEnded is idempotent, so a redundant call is safe.
|
|
4070
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
4071
|
+
this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), endedAt: Date.now() });
|
|
4072
|
+
|
|
3328
4073
|
// Stop the typing indicator on every exit path (complete, error,
|
|
3329
4074
|
// exhausted, abort) so it never sticks after the turn ends.
|
|
3330
4075
|
this.channelRegistry?.stopTyping();
|
|
@@ -3414,7 +4159,10 @@ export class AgentFramework {
|
|
|
3414
4159
|
if (blocks) {
|
|
3415
4160
|
return { toolUseId: callId, content: blocks, isError: false };
|
|
3416
4161
|
}
|
|
3417
|
-
|
|
4162
|
+
// JSON.stringify returns the VALUE undefined (not a string) for undefined
|
|
4163
|
+
// input — a module tool returning `{ success: true }` with no data would
|
|
4164
|
+
// otherwise make `content.length` below throw a TypeError mid-turn.
|
|
4165
|
+
let content = JSON.stringify(afResult.data) ?? '';
|
|
3418
4166
|
if (maxChars && content.length > maxChars) {
|
|
3419
4167
|
content = safeSlice(content, 0, maxChars)
|
|
3420
4168
|
+ '\n\n[truncated — original was ' + content.length + ' chars]';
|
|
@@ -3486,11 +4234,11 @@ export class AgentFramework {
|
|
|
3486
4234
|
}
|
|
3487
4235
|
}
|
|
3488
4236
|
|
|
3489
|
-
// Append
|
|
3490
|
-
|
|
3491
|
-
|
|
3492
|
-
|
|
3493
|
-
this.store.
|
|
4237
|
+
// Append one record per inference. A read-modify-write via setStateJson
|
|
4238
|
+
// would emit a whole-array Set — record size then grows with accumulated
|
|
4239
|
+
// history (O(n²) aggregate disk) and the append_log strategy registered
|
|
4240
|
+
// for this state never sees an append.
|
|
4241
|
+
this.store.appendToStateJson(INFERENCE_LOG_ID, entryToStore);
|
|
3494
4242
|
}
|
|
3495
4243
|
|
|
3496
4244
|
private logProcessEvent(event: ProcessEvent, responses: ModuleProcessResponse[]): void {
|
|
@@ -3510,11 +4258,8 @@ export class AgentFramework {
|
|
|
3510
4258
|
entryToStore.responses = { blobId };
|
|
3511
4259
|
}
|
|
3512
4260
|
|
|
3513
|
-
// Append
|
|
3514
|
-
|
|
3515
|
-
const entries = Array.isArray(data) ? data : [];
|
|
3516
|
-
entries.push(entryToStore);
|
|
3517
|
-
this.store.setStateJson(PROCESS_LOG_ID, entries);
|
|
4261
|
+
// Append one record per event — same rationale as logInference.
|
|
4262
|
+
this.store.appendToStateJson(PROCESS_LOG_ID, entryToStore);
|
|
3518
4263
|
}
|
|
3519
4264
|
|
|
3520
4265
|
/**
|
|
@@ -3744,14 +4489,20 @@ export class AgentFramework {
|
|
|
3744
4489
|
this.noteInferenceExhausted(
|
|
3745
4490
|
(event.agentName as string) ?? 'unknown',
|
|
3746
4491
|
(event.error as string) ?? 'unknown error',
|
|
4492
|
+
event.retryable as boolean | undefined,
|
|
4493
|
+
event.errorType as string | undefined,
|
|
3747
4494
|
);
|
|
3748
4495
|
} else if (event.type === 'inference:completed') {
|
|
3749
4496
|
// A successful response — even mid-turn between tool calls — proves the
|
|
3750
|
-
// agent isn't hard-down; clear its consecutive-failure streak
|
|
4497
|
+
// agent isn't hard-down; clear its consecutive-failure streak (and the
|
|
4498
|
+
// poison-history breaker's rewind budget).
|
|
3751
4499
|
const name = event.agentName as string | undefined;
|
|
3752
4500
|
if (name && this.consecutiveInferenceFailures.get(name)) {
|
|
3753
4501
|
this.consecutiveInferenceFailures.set(name, 0);
|
|
3754
4502
|
}
|
|
4503
|
+
if (name && this.exhaustionRewinds.get(name)) {
|
|
4504
|
+
this.exhaustionRewinds.set(name, 0);
|
|
4505
|
+
}
|
|
3755
4506
|
}
|
|
3756
4507
|
|
|
3757
4508
|
const traceEvent = {
|
|
@@ -3782,14 +4533,152 @@ export class AgentFramework {
|
|
|
3782
4533
|
* inference, so this never causes a retry/wake loop.
|
|
3783
4534
|
* 3. escalation — after N consecutive failures the agent is hard-down;
|
|
3784
4535
|
* log that loudly (a repeated identical failure is the textbook signal).
|
|
4536
|
+
* 4. breaker — when the failures are an `invalid_request` (a 400-class
|
|
4537
|
+
* rejection of the history itself, e.g. corrupted tool_use/tool_result
|
|
4538
|
+
* pairing or an oversized attachment), retrying the same context can
|
|
4539
|
+
* never succeed: every new push event wakes the agent onto the same
|
|
4540
|
+
* poisoned history forever. At the hard-down threshold, automatically
|
|
4541
|
+
* quarantine: shed the newest complete exchange (the same forced-rewind
|
|
4542
|
+
* primitive `/unstick` uses — shedNewestTurn never orphans a
|
|
4543
|
+
* tool_use/thinking block) and retry, bounded by the same rewind cap so
|
|
4544
|
+
* the breaker can never eat the whole history.
|
|
4545
|
+
*/
|
|
4546
|
+
/**
|
|
4547
|
+
* Live health snapshot for /healthz and doctor tooling: gate state, queued
|
|
4548
|
+
* work, per-agent status + last inference activity. Cheap and read-only.
|
|
3785
4549
|
*/
|
|
3786
|
-
|
|
4550
|
+
healthSnapshot(): Record<string, unknown> {
|
|
4551
|
+
return {
|
|
4552
|
+
at: new Date().toISOString(),
|
|
4553
|
+
uptimeSec: Math.round(process.uptime()),
|
|
4554
|
+
gate: this.eventGate?.inferenceDiagnostics() ?? null,
|
|
4555
|
+
pendingRequests: this.pendingRequests.length,
|
|
4556
|
+
activeStreams: [...this.activeStreams.keys()],
|
|
4557
|
+
agents: [...this.agents.entries()].map(([name, agent]) => ({
|
|
4558
|
+
name,
|
|
4559
|
+
status: agent.state.status,
|
|
4560
|
+
consecutiveInferenceFailures: this.consecutiveInferenceFailures.get(name) ?? 0,
|
|
4561
|
+
lastInference: this.lastInferenceAt.get(name) ?? null,
|
|
4562
|
+
refusalStats: this.refusalStats.get(name) ?? null,
|
|
4563
|
+
})),
|
|
4564
|
+
};
|
|
4565
|
+
}
|
|
4566
|
+
|
|
4567
|
+
/** Append a structured JSONL record to logs/failures.log (best-effort).
|
|
4568
|
+
* Durable and independent of journald/unit log redirects — this is what
|
|
4569
|
+
* connectome-doctor and fleet tooling read. Always stamps `at`. Legacy
|
|
4570
|
+
* records carry {at, agent, consecutive, reason}; new records add `kind`
|
|
4571
|
+
* and kind-specific fields (additive only — doctor parses by regex). */
|
|
4572
|
+
private logFailure(record: Record<string, unknown>): void {
|
|
4573
|
+
try {
|
|
4574
|
+
mkdirSync('logs', { recursive: true });
|
|
4575
|
+
appendFileSync(
|
|
4576
|
+
'logs/failures.log',
|
|
4577
|
+
JSON.stringify({ at: new Date().toISOString(), ...record }) + '\n',
|
|
4578
|
+
);
|
|
4579
|
+
} catch { /* best-effort */ }
|
|
4580
|
+
}
|
|
4581
|
+
|
|
4582
|
+
/**
|
|
4583
|
+
* Ops alert — the one escalation path for "a human should hear about this":
|
|
4584
|
+
* (1) durable failures.log record (unless the caller already wrote one),
|
|
4585
|
+
* (2) an `ops:alert` trace so authorized observers get it on the wire,
|
|
4586
|
+
* (3) if CONNECTOME_OPS_WEBHOOK is set, a Discord post throttled to one per
|
|
4587
|
+
* (agent, kind) per cooldown window — a persistent failure re-posts
|
|
4588
|
+
* every ~15 min instead of flooding the channel on every occurrence.
|
|
4589
|
+
* See connectome docs/observability.md.
|
|
4590
|
+
*/
|
|
4591
|
+
private opsAlert(
|
|
4592
|
+
kind: string,
|
|
4593
|
+
agentName: string,
|
|
4594
|
+
message: string,
|
|
4595
|
+
opts?: { data?: Record<string, unknown>; skipLog?: boolean },
|
|
4596
|
+
): void {
|
|
4597
|
+
if (!opts?.skipLog) {
|
|
4598
|
+
this.logFailure({ agent: agentName, kind, reason: message, ...opts?.data });
|
|
4599
|
+
}
|
|
4600
|
+
this.emitTrace({ type: 'ops:alert', kind, agentName, message, data: opts?.data });
|
|
4601
|
+
|
|
4602
|
+
const hook = process.env.CONNECTOME_OPS_WEBHOOK;
|
|
4603
|
+
if (!hook) return;
|
|
4604
|
+
const key = `${agentName}:${kind}`;
|
|
4605
|
+
const now = Date.now();
|
|
4606
|
+
if (now - (this.opsAlertLastSent.get(key) ?? 0) < this.opsAlertCooldownMs) return;
|
|
4607
|
+
// Stamp BEFORE the async post so a burst can't race past the cooldown.
|
|
4608
|
+
this.opsAlertLastSent.set(key, now);
|
|
4609
|
+
fetch(hook, {
|
|
4610
|
+
method: 'POST',
|
|
4611
|
+
headers: { 'content-type': 'application/json' },
|
|
4612
|
+
body: JSON.stringify({
|
|
4613
|
+
content: `\u{1F6A8} **${agentName}** ${kind}: ${message.slice(0, 500)}`,
|
|
4614
|
+
}),
|
|
4615
|
+
}).catch((err) => console.error(
|
|
4616
|
+
`[ops-alert] webhook post failed (${key}):`,
|
|
4617
|
+
err instanceof Error ? err.message : err,
|
|
4618
|
+
));
|
|
4619
|
+
}
|
|
4620
|
+
|
|
4621
|
+
/**
|
|
4622
|
+
* Record a model refusal for observability: per-agent stats + a
|
|
4623
|
+
* consecutive-refusal streak (exposed via healthSnapshot), a durable
|
|
4624
|
+
* failures.log record for EVERY refusal (previously invisible outside
|
|
4625
|
+
* stderr), and a throttled ops alert once refusals repeat — one refusal is
|
|
4626
|
+
* often survivable (auto-rewind may clear it); two in a row means stuck.
|
|
4627
|
+
* The streak is reset by any non-refusal completion in the stream driver.
|
|
4628
|
+
*/
|
|
4629
|
+
private noteRefusal(
|
|
4630
|
+
agentName: string,
|
|
4631
|
+
category: string,
|
|
4632
|
+
tokens?: { input: number; output: number },
|
|
4633
|
+
): number {
|
|
4634
|
+
const stats = this.refusalStats.get(agentName)
|
|
4635
|
+
?? { total: 0, byCategory: {}, lastAt: 0, lastCategory: '' };
|
|
4636
|
+
stats.total += 1;
|
|
4637
|
+
stats.byCategory[category] = (stats.byCategory[category] ?? 0) + 1;
|
|
4638
|
+
stats.lastAt = Date.now();
|
|
4639
|
+
stats.lastCategory = category;
|
|
4640
|
+
this.refusalStats.set(agentName, stats);
|
|
4641
|
+
|
|
4642
|
+
const streak = (this.refusalStreak.get(agentName) ?? 0) + 1;
|
|
4643
|
+
this.refusalStreak.set(agentName, streak);
|
|
4644
|
+
|
|
4645
|
+
this.logFailure({
|
|
4646
|
+
agent: agentName,
|
|
4647
|
+
kind: 'refusal',
|
|
4648
|
+
category,
|
|
4649
|
+
streak,
|
|
4650
|
+
locus: this.channelRegistry?.buildChannelContext()?.incoming?.channelId ?? null,
|
|
4651
|
+
tokens: tokens ?? null,
|
|
4652
|
+
});
|
|
4653
|
+
if (streak >= 2) {
|
|
4654
|
+
this.opsAlert(
|
|
4655
|
+
'refusal',
|
|
4656
|
+
agentName,
|
|
4657
|
+
`refusal streak ${streak}, category=${category}`,
|
|
4658
|
+
{ skipLog: true, data: { category, streak } },
|
|
4659
|
+
);
|
|
4660
|
+
}
|
|
4661
|
+
return streak;
|
|
4662
|
+
}
|
|
4663
|
+
|
|
4664
|
+
private noteInferenceExhausted(
|
|
4665
|
+
agentName: string,
|
|
4666
|
+
reason: string,
|
|
4667
|
+
retryable?: boolean,
|
|
4668
|
+
errorType?: string,
|
|
4669
|
+
): void {
|
|
3787
4670
|
const streak = (this.consecutiveInferenceFailures.get(agentName) ?? 0) + 1;
|
|
3788
4671
|
this.consecutiveInferenceFailures.set(agentName, streak);
|
|
4672
|
+
this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), failedAt: Date.now(), lastError: reason.slice(0, 300) });
|
|
3789
4673
|
|
|
3790
4674
|
// (1) Durable stderr line — works in headless/daemon mode with no client.
|
|
3791
4675
|
console.error(`[inference-failed] agent=${agentName} consecutive=${streak}: ${reason}`);
|
|
3792
4676
|
|
|
4677
|
+
// (1b) Machine-greppable durable record, independent of journald/unit log
|
|
4678
|
+
// redirects: logs/failures.log under the host's working directory. This is
|
|
4679
|
+
// what connectome-doctor reads. Legacy fields kept; `kind` is additive.
|
|
4680
|
+
this.logFailure({ agent: agentName, consecutive: streak, reason, kind: 'inference-exhausted' });
|
|
4681
|
+
|
|
3793
4682
|
// (2) Agent-facing chronicle marker (no inference triggered → no loop).
|
|
3794
4683
|
const agent = this.agents.get(agentName);
|
|
3795
4684
|
if (agent && process.env.SUPPRESS_INFERENCE_FAILED_MARKER !== '1') {
|
|
@@ -3818,7 +4707,80 @@ export class AgentFramework {
|
|
|
3818
4707
|
`[inference-hard-down] agent=${agentName} has FAILED ${streak} consecutive ` +
|
|
3819
4708
|
`inferences — it cannot complete a turn. Last reason: ${reason}`,
|
|
3820
4709
|
);
|
|
4710
|
+
|
|
4711
|
+
// Ops alert (webhook + ops:alert trace), throttled per (agent, kind) —
|
|
4712
|
+
// a hard-down agent re-posts every ~15 min, not on every failed retry.
|
|
4713
|
+
// failures.log already got the per-exhaustion record above.
|
|
4714
|
+
this.opsAlert(
|
|
4715
|
+
'hard-down',
|
|
4716
|
+
agentName,
|
|
4717
|
+
`${streak} consecutive inference failures. Last reason: ${reason}`,
|
|
4718
|
+
{ skipLog: true },
|
|
4719
|
+
);
|
|
4720
|
+
|
|
4721
|
+
// (4) Poison-history breaker: ONLY for `invalid_request` — a 400-class
|
|
4722
|
+
// rejection of the history itself. Deliberately NOT keyed on
|
|
4723
|
+
// `retryable === false`, which membrane also returns for auth (expired
|
|
4724
|
+
// key), abort (deliberate cancel), context_length (compression's job,
|
|
4725
|
+
// and shedding newest is the wrong direction), safety and unsupported —
|
|
4726
|
+
// none of which mean the history is poisoned, and all of which would
|
|
4727
|
+
// otherwise shed good exchanges and stamp a false "the API kept
|
|
4728
|
+
// rejecting your history" marker. `retryable` is kept only for the trace.
|
|
4729
|
+
void retryable;
|
|
4730
|
+
if (errorType === 'invalid_request' && agent) {
|
|
4731
|
+
this.quarantinePoisonedHistory(agent, reason);
|
|
4732
|
+
}
|
|
4733
|
+
}
|
|
4734
|
+
}
|
|
4735
|
+
|
|
4736
|
+
/**
|
|
4737
|
+
* Automatic poison-history quarantine (the actual "breaker"). Sheds the
|
|
4738
|
+
* newest complete exchange from the agent's history — reusing the same
|
|
4739
|
+
* primitives as the refusal auto-rewind / `/unstick` (shedNewestTurn +
|
|
4740
|
+
* the single consolidated episode marker) — and queues a retry so the
|
|
4741
|
+
* rewound history is verified immediately instead of waiting for the next
|
|
4742
|
+
* push event to wake the agent onto the same poisoned context.
|
|
4743
|
+
*
|
|
4744
|
+
* Bounded: at most `refusalHandling.maxRewinds` (default 3, hard cap 10)
|
|
4745
|
+
* sheds per failure episode; the budget resets on any successful inference.
|
|
4746
|
+
* At the cap (or with nothing left to shed) it stops — the agent stays up
|
|
4747
|
+
* and hard-down logging continues, but no further history is consumed.
|
|
4748
|
+
*/
|
|
4749
|
+
private quarantinePoisonedHistory(agent: Agent, reason: string): void {
|
|
4750
|
+
const cap = Math.max(1, Math.min(10, agent.refusalHandling?.maxRewinds ?? 3));
|
|
4751
|
+
const used = this.exhaustionRewinds.get(agent.name) ?? 0;
|
|
4752
|
+
if (used >= cap) {
|
|
4753
|
+
console.error(
|
|
4754
|
+
`[inference-rewind] agent=${agent.name} rewind cap ${cap} reached — ` +
|
|
4755
|
+
`not shedding further history. Manual repair (/unstick or /undo) needed.`,
|
|
4756
|
+
);
|
|
4757
|
+
return;
|
|
4758
|
+
}
|
|
4759
|
+
|
|
4760
|
+
const rec = this.shedNewestTurn(agent);
|
|
4761
|
+
if (!rec) {
|
|
4762
|
+
console.error(
|
|
4763
|
+
`[inference-rewind] agent=${agent.name} has nothing left to shed — ` +
|
|
4764
|
+
`history is only system markers. Manual repair needed.`,
|
|
4765
|
+
);
|
|
4766
|
+
return;
|
|
3821
4767
|
}
|
|
4768
|
+
|
|
4769
|
+
this.exhaustionRewinds.set(agent.name, used + 1);
|
|
4770
|
+
const count = this.updateRewindMarker(agent, truncateReason(reason), 'inference-failure');
|
|
4771
|
+
console.error(
|
|
4772
|
+
`[inference-rewind] agent=${agent.name} auto-quarantined ${rec.descriptor} ` +
|
|
4773
|
+
`(${count} turn(s) shed this episode, cap ${cap}) after non-retryable ` +
|
|
4774
|
+
`inference failures — retrying on the rewound history.`,
|
|
4775
|
+
);
|
|
4776
|
+
|
|
4777
|
+
// Retry immediately on the repaired history (bounded by the cap above).
|
|
4778
|
+
this.pendingRequests.push({
|
|
4779
|
+
agentName: agent.name,
|
|
4780
|
+
reason: 'inference-failure-rewind-retry',
|
|
4781
|
+
source: 'framework',
|
|
4782
|
+
timestamp: Date.now(),
|
|
4783
|
+
});
|
|
3822
4784
|
}
|
|
3823
4785
|
|
|
3824
4786
|
// ==========================================================================
|
|
@@ -3930,8 +4892,9 @@ export class AgentFramework {
|
|
|
3930
4892
|
},
|
|
3931
4893
|
);
|
|
3932
4894
|
|
|
3933
|
-
// Host capabilities advertised during the MCP handshake
|
|
3934
|
-
|
|
4895
|
+
// Host capabilities advertised during the MCP handshake — stored so
|
|
4896
|
+
// servers can also be connected later at runtime (connectMcplServer).
|
|
4897
|
+
this.mcplHostCapabilities = {
|
|
3935
4898
|
version: '0.4',
|
|
3936
4899
|
pushEvents: true,
|
|
3937
4900
|
contextHooks: {
|
|
@@ -3943,66 +4906,242 @@ export class AgentFramework {
|
|
|
3943
4906
|
|
|
3944
4907
|
for (const config of serverConfigs) {
|
|
3945
4908
|
try {
|
|
3946
|
-
|
|
3947
|
-
|
|
3948
|
-
|
|
3949
|
-
|
|
3950
|
-
}
|
|
4909
|
+
await this.connectMcplServerInternal(config);
|
|
4910
|
+
} catch (error) {
|
|
4911
|
+
// Fail-open: log and continue with remaining servers
|
|
4912
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
4913
|
+
console.error(`Failed to connect MCPL server "${config.id}":`, err.message);
|
|
4914
|
+
// Reaching this catch means no stub was created, so nothing will
|
|
4915
|
+
// retry — except in the edge case where addServer succeeded (with a
|
|
4916
|
+
// reconnecting stub) and a later setup step threw; config.reconnect
|
|
4917
|
+
// is the best cheap approximation of that.
|
|
4918
|
+
this.emitTrace({
|
|
4919
|
+
type: 'mcpl:server-connect-failed',
|
|
4920
|
+
serverId: config.id,
|
|
4921
|
+
error: err.message,
|
|
4922
|
+
attempt: 0,
|
|
4923
|
+
willRetry: config.reconnect === true,
|
|
4924
|
+
});
|
|
4925
|
+
}
|
|
4926
|
+
}
|
|
3951
4927
|
|
|
3952
|
-
|
|
3953
|
-
|
|
3954
|
-
|
|
3955
|
-
// handshake window (between setupMessageRouting and now).
|
|
3956
|
-
this.wireMcplEvents(connection);
|
|
3957
|
-
connection.ready();
|
|
3958
|
-
|
|
3959
|
-
// Initialize feature sets if server advertises MCPL capabilities
|
|
3960
|
-
if (connection.capabilities) {
|
|
3961
|
-
const updateParams = this.featureSetManager.initializeServer(
|
|
3962
|
-
config.id,
|
|
3963
|
-
connection.capabilities,
|
|
3964
|
-
{
|
|
3965
|
-
enabledFeatureSets: config.enabledFeatureSets,
|
|
3966
|
-
disabledFeatureSets: config.disabledFeatureSets,
|
|
3967
|
-
},
|
|
3968
|
-
);
|
|
4928
|
+
// Discover tools from all connected servers
|
|
4929
|
+
await this.refreshMcplTools();
|
|
4930
|
+
}
|
|
3969
4931
|
|
|
3970
|
-
|
|
3971
|
-
|
|
3972
|
-
|
|
3973
|
-
|
|
4932
|
+
/**
|
|
4933
|
+
* Connect a single MCPL server: register routing entries, open the
|
|
4934
|
+
* connection, wire events, and initialize feature sets / scopes /
|
|
4935
|
+
* checkpoints. Shared by startup (initializeMcpl) and the runtime
|
|
4936
|
+
* lifecycle API (connectMcplServer). Throws on connection failure.
|
|
4937
|
+
*/
|
|
4938
|
+
private async connectMcplServerInternal(
|
|
4939
|
+
config: import('./mcpl/types.js').McplServerConfig,
|
|
4940
|
+
): Promise<void> {
|
|
4941
|
+
if (!this.mcplServerRegistry || !this.mcplHostCapabilities) {
|
|
4942
|
+
throw new Error('MCPL subsystem is not initialized');
|
|
4943
|
+
}
|
|
3974
4944
|
|
|
3975
|
-
|
|
3976
|
-
|
|
3977
|
-
|
|
3978
|
-
|
|
4945
|
+
// Register prefix + config for tool dispatch routing (idempotent with
|
|
4946
|
+
// the pre-registration pass in initializeMcpl).
|
|
4947
|
+
const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
|
|
4948
|
+
this.mcplPrefixMap.set(prefix, config.id);
|
|
4949
|
+
this.mcplServerConfigs.set(config.id, config);
|
|
3979
4950
|
|
|
3980
|
-
|
|
3981
|
-
|
|
3982
|
-
|
|
3983
|
-
|
|
3984
|
-
|
|
3985
|
-
|
|
3986
|
-
|
|
3987
|
-
|
|
3988
|
-
|
|
3989
|
-
|
|
3990
|
-
|
|
3991
|
-
|
|
3992
|
-
|
|
4951
|
+
// Record per-server channel subscription policy before the server
|
|
4952
|
+
// registers channels — handleRegister fires during the handshake.
|
|
4953
|
+
if (config.channelSubscription !== undefined && this.channelRegistry) {
|
|
4954
|
+
this.channelRegistry.setSubscriptionPolicy(config.id, config.channelSubscription);
|
|
4955
|
+
}
|
|
4956
|
+
|
|
4957
|
+
const connection = await this.mcplServerRegistry.addServer(config, this.mcplHostCapabilities);
|
|
4958
|
+
|
|
4959
|
+
// Wire event listeners then flush any events that arrived during the
|
|
4960
|
+
// handshake window (between setupMessageRouting and now).
|
|
4961
|
+
this.wireMcplEvents(connection);
|
|
4962
|
+
connection.ready();
|
|
4963
|
+
|
|
4964
|
+
// Initialize feature sets if server advertises MCPL capabilities
|
|
4965
|
+
this.registerMcplServerFeatures(config, connection);
|
|
4966
|
+
|
|
4967
|
+
this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` });
|
|
4968
|
+
}
|
|
4969
|
+
|
|
4970
|
+
/**
|
|
4971
|
+
* (Re-)establish a server's host-side MCPL registration: feature-set state
|
|
4972
|
+
* (plus the featureSets/update sent to the server), scope patterns, and
|
|
4973
|
+
* stateful-feature-set checkpoint registration.
|
|
4974
|
+
*
|
|
4975
|
+
* Called on initial connect (connectMcplServerInternal) AND after every
|
|
4976
|
+
* auto-reconnect. The 'close' handler drops the feature-set registration, so
|
|
4977
|
+
* without the reconnect re-run FeatureSetManager.validateInbound would throw
|
|
4978
|
+
* "Unknown server" forever — every push event and inference request from the
|
|
4979
|
+
* revived server silently rejected until a full host restart.
|
|
4980
|
+
*
|
|
4981
|
+
* CheckpointManager.registerFeatureSet is idempotent (no-op for a key that
|
|
4982
|
+
* already has a tree), so checkpoint trees preserved across a transient
|
|
4983
|
+
* disconnect are resumed, not reset.
|
|
4984
|
+
*/
|
|
4985
|
+
private registerMcplServerFeatures(
|
|
4986
|
+
config: import('./mcpl/types.js').McplServerConfig,
|
|
4987
|
+
connection: McplServerConnection,
|
|
4988
|
+
): void {
|
|
4989
|
+
if (!connection.capabilities) return;
|
|
4990
|
+
|
|
4991
|
+
const updateParams = this.featureSetManager!.initializeServer(
|
|
4992
|
+
config.id,
|
|
4993
|
+
connection.capabilities,
|
|
4994
|
+
{
|
|
4995
|
+
enabledFeatureSets: config.enabledFeatureSets,
|
|
4996
|
+
disabledFeatureSets: config.disabledFeatureSets,
|
|
4997
|
+
},
|
|
4998
|
+
);
|
|
4999
|
+
|
|
5000
|
+
// Inform server which feature sets are enabled/disabled
|
|
5001
|
+
if (updateParams.enabled?.length || updateParams.disabled?.length) {
|
|
5002
|
+
connection.sendFeatureSetsUpdate(updateParams);
|
|
5003
|
+
}
|
|
5004
|
+
|
|
5005
|
+
// Configure scope whitelist/blacklist patterns
|
|
5006
|
+
if (config.scopes) {
|
|
5007
|
+
this.scopeManager!.configureAll(config.scopes);
|
|
5008
|
+
}
|
|
5009
|
+
|
|
5010
|
+
// Register stateful feature sets with checkpoint manager (Step 8)
|
|
5011
|
+
if (this.checkpointManager) {
|
|
5012
|
+
const declared = this.featureSetManager!.getDeclaredFeatureSets(config.id);
|
|
5013
|
+
if (declared) {
|
|
5014
|
+
for (const [fsName, fsDecl] of Object.entries(declared)) {
|
|
5015
|
+
if (fsDecl.rollback || fsDecl.hostState) {
|
|
5016
|
+
this.checkpointManager.registerFeatureSet(config.id, fsName, {
|
|
5017
|
+
hostState: fsDecl.hostState ?? false,
|
|
5018
|
+
rollback: fsDecl.rollback ?? false,
|
|
5019
|
+
});
|
|
3993
5020
|
}
|
|
3994
5021
|
}
|
|
3995
|
-
|
|
3996
|
-
this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` });
|
|
3997
|
-
} catch (error) {
|
|
3998
|
-
// Fail-open: log and continue with remaining servers
|
|
3999
|
-
const err = error instanceof Error ? error : new Error(String(error));
|
|
4000
|
-
console.error(`Failed to connect MCPL server "${config.id}":`, err.message);
|
|
4001
5022
|
}
|
|
4002
5023
|
}
|
|
5024
|
+
}
|
|
4003
5025
|
|
|
4004
|
-
|
|
5026
|
+
// ==========================================================================
|
|
5027
|
+
// Runtime MCPL server lifecycle (agent-facing hot deploy/restart/unload)
|
|
5028
|
+
// ==========================================================================
|
|
5029
|
+
|
|
5030
|
+
/**
|
|
5031
|
+
* Connect a new MCPL server at runtime. Refreshes the tool list and
|
|
5032
|
+
* notifies the agent of newly available tools. Throws if the MCPL
|
|
5033
|
+
* subsystem is not initialized, the id is already connected, or the
|
|
5034
|
+
* connection/handshake fails.
|
|
5035
|
+
*/
|
|
5036
|
+
async connectMcplServer(
|
|
5037
|
+
config: import('./mcpl/types.js').McplServerConfig,
|
|
5038
|
+
): Promise<void> {
|
|
5039
|
+
// Lazily bring up the MCPL subsystem — a framework that started with zero
|
|
5040
|
+
// configured servers can still deploy its first one at runtime.
|
|
5041
|
+
if (!this.mcplServerRegistry || !this.mcplHostCapabilities) {
|
|
5042
|
+
await this.initializeMcpl([], this.mcplInferenceRoutingConfig ?? undefined);
|
|
5043
|
+
}
|
|
5044
|
+
|
|
5045
|
+
// Same collision rules create() enforces at startup: the tool prefix must
|
|
5046
|
+
// not shadow a module name or another server's prefix.
|
|
5047
|
+
const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
|
|
5048
|
+
if (this.moduleRegistry.getAllModules().some(m => m.name === prefix)) {
|
|
5049
|
+
throw new Error(
|
|
5050
|
+
`MCPL server "${config.id}" toolPrefix "${prefix}" collides with module "${prefix}"`,
|
|
5051
|
+
);
|
|
5052
|
+
}
|
|
5053
|
+
const prefixOwner = this.mcplPrefixMap.get(prefix);
|
|
5054
|
+
if (prefixOwner && prefixOwner !== config.id) {
|
|
5055
|
+
throw new Error(
|
|
5056
|
+
`MCPL server "${config.id}" toolPrefix "${prefix}" collides with server "${prefixOwner}"`,
|
|
5057
|
+
);
|
|
5058
|
+
}
|
|
5059
|
+
|
|
5060
|
+
const oldToolNames = new Set(this.mcplTools.map(t => t.name));
|
|
5061
|
+
await this.connectMcplServerInternal(config);
|
|
4005
5062
|
await this.refreshMcplTools();
|
|
5063
|
+
this.emitMcplToolDiff(oldToolNames, config.id);
|
|
5064
|
+
}
|
|
5065
|
+
|
|
5066
|
+
/**
|
|
5067
|
+
* Disconnect an MCPL server at runtime: close the connection, destroy its
|
|
5068
|
+
* feature-set and checkpoint state (permanent removal — unlike a transient
|
|
5069
|
+
* transport close, which preserves checkpoints for the reconnect), remove
|
|
5070
|
+
* its channels from the registry, drop routing entries, and refresh tools.
|
|
5071
|
+
* No-op-ish if the server is not connected (still clears routing state).
|
|
5072
|
+
*/
|
|
5073
|
+
async disconnectMcplServer(id: string): Promise<void> {
|
|
5074
|
+
if (!this.mcplServerRegistry) {
|
|
5075
|
+
throw new Error('MCPL subsystem is not initialized');
|
|
5076
|
+
}
|
|
5077
|
+
const config = this.mcplServerConfigs.get(id);
|
|
5078
|
+
const oldToolNames = new Set(this.mcplTools.map(t => t.name));
|
|
5079
|
+
|
|
5080
|
+
await this.mcplServerRegistry.removeServer(id);
|
|
5081
|
+
this.channelRegistry?.removeServer(id);
|
|
5082
|
+
|
|
5083
|
+
// Permanent removal: also destroy feature-set and checkpoint state
|
|
5084
|
+
// explicitly. The 'close' handler usually does this, but a connection that
|
|
5085
|
+
// already transiently closed (reconnect pending) emits no second 'close'
|
|
5086
|
+
// from close(), and the transient path deliberately preserves checkpoints.
|
|
5087
|
+
this.featureSetManager?.removeServer(id);
|
|
5088
|
+
this.checkpointManager?.removeServer(id);
|
|
5089
|
+
|
|
5090
|
+
const prefix = config?.toolPrefix ?? `mcpl--${id}`;
|
|
5091
|
+
this.mcplPrefixMap.delete(prefix);
|
|
5092
|
+
this.mcplServerConfigs.delete(id);
|
|
5093
|
+
|
|
5094
|
+
await this.refreshMcplTools();
|
|
5095
|
+
this.emitMcplToolDiff(oldToolNames, id);
|
|
5096
|
+
}
|
|
5097
|
+
|
|
5098
|
+
/**
|
|
5099
|
+
* Restart an MCPL server at runtime: disconnect, then reconnect with the
|
|
5100
|
+
* same (or an updated) config. This actually respawns a stdio child —
|
|
5101
|
+
* unlike `reconnect: true`, which only retries after transport-level
|
|
5102
|
+
* failures. Throws if the server has no stored config and none is given.
|
|
5103
|
+
*/
|
|
5104
|
+
async restartMcplServer(
|
|
5105
|
+
id: string,
|
|
5106
|
+
newConfig?: import('./mcpl/types.js').McplServerConfig,
|
|
5107
|
+
): Promise<void> {
|
|
5108
|
+
const config = newConfig ?? this.mcplServerConfigs.get(id);
|
|
5109
|
+
if (!config) {
|
|
5110
|
+
throw new Error(`MCPL server "${id}" is not configured`);
|
|
5111
|
+
}
|
|
5112
|
+
await this.disconnectMcplServer(id);
|
|
5113
|
+
await this.connectMcplServer(config);
|
|
5114
|
+
}
|
|
5115
|
+
|
|
5116
|
+
/**
|
|
5117
|
+
* List configured MCPL servers with live connection status and the number
|
|
5118
|
+
* of tools each currently contributes.
|
|
5119
|
+
*/
|
|
5120
|
+
listMcplServers(): Array<{
|
|
5121
|
+
id: string;
|
|
5122
|
+
connected: boolean;
|
|
5123
|
+
toolPrefix: string;
|
|
5124
|
+
toolCount: number;
|
|
5125
|
+
command?: string;
|
|
5126
|
+
url?: string;
|
|
5127
|
+
}> {
|
|
5128
|
+
const result: Array<{
|
|
5129
|
+
id: string; connected: boolean; toolPrefix: string; toolCount: number;
|
|
5130
|
+
command?: string; url?: string;
|
|
5131
|
+
}> = [];
|
|
5132
|
+
for (const [id, config] of this.mcplServerConfigs) {
|
|
5133
|
+
const prefix = config.toolPrefix ?? `mcpl--${id}`;
|
|
5134
|
+
const connection = this.mcplServerRegistry?.getServer(id) ?? null;
|
|
5135
|
+
result.push({
|
|
5136
|
+
id,
|
|
5137
|
+
connected: connection?.isConnected ?? false,
|
|
5138
|
+
toolPrefix: prefix,
|
|
5139
|
+
toolCount: this.mcplTools.filter(t => t.name.startsWith(`${prefix}--`)).length,
|
|
5140
|
+
command: config.command,
|
|
5141
|
+
url: config.url,
|
|
5142
|
+
});
|
|
5143
|
+
}
|
|
5144
|
+
return result;
|
|
4006
5145
|
}
|
|
4007
5146
|
|
|
4008
5147
|
/**
|
|
@@ -4100,15 +5239,116 @@ export class AgentFramework {
|
|
|
4100
5239
|
this.handleToolsListChanged(connection.id);
|
|
4101
5240
|
});
|
|
4102
5241
|
|
|
4103
|
-
//
|
|
4104
|
-
|
|
5242
|
+
// Re-establish full server registration on reconnect. The 'close' handler
|
|
5243
|
+
// removed the feature-set state, so pushes / inference requests from the
|
|
5244
|
+
// revived server would otherwise be rejected with "Unknown server" until a
|
|
5245
|
+
// host restart. Re-runs the same init as the initial connect path (the
|
|
5246
|
+
// fresh handshake refreshed connection.capabilities); checkpoint trees
|
|
5247
|
+
// preserved across the transient close are resumed by idempotent
|
|
5248
|
+
// registration. Then refresh tools (server may have different tools).
|
|
5249
|
+
connection.on('reconnect', (info?: { attempts?: number }) => {
|
|
5250
|
+
try {
|
|
5251
|
+
const config = this.mcplServerConfigs.get(connection.id);
|
|
5252
|
+
if (config) {
|
|
5253
|
+
this.registerMcplServerFeatures(config, connection);
|
|
5254
|
+
}
|
|
5255
|
+
} catch (error) {
|
|
5256
|
+
console.error(
|
|
5257
|
+
`MCPL server "${connection.id}" re-registration after reconnect failed:`,
|
|
5258
|
+
error instanceof Error ? error.message : error,
|
|
5259
|
+
);
|
|
5260
|
+
}
|
|
4105
5261
|
this.handleToolsListChanged(connection.id);
|
|
5262
|
+
this.emitTrace({
|
|
5263
|
+
type: 'mcpl:server-reconnected',
|
|
5264
|
+
serverId: connection.id,
|
|
5265
|
+
attempts: info?.attempts ?? 0,
|
|
5266
|
+
});
|
|
5267
|
+
// Mirror the module:removed emitted on 'close' so module-lifecycle
|
|
5268
|
+
// consumers see the server come back, not just vanish.
|
|
5269
|
+
this.emitTrace({ type: 'module:added', moduleName: `mcpl:${connection.id}` });
|
|
5270
|
+
});
|
|
5271
|
+
|
|
5272
|
+
// Surface connect/reconnect failures. Before these traces existed the
|
|
5273
|
+
// only receipt was a console.error on the host's own stderr — invisible
|
|
5274
|
+
// unless someone ssh'd in and read the process log.
|
|
5275
|
+
connection.on('connect-failed', (params: { error: string; attempt: number }) => {
|
|
5276
|
+
this.emitTrace({
|
|
5277
|
+
type: 'mcpl:server-connect-failed',
|
|
5278
|
+
serverId: connection.id,
|
|
5279
|
+
error: params.error,
|
|
5280
|
+
attempt: params.attempt,
|
|
5281
|
+
willRetry: connection.willReconnect,
|
|
5282
|
+
});
|
|
5283
|
+
});
|
|
5284
|
+
connection.on('reconnect-failed', (params: { error: string; attempt: number }) => {
|
|
5285
|
+
this.emitTrace({
|
|
5286
|
+
type: 'mcpl:server-connect-failed',
|
|
5287
|
+
serverId: connection.id,
|
|
5288
|
+
error: params.error,
|
|
5289
|
+
attempt: params.attempt,
|
|
5290
|
+
willRetry: connection.willReconnect,
|
|
5291
|
+
});
|
|
5292
|
+
// The reconnect loop never gives up (backoff caps at ~300s), so
|
|
5293
|
+
// "the server is effectively down" is an attempt-count judgment:
|
|
5294
|
+
// 5 failed attempts ≈ a few minutes of outage. Throttled per
|
|
5295
|
+
// (serverId, kind), so a long outage re-posts every ~15 min.
|
|
5296
|
+
if (params.attempt >= 5) {
|
|
5297
|
+
this.opsAlert(
|
|
5298
|
+
'mcpl-down',
|
|
5299
|
+
connection.id,
|
|
5300
|
+
`MCPL server unreachable (attempt ${params.attempt}): ${params.error}`,
|
|
5301
|
+
);
|
|
5302
|
+
}
|
|
4106
5303
|
});
|
|
4107
5304
|
|
|
4108
|
-
//
|
|
4109
|
-
|
|
5305
|
+
// A late response to a tools/call that already timed out, carrying
|
|
5306
|
+
// stateful data the server advanced to. The host can't re-inject it (the
|
|
5307
|
+
// dispatch context is gone), but surfacing it makes the checkpoint-tree
|
|
5308
|
+
// divergence greppable instead of a silent drift into stale state.
|
|
5309
|
+
connection.on('orphaned-response', (info: {
|
|
5310
|
+
id: string | number;
|
|
5311
|
+
hadState: boolean;
|
|
5312
|
+
hadCheckpoint: boolean;
|
|
5313
|
+
}) => {
|
|
5314
|
+
this.emitTrace({
|
|
5315
|
+
type: 'mcpl:orphaned-response',
|
|
5316
|
+
serverId: connection.id,
|
|
5317
|
+
responseId: info.id,
|
|
5318
|
+
hadState: info.hadState,
|
|
5319
|
+
hadCheckpoint: info.hadCheckpoint,
|
|
5320
|
+
});
|
|
5321
|
+
});
|
|
5322
|
+
|
|
5323
|
+
// Connection-level errors (e.g. child process 'error' after spawn).
|
|
5324
|
+
// Attaching this listener also keeps an unhandled EventEmitter 'error'
|
|
5325
|
+
// from crashing the host process.
|
|
5326
|
+
connection.on('error', (err: Error) => {
|
|
5327
|
+
this.emitTrace({
|
|
5328
|
+
type: 'mcpl:server-error',
|
|
5329
|
+
serverId: connection.id,
|
|
5330
|
+
error: err.message,
|
|
5331
|
+
});
|
|
5332
|
+
});
|
|
5333
|
+
|
|
5334
|
+
// Cleanup on disconnect. Feature-set state is always dropped (the server
|
|
5335
|
+
// can't be validated while down; reconnect re-registers it). Checkpoint
|
|
5336
|
+
// trees, however, are durable state: CheckpointManager.removeServer deletes
|
|
5337
|
+
// them AND persists the deletion to Chronicle. The close handler must
|
|
5338
|
+
// therefore NEVER destroy checkpoints — `willReconnect` is false on a clean
|
|
5339
|
+
// stop() too (reconnectEnabled is cleared before 'close' fires), so gating
|
|
5340
|
+
// on it would erase every checkpoint tree on an ordinary host restart while
|
|
5341
|
+
// a SIGKILL preserves them. Permanent removal is owned solely by
|
|
5342
|
+
// disconnectMcplServer, which deletes the trees explicitly.
|
|
5343
|
+
connection.on('close', (code?: number | null, signal?: string | null) => {
|
|
4110
5344
|
this.featureSetManager?.removeServer(connection.id);
|
|
4111
|
-
this.
|
|
5345
|
+
this.emitTrace({
|
|
5346
|
+
type: 'mcpl:server-closed',
|
|
5347
|
+
serverId: connection.id,
|
|
5348
|
+
code: code ?? null,
|
|
5349
|
+
signal: signal ?? null,
|
|
5350
|
+
willReconnect: connection.willReconnect,
|
|
5351
|
+
});
|
|
4112
5352
|
this.emitTrace({ type: 'module:removed', moduleName: `mcpl:${connection.id}` });
|
|
4113
5353
|
});
|
|
4114
5354
|
}
|
|
@@ -4507,7 +5747,7 @@ export class AgentFramework {
|
|
|
4507
5747
|
return;
|
|
4508
5748
|
}
|
|
4509
5749
|
|
|
4510
|
-
const { until } = gate.setSleep(seconds, input.message);
|
|
5750
|
+
const { until } = gate.setSleep(seconds, input.message, agentName);
|
|
4511
5751
|
const announce = input.announce !== false; // default true
|
|
4512
5752
|
const human =
|
|
4513
5753
|
seconds >= 3600 ? `${(seconds / 3600).toFixed(1)}h`
|