@animalabs/agent-framework 0.5.7 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/agent.d.ts +4 -0
- package/dist/src/agent.d.ts.map +1 -1
- package/dist/src/agent.js +15 -0
- package/dist/src/agent.js.map +1 -1
- package/dist/src/framework.d.ts +243 -0
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +1378 -114
- package/dist/src/framework.js.map +1 -1
- package/dist/src/gate/event-gate.d.ts +64 -2
- package/dist/src/gate/event-gate.d.ts.map +1 -1
- package/dist/src/gate/event-gate.js +212 -12
- package/dist/src/gate/event-gate.js.map +1 -1
- package/dist/src/gate/gate-script-worker.d.ts +2 -0
- package/dist/src/gate/gate-script-worker.d.ts.map +1 -0
- package/dist/src/gate/gate-script-worker.js +69 -0
- package/dist/src/gate/gate-script-worker.js.map +1 -0
- package/dist/src/gate/gate-script.d.ts +44 -0
- package/dist/src/gate/gate-script.d.ts.map +1 -0
- package/dist/src/gate/gate-script.js +164 -0
- package/dist/src/gate/gate-script.js.map +1 -0
- package/dist/src/gate/types.d.ts +18 -3
- package/dist/src/gate/types.d.ts.map +1 -1
- package/dist/src/index.d.ts +3 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +4 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/mcpl/channel-registry.d.ts +60 -2
- package/dist/src/mcpl/channel-registry.d.ts.map +1 -1
- package/dist/src/mcpl/channel-registry.js +114 -9
- package/dist/src/mcpl/channel-registry.js.map +1 -1
- package/dist/src/mcpl/checkpoint-manager.d.ts +7 -0
- package/dist/src/mcpl/checkpoint-manager.d.ts.map +1 -1
- package/dist/src/mcpl/checkpoint-manager.js +15 -0
- package/dist/src/mcpl/checkpoint-manager.js.map +1 -1
- package/dist/src/mcpl/push-handler.d.ts.map +1 -1
- package/dist/src/mcpl/push-handler.js +5 -0
- package/dist/src/mcpl/push-handler.js.map +1 -1
- package/dist/src/mcpl/server-connection.d.ts +64 -30
- package/dist/src/mcpl/server-connection.d.ts.map +1 -1
- package/dist/src/mcpl/server-connection.js +247 -210
- package/dist/src/mcpl/server-connection.js.map +1 -1
- package/dist/src/mcpl/transport.d.ts +98 -0
- package/dist/src/mcpl/transport.d.ts.map +1 -0
- package/dist/src/mcpl/transport.js +281 -0
- package/dist/src/mcpl/transport.js.map +1 -0
- package/dist/src/mcpl/types.d.ts +59 -3
- package/dist/src/mcpl/types.d.ts.map +1 -1
- package/dist/src/mcpl/types.js.map +1 -1
- package/dist/src/module-registry.d.ts +17 -1
- package/dist/src/module-registry.d.ts.map +1 -1
- package/dist/src/module-registry.js +101 -6
- package/dist/src/module-registry.js.map +1 -1
- package/dist/src/prose-segments.d.ts +18 -0
- package/dist/src/prose-segments.d.ts.map +1 -0
- package/dist/src/prose-segments.js +39 -0
- package/dist/src/prose-segments.js.map +1 -0
- package/dist/src/runtime/liveness-watchdog-worker.d.ts +2 -0
- package/dist/src/runtime/liveness-watchdog-worker.d.ts.map +1 -0
- package/dist/src/runtime/liveness-watchdog-worker.js +57 -0
- package/dist/src/runtime/liveness-watchdog-worker.js.map +1 -0
- package/dist/src/runtime/liveness-watchdog.d.ts +36 -0
- package/dist/src/runtime/liveness-watchdog.d.ts.map +1 -0
- package/dist/src/runtime/liveness-watchdog.js +106 -0
- package/dist/src/runtime/liveness-watchdog.js.map +1 -0
- package/dist/src/types/agent.d.ts +47 -0
- package/dist/src/types/agent.d.ts.map +1 -1
- package/dist/src/types/framework.d.ts +14 -0
- package/dist/src/types/framework.d.ts.map +1 -1
- package/dist/src/types/module.d.ts +58 -0
- package/dist/src/types/module.d.ts.map +1 -1
- package/dist/src/types/trace.d.ts +42 -0
- package/dist/src/types/trace.d.ts.map +1 -1
- package/dist/test/agent-cache-ttl.test.d.ts +2 -0
- package/dist/test/agent-cache-ttl.test.d.ts.map +1 -0
- package/dist/test/agent-cache-ttl.test.js +25 -0
- package/dist/test/agent-cache-ttl.test.js.map +1 -0
- package/dist/test/channel-registry-routing.test.js +108 -1
- package/dist/test/channel-registry-routing.test.js.map +1 -1
- package/dist/test/checkpoint-attribution.test.d.ts +2 -0
- package/dist/test/checkpoint-attribution.test.d.ts.map +1 -0
- package/dist/test/checkpoint-attribution.test.js +40 -0
- package/dist/test/checkpoint-attribution.test.js.map +1 -0
- package/dist/test/conversation-fork-injection-scoping.test.d.ts +2 -0
- package/dist/test/conversation-fork-injection-scoping.test.d.ts.map +1 -0
- package/dist/test/conversation-fork-injection-scoping.test.js +184 -0
- package/dist/test/conversation-fork-injection-scoping.test.js.map +1 -0
- package/dist/test/event-gate-policy-mutation.test.d.ts +2 -0
- package/dist/test/event-gate-policy-mutation.test.d.ts.map +1 -0
- package/dist/test/event-gate-policy-mutation.test.js +137 -0
- package/dist/test/event-gate-policy-mutation.test.js.map +1 -0
- package/dist/test/event-gate-script.test.d.ts +2 -0
- package/dist/test/event-gate-script.test.d.ts.map +1 -0
- package/dist/test/event-gate-script.test.js +95 -0
- package/dist/test/event-gate-script.test.js.map +1 -0
- package/dist/test/event-gate-tags.test.d.ts +2 -0
- package/dist/test/event-gate-tags.test.d.ts.map +1 -0
- package/dist/test/event-gate-tags.test.js +76 -0
- package/dist/test/event-gate-tags.test.js.map +1 -0
- package/dist/test/gather-context-timeout.test.d.ts +2 -0
- package/dist/test/gather-context-timeout.test.d.ts.map +1 -0
- package/dist/test/gather-context-timeout.test.js +120 -0
- package/dist/test/gather-context-timeout.test.js.map +1 -0
- package/dist/test/inference-exhaustion-rewind.test.d.ts +2 -0
- package/dist/test/inference-exhaustion-rewind.test.d.ts.map +1 -0
- package/dist/test/inference-exhaustion-rewind.test.js +181 -0
- package/dist/test/inference-exhaustion-rewind.test.js.map +1 -0
- package/dist/test/inference-failure-observability.test.js +4 -0
- package/dist/test/inference-failure-observability.test.js.map +1 -1
- package/dist/test/liveness-watchdog.test.d.ts +2 -0
- package/dist/test/liveness-watchdog.test.d.ts.map +1 -0
- package/dist/test/liveness-watchdog.test.js +78 -0
- package/dist/test/liveness-watchdog.test.js.map +1 -0
- package/dist/test/mcpl-reconnect-events.test.d.ts +2 -0
- package/dist/test/mcpl-reconnect-events.test.d.ts.map +1 -0
- package/dist/test/mcpl-reconnect-events.test.js +119 -0
- package/dist/test/mcpl-reconnect-events.test.js.map +1 -0
- package/dist/test/mcpl-reconnect-reregistration.test.d.ts +2 -0
- package/dist/test/mcpl-reconnect-reregistration.test.d.ts.map +1 -0
- package/dist/test/mcpl-reconnect-reregistration.test.js +181 -0
- package/dist/test/mcpl-reconnect-reregistration.test.js.map +1 -0
- package/dist/test/mcpl-request-timeout.test.d.ts +2 -0
- package/dist/test/mcpl-request-timeout.test.d.ts.map +1 -0
- package/dist/test/mcpl-request-timeout.test.js +123 -0
- package/dist/test/mcpl-request-timeout.test.js.map +1 -0
- package/dist/test/mcpl-runtime-lifecycle.test.d.ts +7 -0
- package/dist/test/mcpl-runtime-lifecycle.test.d.ts.map +1 -0
- package/dist/test/mcpl-runtime-lifecycle.test.js +142 -0
- package/dist/test/mcpl-runtime-lifecycle.test.js.map +1 -0
- package/dist/test/mcpl-ws-transport.test.d.ts +2 -0
- package/dist/test/mcpl-ws-transport.test.d.ts.map +1 -0
- package/dist/test/mcpl-ws-transport.test.js +155 -0
- package/dist/test/mcpl-ws-transport.test.js.map +1 -0
- package/dist/test/prose-segments.test.d.ts +2 -0
- package/dist/test/prose-segments.test.d.ts.map +1 -0
- package/dist/test/prose-segments.test.js +52 -0
- package/dist/test/prose-segments.test.js.map +1 -0
- package/dist/test/refusal-rewind.test.d.ts +2 -0
- package/dist/test/refusal-rewind.test.d.ts.map +1 -0
- package/dist/test/refusal-rewind.test.js +108 -0
- package/dist/test/refusal-rewind.test.js.map +1 -0
- package/dist/test/state-scaling-e2e.test.d.ts +22 -0
- package/dist/test/state-scaling-e2e.test.d.ts.map +1 -0
- package/dist/test/state-scaling-e2e.test.js +200 -0
- package/dist/test/state-scaling-e2e.test.js.map +1 -0
- package/dist/test/state-scaling.test.d.ts +12 -0
- package/dist/test/state-scaling.test.d.ts.map +1 -0
- package/dist/test/state-scaling.test.js +243 -0
- package/dist/test/state-scaling.test.js.map +1 -0
- package/dist/test/tool-result-content.test.d.ts +2 -0
- package/dist/test/tool-result-content.test.d.ts.map +1 -0
- package/dist/test/tool-result-content.test.js +37 -0
- package/dist/test/tool-result-content.test.js.map +1 -0
- package/dist/test/trunk-channel-routing.test.d.ts +19 -0
- package/dist/test/trunk-channel-routing.test.d.ts.map +1 -0
- package/dist/test/trunk-channel-routing.test.js +175 -0
- package/dist/test/trunk-channel-routing.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +4 -3
- package/src/agent.ts +16 -0
- package/src/framework.ts +1576 -121
- package/src/gate/event-gate.ts +209 -12
- package/src/gate/gate-script-worker.ts +73 -0
- package/src/gate/gate-script.ts +180 -0
- package/src/gate/types.ts +19 -3
- package/src/index.ts +10 -0
- package/src/mcpl/channel-registry.ts +143 -8
- package/src/mcpl/checkpoint-manager.ts +16 -0
- package/src/mcpl/push-handler.ts +5 -0
- package/src/mcpl/server-connection.ts +262 -227
- package/src/mcpl/transport.ts +307 -0
- package/src/mcpl/types.ts +68 -3
- package/src/module-registry.ts +126 -6
- package/src/prose-segments.ts +42 -0
- package/src/runtime/liveness-watchdog-worker.ts +66 -0
- package/src/runtime/liveness-watchdog.ts +122 -0
- package/src/types/agent.ts +48 -0
- package/src/types/framework.ts +15 -0
- package/src/types/module.ts +65 -0
- package/src/types/trace.ts +51 -0
package/dist/src/framework.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { join } from 'node:path';
|
|
2
|
+
import { appendFileSync, mkdirSync } from 'node:fs';
|
|
2
3
|
import { JsStore } from '@animalabs/chronicle';
|
|
3
4
|
import { MembraneError } from '@animalabs/membrane';
|
|
4
5
|
import { ContextManager, PassthroughStrategy } from '@animalabs/context-manager';
|
|
5
6
|
import { ProcessQueueImpl } from './queue.js';
|
|
6
7
|
import { Agent } from './agent.js';
|
|
7
|
-
import { ModuleRegistry } from './module-registry.js';
|
|
8
|
+
import { ModuleRegistry, isStateExistsError } from './module-registry.js';
|
|
8
9
|
import { McplServerRegistry } from './mcpl/server-registry.js';
|
|
9
10
|
import { FeatureSetManager } from './mcpl/feature-set-manager.js';
|
|
10
11
|
import { ScopeManager } from './mcpl/scope-manager.js';
|
|
@@ -15,6 +16,7 @@ import { ChannelRegistry } from './mcpl/channel-registry.js';
|
|
|
15
16
|
import { ConversationRouter } from './mcpl/conversation-router.js';
|
|
16
17
|
import { safeSlice } from './safe-slice.js';
|
|
17
18
|
import { toolResultDataToHistoryString } from './tool-result-history.js';
|
|
19
|
+
import { splitProseSegments } from './prose-segments.js';
|
|
18
20
|
import { CheckpointManager } from './mcpl/checkpoint-manager.js';
|
|
19
21
|
import { isToolAllowed } from './mcpl/tool-policy.js';
|
|
20
22
|
import { EventGate } from './gate/event-gate.js';
|
|
@@ -23,7 +25,8 @@ const FRAMEWORK_STATE_ID = 'framework/state';
|
|
|
23
25
|
const CONVERSATION_ROUTER_STATE_ID = 'framework/conversation-router';
|
|
24
26
|
const INFERENCE_LOG_ID = 'framework/inference-log';
|
|
25
27
|
const PROCESS_LOG_ID = 'framework/process-log';
|
|
26
|
-
const TURN_CHECKPOINTS_ID = 'framework/turn-checkpoints';
|
|
28
|
+
const TURN_CHECKPOINTS_ID = 'framework/turn-checkpoints'; // legacy single-map layout, read-only fallback
|
|
29
|
+
const TURN_CHECKPOINTS_TREE_ID = 'framework/turn-checkpoints/tree';
|
|
27
30
|
/** Maximum number of turn checkpoints to keep per agent. */
|
|
28
31
|
const MAX_TURN_CHECKPOINTS = 20;
|
|
29
32
|
/**
|
|
@@ -94,6 +97,12 @@ function extractGateFields(event) {
|
|
|
94
97
|
: undefined;
|
|
95
98
|
return { content, metadata, mount, paths };
|
|
96
99
|
}
|
|
100
|
+
/** Cap an API error message for inline use in a rewind marker: keep enough to
|
|
101
|
+
* identify the failure class without pasting a wall of provider JSON into the
|
|
102
|
+
* agent's context. */
|
|
103
|
+
function truncateReason(reason, max = 160) {
|
|
104
|
+
return reason.length <= max ? reason : reason.slice(0, max) + '…';
|
|
105
|
+
}
|
|
97
106
|
export class AgentFramework {
|
|
98
107
|
store;
|
|
99
108
|
ownsStore;
|
|
@@ -104,11 +113,25 @@ export class AgentFramework {
|
|
|
104
113
|
inferencePolicy;
|
|
105
114
|
errorPolicy;
|
|
106
115
|
pendingRequests = [];
|
|
116
|
+
/**
|
|
117
|
+
* Per-agent channel that triggered the agent's CURRENT inference turn, if any
|
|
118
|
+
* (item-3 redux). Read by the ChannelRegistry's `activeChannelResolver` to
|
|
119
|
+
* route a single-trunk agent's plain-text speech back to the channel it is
|
|
120
|
+
* answering, instead of the process-global most-recent-inbound locus that a
|
|
121
|
+
* concurrent message elsewhere can hijack. Set at turn start (or cleared for a
|
|
122
|
+
* heartbeat / no-trigger turn) in startAgentStream; overwritten by the next
|
|
123
|
+
* turn. Never read between turns (a given agent runs one turn at a time).
|
|
124
|
+
*/
|
|
125
|
+
activeTriggerChannels = new Map();
|
|
107
126
|
running = false;
|
|
108
127
|
loopPromise = null;
|
|
109
128
|
traceListeners = [];
|
|
110
129
|
syncIntervalMs;
|
|
111
130
|
syncTimer = null;
|
|
131
|
+
/** Last time we console-warned about stale (busy-requeued) inference requests, per agent. */
|
|
132
|
+
staleWarnAt = new Map();
|
|
133
|
+
/** Per-agent last inference activity (epoch ms), for /healthz + doctor tooling. */
|
|
134
|
+
lastInferenceAt = new Map();
|
|
112
135
|
processLoggingPersist;
|
|
113
136
|
processLoggingBroadcast;
|
|
114
137
|
activeStreams = new Map();
|
|
@@ -119,6 +142,23 @@ export class AgentFramework {
|
|
|
119
142
|
/** N consecutive failed inferences ⇒ the agent is treated as hard-down and
|
|
120
143
|
* escalated loudly to stderr. */
|
|
121
144
|
inferenceFailureEscalationThreshold = 3;
|
|
145
|
+
/** Per-agent count of consecutive refusal-driven rewinds in the current turn
|
|
146
|
+
* chain (reset when a turn completes without a refusal). Bounds the auto
|
|
147
|
+
* rewind loop — see refusalHandling + rewindTriggeringTurn. */
|
|
148
|
+
refusalRewinds = new Map();
|
|
149
|
+
/** Per-agent count of poison-history rewinds performed by the hard-down
|
|
150
|
+
* breaker (noteInferenceExhausted). Reset on any successful inference.
|
|
151
|
+
* Bounds the automatic quarantine loop the same way refusalRewinds bounds
|
|
152
|
+
* the refusal loop, so the breaker can never shed the whole history. */
|
|
153
|
+
exhaustionRewinds = new Map();
|
|
154
|
+
/** Per-agent current rewind episode: the single consolidated marker's id and
|
|
155
|
+
* how many turns have been shed so far. One marker per episode, updated in
|
|
156
|
+
* place (see updateRewindMarker); cleared when the episode ends. */
|
|
157
|
+
rewindEpisode = new Map();
|
|
158
|
+
/** Active `/unstick` sessions: an admin-forced rewind-until-clean loop that
|
|
159
|
+
* runs even when the agent's autoRewind toggle is off. Tracks the remaining
|
|
160
|
+
* budget, what was shed (for the report), and where to post the outcome. */
|
|
161
|
+
forcedRewind = new Map();
|
|
122
162
|
/** Name of the primary (non-ephemeral) agent for routing framework messages. */
|
|
123
163
|
primaryAgentName = null;
|
|
124
164
|
// Messages deferred while an agent is waiting_for_tools (to preserve
|
|
@@ -127,6 +167,8 @@ export class AgentFramework {
|
|
|
127
167
|
// Undo/redo state
|
|
128
168
|
turnCounters = new Map(); // agentName → next turnIndex
|
|
129
169
|
redoStacks = new Map(); // agentName → redo entries
|
|
170
|
+
/** Liveness watchdog (fail hard on a wedged main thread). Null unless enabled. */
|
|
171
|
+
livenessWatchdog = null;
|
|
130
172
|
// MCPL subsystems (null when no mcplServers configured)
|
|
131
173
|
mcplServerRegistry = null;
|
|
132
174
|
featureSetManager = null;
|
|
@@ -158,6 +200,13 @@ export class AgentFramework {
|
|
|
158
200
|
mcplPrefixMap = new Map();
|
|
159
201
|
/** Maps serverId → McplServerConfig for prefix lookup. */
|
|
160
202
|
mcplServerConfigs = new Map();
|
|
203
|
+
/** Host capabilities advertised during the MCP handshake — stored so servers
|
|
204
|
+
* can be connected at runtime (connectMcplServer) after initialization. */
|
|
205
|
+
mcplHostCapabilities = null;
|
|
206
|
+
/** Inference routing policy from FrameworkConfig — stored so the MCPL
|
|
207
|
+
* subsystem can be lazily initialized by connectMcplServer when the
|
|
208
|
+
* framework started with zero configured servers. */
|
|
209
|
+
mcplInferenceRoutingConfig = null;
|
|
161
210
|
// EventGate (null when FrameworkConfig.gate is omitted)
|
|
162
211
|
eventGate = null;
|
|
163
212
|
// Session-level token usage tracking (always-on)
|
|
@@ -185,6 +234,7 @@ export class AgentFramework {
|
|
|
185
234
|
queryMessages: (filter) => this.queryMessages(filter),
|
|
186
235
|
pushEvent: (event) => this.pushEvent(event),
|
|
187
236
|
onTrace: (listener) => this.onTrace(listener),
|
|
237
|
+
callTool: (call) => this.executeToolCall(call),
|
|
188
238
|
});
|
|
189
239
|
}
|
|
190
240
|
/**
|
|
@@ -223,11 +273,14 @@ export class AgentFramework {
|
|
|
223
273
|
catch {
|
|
224
274
|
// Already registered
|
|
225
275
|
}
|
|
276
|
+
// The legacy single-map checkpoint state (TURN_CHECKPOINTS_ID) is no longer
|
|
277
|
+
// registered for new stores — it's read-only fallback data in old ones.
|
|
226
278
|
try {
|
|
227
|
-
store.registerState({ id:
|
|
279
|
+
store.registerState({ id: TURN_CHECKPOINTS_TREE_ID, strategy: 'tree' });
|
|
228
280
|
}
|
|
229
|
-
catch {
|
|
230
|
-
|
|
281
|
+
catch (error) {
|
|
282
|
+
if (!isStateExistsError(error))
|
|
283
|
+
throw error;
|
|
231
284
|
}
|
|
232
285
|
// Process logging config (default: disabled)
|
|
233
286
|
const processLoggingPersist = config.processLogging?.persist ?? false;
|
|
@@ -301,6 +354,21 @@ export class AgentFramework {
|
|
|
301
354
|
getAgentNames: () => [...framework.agents.keys()],
|
|
302
355
|
});
|
|
303
356
|
}
|
|
357
|
+
// Liveness watchdog: fail hard if the main thread wedges (opt-in).
|
|
358
|
+
if (config.watchdog?.enabled) {
|
|
359
|
+
const { LivenessWatchdog } = await import('./runtime/liveness-watchdog.js');
|
|
360
|
+
framework.livenessWatchdog = new LivenessWatchdog({
|
|
361
|
+
enabled: true,
|
|
362
|
+
thresholdMs: config.watchdog.thresholdMs,
|
|
363
|
+
action: config.watchdog.action,
|
|
364
|
+
reportPath: config.watchdog.reportPath
|
|
365
|
+
?? (config.storePath ? join(config.storePath, 'watchdog-wedge.jsonl') : undefined),
|
|
366
|
+
});
|
|
367
|
+
framework.livenessWatchdog.start();
|
|
368
|
+
}
|
|
369
|
+
// Stored for lazy MCPL initialization (connectMcplServer on a framework
|
|
370
|
+
// that started with zero configured servers).
|
|
371
|
+
framework.mcplInferenceRoutingConfig = config.inferenceRouting ?? null;
|
|
304
372
|
// Initialize MCPL subsystems if configured
|
|
305
373
|
if (config.mcplServers && config.mcplServers.length > 0) {
|
|
306
374
|
// Validate tool prefixes: no collisions with module names or between servers
|
|
@@ -319,6 +387,30 @@ export class AgentFramework {
|
|
|
319
387
|
}
|
|
320
388
|
await framework.initializeMcpl(config.mcplServers, config.inferenceRouting);
|
|
321
389
|
}
|
|
390
|
+
// Diagnostics: `kill -USR2 <pid>` dumps live wake/inference state to stderr
|
|
391
|
+
// (journal) without a restart — for catching the wake-wedge on the running
|
|
392
|
+
// process. Shows the gate's `inferring` set + buffered-event count (the
|
|
393
|
+
// wedge signature), active streams, and pending inference requests.
|
|
394
|
+
try {
|
|
395
|
+
process.on('SIGUSR2', () => {
|
|
396
|
+
try {
|
|
397
|
+
const gate = framework.eventGate?.inferenceDiagnostics() ?? null;
|
|
398
|
+
console.error('[diagnostics] ' + JSON.stringify({
|
|
399
|
+
at: new Date().toISOString(),
|
|
400
|
+
gate,
|
|
401
|
+
activeStreams: [...framework.activeStreams.keys()],
|
|
402
|
+
pendingRequests: framework.pendingRequests.length,
|
|
403
|
+
agents: [...framework.agents.keys()],
|
|
404
|
+
}));
|
|
405
|
+
}
|
|
406
|
+
catch (err) {
|
|
407
|
+
console.error('[diagnostics] dump failed:', err instanceof Error ? err.message : err);
|
|
408
|
+
}
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
catch {
|
|
412
|
+
// SIGUSR2 not available on this platform — non-fatal.
|
|
413
|
+
}
|
|
322
414
|
return framework;
|
|
323
415
|
}
|
|
324
416
|
/**
|
|
@@ -371,6 +463,7 @@ export class AgentFramework {
|
|
|
371
463
|
this.channelRegistry?.stopAll();
|
|
372
464
|
// Dispose EventGate (clear debounce timers)
|
|
373
465
|
this.eventGate?.dispose();
|
|
466
|
+
this.livenessWatchdog?.stop();
|
|
374
467
|
// Stop modules and MCPL servers in parallel
|
|
375
468
|
const shutdownPromises = [this.moduleRegistry.stopAll()];
|
|
376
469
|
if (this.mcplServerRegistry) {
|
|
@@ -487,7 +580,12 @@ export class AgentFramework {
|
|
|
487
580
|
const moduleTools = this.moduleRegistry.getAllTools();
|
|
488
581
|
const channelTools = this.channelRegistry?.getChannelTools() ?? [];
|
|
489
582
|
const gateTools = this.eventGate
|
|
490
|
-
? [
|
|
583
|
+
? [
|
|
584
|
+
this.eventGate.getToolDefinition(),
|
|
585
|
+
...AgentFramework.SLEEP_TOOLS,
|
|
586
|
+
...AgentFramework.WAKE_RULE_TOOLS,
|
|
587
|
+
AgentFramework.EVENT_TAGS_TOOL,
|
|
588
|
+
]
|
|
491
589
|
: [];
|
|
492
590
|
if (this.mcplTools.length === 0 && channelTools.length === 0 && gateTools.length === 0) {
|
|
493
591
|
return moduleTools;
|
|
@@ -536,8 +634,9 @@ export class AgentFramework {
|
|
|
536
634
|
// Module gatherContext (fail-open, matches startAgentStream)
|
|
537
635
|
try {
|
|
538
636
|
const moduleInjections = await this.moduleRegistry.gatherContext(agentName);
|
|
539
|
-
|
|
540
|
-
|
|
637
|
+
const scopedModuleInjections = this.scopeInjectionsForAgent(agentName, moduleInjections);
|
|
638
|
+
if (scopedModuleInjections.length > 0) {
|
|
639
|
+
injections = scopedModuleInjections;
|
|
541
640
|
}
|
|
542
641
|
}
|
|
543
642
|
catch (error) {
|
|
@@ -548,7 +647,7 @@ export class AgentFramework {
|
|
|
548
647
|
if (this.hookOrchestrator) {
|
|
549
648
|
try {
|
|
550
649
|
const hookParams = this.buildBeforeInferenceParams(agent);
|
|
551
|
-
const hookInjections = await this.hookOrchestrator.beforeInference(hookParams);
|
|
650
|
+
const hookInjections = this.scopeInjectionsForAgent(agentName, await this.hookOrchestrator.beforeInference(hookParams));
|
|
552
651
|
if (hookInjections.length > 0) {
|
|
553
652
|
injections = injections ? [...injections, ...hookInjections] : hookInjections;
|
|
554
653
|
}
|
|
@@ -580,6 +679,38 @@ export class AgentFramework {
|
|
|
580
679
|
getModule(name) {
|
|
581
680
|
return this.moduleRegistry.getModule(name);
|
|
582
681
|
}
|
|
682
|
+
// =========================================================================
|
|
683
|
+
// Wake-rule surface (gate policy mutation) — backs wake_add_rule /
|
|
684
|
+
// wake_remove_rule and lets host modules compose higher-level wake modes
|
|
685
|
+
// (e.g. an "every-message-debounced" channel mode) without reaching into
|
|
686
|
+
// the private EventGate.
|
|
687
|
+
// =========================================================================
|
|
688
|
+
/**
|
|
689
|
+
* Add or replace (upsert) a gate policy at runtime. Validated and persisted
|
|
690
|
+
* to gate.json; hot-applied in memory. Throws if no gate is configured or the
|
|
691
|
+
* policy is invalid. `position: 'prepend'` puts a wake rule ahead of broad
|
|
692
|
+
* defer/debounce rules (first match wins).
|
|
693
|
+
*/
|
|
694
|
+
addGatePolicy(rawPolicy, options) {
|
|
695
|
+
if (!this.eventGate) {
|
|
696
|
+
throw new Error('No EventGate configured (FrameworkConfig.gate is unset).');
|
|
697
|
+
}
|
|
698
|
+
return this.eventGate.addPolicy(rawPolicy, options);
|
|
699
|
+
}
|
|
700
|
+
/**
|
|
701
|
+
* Remove a gate policy by name at runtime. Returns false if it didn't exist.
|
|
702
|
+
* Throws if no gate is configured.
|
|
703
|
+
*/
|
|
704
|
+
removeGatePolicy(name) {
|
|
705
|
+
if (!this.eventGate) {
|
|
706
|
+
throw new Error('No EventGate configured (FrameworkConfig.gate is unset).');
|
|
707
|
+
}
|
|
708
|
+
return this.eventGate.removePolicy(name);
|
|
709
|
+
}
|
|
710
|
+
/** Current gate policy names (freshest on-disk view). Empty when no gate. */
|
|
711
|
+
getGatePolicyNames() {
|
|
712
|
+
return this.eventGate?.listPolicyNames() ?? [];
|
|
713
|
+
}
|
|
583
714
|
/**
|
|
584
715
|
* Get the Membrane instance.
|
|
585
716
|
*/
|
|
@@ -679,6 +810,7 @@ export class AgentFramework {
|
|
|
679
810
|
clearInterval(completionWatchdog);
|
|
680
811
|
this.offTrace(traceListener);
|
|
681
812
|
this.agents.delete(agent.name);
|
|
813
|
+
this.evictTurnCheckpoints(agent.name);
|
|
682
814
|
};
|
|
683
815
|
// Watchdog: if inference never starts within 30s, the agent is a zombie.
|
|
684
816
|
// This catches cases where the event loop stalled, the agent was
|
|
@@ -828,6 +960,147 @@ export class AgentFramework {
|
|
|
828
960
|
inputSchema: { type: 'object' },
|
|
829
961
|
},
|
|
830
962
|
];
|
|
963
|
+
/** Synthesized wake-rule tools: add/remove a gate.json policy at runtime.
|
|
964
|
+
* Present when a gate is wired. They write validated policies into the
|
|
965
|
+
* hot-reloaded gate.json (same validation as load), so a rule takes effect
|
|
966
|
+
* immediately and survives restart. */
|
|
967
|
+
static WAKE_RULE_TOOLS = [
|
|
968
|
+
{
|
|
969
|
+
name: 'wake_add_rule',
|
|
970
|
+
description: 'Add or replace a wake rule (a gate.json policy) at runtime — no need to ' +
|
|
971
|
+
'hand-edit the file. The rule is validated and hot-applied immediately. ' +
|
|
972
|
+
'A rule with the same `name` replaces the existing one in place. Use ' +
|
|
973
|
+
'`position: "prepend"` to put a wake rule ahead of broad defer/debounce ' +
|
|
974
|
+
'rules (first match wins). Two common shapes:\n' +
|
|
975
|
+
'• Watch a FILE/workspace path: match on `mount` + `pathGlob`, e.g. ' +
|
|
976
|
+
'`{ name: "watch-notes", match: { scope: ["workspace:modified"], mount: "project", pathGlob: "notes/*.md" }, behavior: "always" }`.\n' +
|
|
977
|
+
'• Watch a CHANNEL: match on `source` + `channel` (and/or `tagsAny: ["chat:ambient"]`), ' +
|
|
978
|
+
'e.g. `{ name: "watch-cairn", match: { source: "discord", channel: "discord:*:12345", tagsAny: ["chat:ambient"] }, behavior: { debounce: 60000 } }` ' +
|
|
979
|
+
'(the channel must be subscribed for ambient events to arrive — see channel mode).',
|
|
980
|
+
inputSchema: {
|
|
981
|
+
type: 'object',
|
|
982
|
+
properties: {
|
|
983
|
+
name: { type: 'string', description: 'Unique rule name. Reusing a name replaces that rule.' },
|
|
984
|
+
match: {
|
|
985
|
+
type: 'object',
|
|
986
|
+
description: 'Match criteria (all AND together). Omitted fields match anything.',
|
|
987
|
+
properties: {
|
|
988
|
+
scope: { type: 'array', items: { type: 'string' }, description: 'Event types, e.g. ["mcpl:channel-incoming","workspace:modified"].' },
|
|
989
|
+
source: { type: 'string', description: 'Integration/serverId, glob ok (e.g. "discord").' },
|
|
990
|
+
channel: { type: 'string', description: 'Channel id, glob ok (e.g. "discord:*:12345").' },
|
|
991
|
+
mount: { type: 'string', description: 'Workspace mount name, glob ok (workspace:* events).' },
|
|
992
|
+
pathGlob: { type: 'string', description: 'Glob over touched paths (workspace:* events).' },
|
|
993
|
+
tagsAny: { type: 'array', items: { type: 'string' }, description: 'Match if ANY tag matches (globs ok).' },
|
|
994
|
+
tagsAll: { type: 'array', items: { type: 'string' }, description: 'Match only if EVERY tag matches.' },
|
|
995
|
+
tagsNone: { type: 'array', items: { type: 'string' }, description: 'Match only if NONE match.' },
|
|
996
|
+
metadataTrue: { type: 'array', items: { type: 'string' }, description: 'Match if ANY listed metadata field is truthy.' },
|
|
997
|
+
filter: {
|
|
998
|
+
type: 'object',
|
|
999
|
+
description: 'Content filter.',
|
|
1000
|
+
properties: {
|
|
1001
|
+
type: { type: 'string', enum: ['text', 'regex'] },
|
|
1002
|
+
pattern: { type: 'string' },
|
|
1003
|
+
},
|
|
1004
|
+
required: ['type', 'pattern'],
|
|
1005
|
+
},
|
|
1006
|
+
},
|
|
1007
|
+
},
|
|
1008
|
+
behavior: {
|
|
1009
|
+
type: 'string',
|
|
1010
|
+
enum: ['always', 'defer', 'skip'],
|
|
1011
|
+
description: 'Simple behavior: "always" (wake now) or "defer" (don\'t wake; still enters ' +
|
|
1012
|
+
'context). For debounce / rate-limit / sampling, use debounceMs / rateLimit / ' +
|
|
1013
|
+
'passiveSample below instead of this field. Exactly one behavior must be given.',
|
|
1014
|
+
},
|
|
1015
|
+
debounceMs: {
|
|
1016
|
+
type: 'number',
|
|
1017
|
+
description: 'Shorthand for { debounce: ms }: wake once after ms of quiet (100–300000).',
|
|
1018
|
+
},
|
|
1019
|
+
rateLimit: {
|
|
1020
|
+
type: 'object',
|
|
1021
|
+
description: 'Token-bucket wake: at most `tokens` wakes per window; refills one per refillIntervalMs.',
|
|
1022
|
+
properties: {
|
|
1023
|
+
tokens: { type: 'number', description: 'Bucket capacity (> 0).' },
|
|
1024
|
+
refillIntervalMs: { type: 'number', description: 'Ms between token refills (> 0).' },
|
|
1025
|
+
keyBy: { type: 'string', description: 'Metadata field to partition buckets by (e.g. "channelId").' },
|
|
1026
|
+
},
|
|
1027
|
+
required: ['tokens', 'refillIntervalMs'],
|
|
1028
|
+
},
|
|
1029
|
+
passiveSample: {
|
|
1030
|
+
type: 'object',
|
|
1031
|
+
description: 'Wake every Nth matching event.',
|
|
1032
|
+
properties: {
|
|
1033
|
+
every: { type: 'number', description: 'Fire every N matches (positive integer).' },
|
|
1034
|
+
keyBy: { type: 'string', description: 'Metadata field for separate per-key counters.' },
|
|
1035
|
+
},
|
|
1036
|
+
required: ['every'],
|
|
1037
|
+
},
|
|
1038
|
+
position: {
|
|
1039
|
+
type: 'string',
|
|
1040
|
+
enum: ['append', 'prepend'],
|
|
1041
|
+
description: 'Where to insert a NEW rule (ignored when replacing by name). Default "append".',
|
|
1042
|
+
},
|
|
1043
|
+
resets: {
|
|
1044
|
+
type: 'array',
|
|
1045
|
+
items: { type: 'string' },
|
|
1046
|
+
description: 'Names of other rules whose runtime state to clear when this rule fires.',
|
|
1047
|
+
},
|
|
1048
|
+
},
|
|
1049
|
+
required: ['name'],
|
|
1050
|
+
},
|
|
1051
|
+
},
|
|
1052
|
+
{
|
|
1053
|
+
name: 'wake_remove_rule',
|
|
1054
|
+
description: 'Remove a wake rule (gate.json policy) by name at runtime. Any pending ' +
|
|
1055
|
+
'debounce batch for that rule is delivered first. Returns whether a rule ' +
|
|
1056
|
+
'was removed (false if no rule had that name).',
|
|
1057
|
+
inputSchema: {
|
|
1058
|
+
type: 'object',
|
|
1059
|
+
properties: {
|
|
1060
|
+
name: { type: 'string', description: 'The rule name to remove.' },
|
|
1061
|
+
},
|
|
1062
|
+
required: ['name'],
|
|
1063
|
+
},
|
|
1064
|
+
},
|
|
1065
|
+
];
|
|
1066
|
+
/** Discoverability for event tags (MCPL RFC-001): the reserved chat:* core,
|
|
1067
|
+
* each connected server's declared tag ontology, and gate.js status. */
|
|
1068
|
+
static EVENT_TAGS_TOOL = {
|
|
1069
|
+
name: 'event_tags',
|
|
1070
|
+
description: 'List the event tags available for gating: the reserved cross-platform ' +
|
|
1071
|
+
'chat:* core, each connected server\'s declared tag ontology (descriptions, ' +
|
|
1072
|
+
'implications, suggested treatments), and the status of your programmable ' +
|
|
1073
|
+
'gate (gate.js). Use these tag names in gate.json policies (tagsAny / ' +
|
|
1074
|
+
'tagsAll / tagsNone) or in gate.js.',
|
|
1075
|
+
inputSchema: { type: 'object' },
|
|
1076
|
+
};
|
|
1077
|
+
/** Reserved chat:* core vocabulary (MCPL RFC-001 §4) — short descriptions so
|
|
1078
|
+
* the agent can author rules without reading the spec. */
|
|
1079
|
+
static CHAT_CORE_TAGS = {
|
|
1080
|
+
'chat:addressed': 'Directed at you (umbrella: dm/mention/reply)',
|
|
1081
|
+
'chat:mention': 'You were explicitly @-mentioned',
|
|
1082
|
+
'chat:reply': 'A reply to your own message',
|
|
1083
|
+
'chat:dm': 'A direct/private message to you',
|
|
1084
|
+
'chat:ambient': 'Overheard in a followed channel; not addressed',
|
|
1085
|
+
'chat:broadcast': 'Channel-wide ping (@everyone / channel post)',
|
|
1086
|
+
'chat:to-self': 'Acts on your own content (reaction/reply to you)',
|
|
1087
|
+
'chat:from-human': 'Authored by a human',
|
|
1088
|
+
'chat:from-bot': 'Authored by a bot/automation',
|
|
1089
|
+
'chat:from-self': 'Your own message, echoed back',
|
|
1090
|
+
'chat:from-agent': 'Authored by another persona/agent',
|
|
1091
|
+
'chat:edited': 'An edit of an existing message',
|
|
1092
|
+
'chat:deleted': 'A deletion',
|
|
1093
|
+
'chat:reaction': 'An emoji reaction was added',
|
|
1094
|
+
'chat:reaction-remove': 'A reaction was removed',
|
|
1095
|
+
'chat:has-image': 'Has an image attachment',
|
|
1096
|
+
'chat:has-audio': 'Has an audio attachment',
|
|
1097
|
+
'chat:has-file': 'Has a file attachment',
|
|
1098
|
+
'chat:has-link': 'Contains a link',
|
|
1099
|
+
'chat:command': 'A slash/bot command invocation',
|
|
1100
|
+
'chat:private': 'Private conversation',
|
|
1101
|
+
'chat:group': 'Group (multi-party) conversation',
|
|
1102
|
+
'chat:thread': 'Occurred in a thread',
|
|
1103
|
+
};
|
|
831
1104
|
/** Refusal category → Discord reaction emoji. Unknown categories get 🛑. */
|
|
832
1105
|
static REFUSAL_REACTIONS = {
|
|
833
1106
|
bio: '☣️',
|
|
@@ -875,6 +1148,181 @@ export class AgentFramework {
|
|
|
875
1148
|
console.error('[inference-refusal] reaction failed:', err instanceof Error ? err.message : err);
|
|
876
1149
|
}
|
|
877
1150
|
}
|
|
1151
|
+
/**
|
|
1152
|
+
* Rewind the single turn that fed a refused generation: redact the newest
|
|
1153
|
+
* message the agent did NOT author (a tool result, or an incoming message)
|
|
1154
|
+
* and inject a metadata-only marker in its place. The marker carries a
|
|
1155
|
+
* *description* of what was withheld (kind + size), never the content — so it
|
|
1156
|
+
* cannot itself re-trip the classifier — while the raw record survives in the
|
|
1157
|
+
* chronicle log for forensics. Returns a record describing the removal, or
|
|
1158
|
+
* null if there is nothing eligible to rewind (e.g. only the agent's own
|
|
1159
|
+
* turns, or a prior system marker, remain).
|
|
1160
|
+
*
|
|
1161
|
+
* Shared by the auto path (on `stop_reason: refusal`) and the `/rewind`
|
|
1162
|
+
* host command.
|
|
1163
|
+
*/
|
|
1164
|
+
shedNewestTurn(agent) {
|
|
1165
|
+
const cm = agent.getContextManager();
|
|
1166
|
+
const all = cm.getAllMessages();
|
|
1167
|
+
const typeOf = (b) => b.type;
|
|
1168
|
+
const hasBlock = (m, t) => Array.isArray(m?.content) && m.content.some((b) => typeOf(b) === t);
|
|
1169
|
+
// Newest message that is not our own episode marker. We shed strictly
|
|
1170
|
+
// newest-first, in sequence — INCLUDING the agent's own turns (poison lives
|
|
1171
|
+
// in tool_use/narration turns too), because newest-first reaches whatever is
|
|
1172
|
+
// poisoning the context if allowed to run deep enough. The one invariant:
|
|
1173
|
+
// shed COMPLETE exchanges. Removing a `tool_result` also removes its paired
|
|
1174
|
+
// `tool_use` assistant turn, so we never leave an orphaned tool_use / signed
|
|
1175
|
+
// `thinking` block — which the API rejects with a 400 ("thinking blocks in
|
|
1176
|
+
// the latest assistant message cannot be modified"). We do NOT add a marker
|
|
1177
|
+
// here; the caller keeps one consolidated marker (updateRewindMarker).
|
|
1178
|
+
let idx = -1;
|
|
1179
|
+
for (let i = all.length - 1; i >= 0; i--) {
|
|
1180
|
+
const meta = (all[i].metadata ?? {});
|
|
1181
|
+
if (meta.system)
|
|
1182
|
+
continue;
|
|
1183
|
+
idx = i;
|
|
1184
|
+
break;
|
|
1185
|
+
}
|
|
1186
|
+
if (idx < 0)
|
|
1187
|
+
return null;
|
|
1188
|
+
const msg = all[idx];
|
|
1189
|
+
const md = (msg.metadata ?? {});
|
|
1190
|
+
const content = Array.isArray(msg.content) ? msg.content : [];
|
|
1191
|
+
const toolBlocks = content.filter((b) => typeOf(b) === 'tool_result');
|
|
1192
|
+
const images = content.filter((b) => typeOf(b) === 'image').length;
|
|
1193
|
+
const textLen = content
|
|
1194
|
+
.filter((b) => typeOf(b) === 'text')
|
|
1195
|
+
.reduce((n, b) => n + String(b.text ?? '').length, 0);
|
|
1196
|
+
// Determine the complete exchange to remove.
|
|
1197
|
+
const removedIds = [msg.id];
|
|
1198
|
+
let kind;
|
|
1199
|
+
let descriptor;
|
|
1200
|
+
if (toolBlocks.length > 0) {
|
|
1201
|
+
kind = 'tool';
|
|
1202
|
+
// Pair the tool_result with the tool_use assistant turn right before it.
|
|
1203
|
+
if (idx - 1 >= 0 && hasBlock(all[idx - 1], 'tool_use')) {
|
|
1204
|
+
removedIds.push(all[idx - 1].id);
|
|
1205
|
+
}
|
|
1206
|
+
const sz = toolBlocks.reduce((n, b) => n + String(b.content ?? '').length, 0);
|
|
1207
|
+
descriptor = `a tool exchange (~${Math.max(1, Math.round(sz / 1024))}KB` +
|
|
1208
|
+
`${images ? `, ${images} image(s)` : ''})`;
|
|
1209
|
+
}
|
|
1210
|
+
else if (md.messageId) {
|
|
1211
|
+
kind = 'human';
|
|
1212
|
+
descriptor = `an incoming message from ${msg.participant} ` +
|
|
1213
|
+
`(${textLen} chars${images ? `, ${images} image(s)` : ''})`;
|
|
1214
|
+
}
|
|
1215
|
+
else {
|
|
1216
|
+
kind = 'other';
|
|
1217
|
+
descriptor = `a ${msg.participant} turn ` +
|
|
1218
|
+
`(${textLen} chars${images ? `, ${images} image(s)` : ''})`;
|
|
1219
|
+
}
|
|
1220
|
+
const discordRef = md.messageId && md.channelId
|
|
1221
|
+
? { channelId: String(md.channelId), messageId: String(md.messageId) }
|
|
1222
|
+
: undefined;
|
|
1223
|
+
for (const id of removedIds)
|
|
1224
|
+
cm.removeMessage(id);
|
|
1225
|
+
return { kind, descriptor, removedIds, discordRef };
|
|
1226
|
+
}
|
|
1227
|
+
/**
|
|
1228
|
+
* Maintain exactly ONE consolidated marker for the current rewind episode,
|
|
1229
|
+
* updated in place as more turns are shed. Six rewinds ⇒ one message that
|
|
1230
|
+
* says "the 6 most recent turns were set aside", not six separate notes — so
|
|
1231
|
+
* the context converges (shed N, add 1) instead of growing, and the marker
|
|
1232
|
+
* sits at the tail giving the model something actionable to answer once the
|
|
1233
|
+
* refusal clears. Returns the running count.
|
|
1234
|
+
*/
|
|
1235
|
+
updateRewindMarker(agent, category, cause = 'refusal') {
|
|
1236
|
+
const cm = agent.getContextManager();
|
|
1237
|
+
const ep = this.rewindEpisode.get(agent.name);
|
|
1238
|
+
const count = (ep?.count ?? 0) + 1;
|
|
1239
|
+
const why = cause === 'refusal'
|
|
1240
|
+
? `the model refused on them (content filter: ${category}). Their content is ` +
|
|
1241
|
+
`not reproduced here (so this note can't re-trigger the filter); the ` +
|
|
1242
|
+
`originals remain in the raw record`
|
|
1243
|
+
: `the model API kept rejecting the conversation on them (${category}). ` +
|
|
1244
|
+
`Their content is not reproduced here (so this note can't re-trigger the ` +
|
|
1245
|
+
`rejection); the originals remain in the raw record`;
|
|
1246
|
+
const text = `[refusal-rewind] The ${count} most recent turn(s) were set aside because ` +
|
|
1247
|
+
`${why}. You are clear to continue — if you ` +
|
|
1248
|
+
`were mid-task, take a different approach; otherwise carry on with whatever ` +
|
|
1249
|
+
`is now in front of you, or briefly acknowledge the gap and ask what's next.`;
|
|
1250
|
+
const blocks = [{ type: 'text', text }];
|
|
1251
|
+
if (ep) {
|
|
1252
|
+
cm.editMessage(ep.markerId, blocks);
|
|
1253
|
+
this.rewindEpisode.set(agent.name, { markerId: ep.markerId, count, category });
|
|
1254
|
+
}
|
|
1255
|
+
else {
|
|
1256
|
+
const id = cm.addMessage('user', blocks, {
|
|
1257
|
+
system: true, kind: 'refusal-rewind', category, count, cause,
|
|
1258
|
+
});
|
|
1259
|
+
this.rewindEpisode.set(agent.name, { markerId: id, count, category });
|
|
1260
|
+
}
|
|
1261
|
+
return count;
|
|
1262
|
+
}
|
|
1263
|
+
/**
|
|
1264
|
+
* Announce a refusal-rewind on the conversational surface (Discord), used
|
|
1265
|
+
* when the withheld turn was a *human* message so it isn't dropped silently.
|
|
1266
|
+
* Best-effort; mirrors reactToRefusal's locus resolution.
|
|
1267
|
+
*/
|
|
1268
|
+
async announceRewind(agentName, rec, category) {
|
|
1269
|
+
try {
|
|
1270
|
+
const incoming = this.channelRegistry?.buildChannelContext()?.incoming;
|
|
1271
|
+
if (!incoming)
|
|
1272
|
+
return;
|
|
1273
|
+
const parts = incoming.channelId.split(':');
|
|
1274
|
+
if (parts[0] !== 'discord')
|
|
1275
|
+
return;
|
|
1276
|
+
const channelId = parts[parts.length - 1];
|
|
1277
|
+
const serverId = this.channelRegistry?.getChannelServerId(incoming.channelId);
|
|
1278
|
+
const server = serverId ? this.mcplServerRegistry?.getServer(serverId) : null;
|
|
1279
|
+
if (!server)
|
|
1280
|
+
return;
|
|
1281
|
+
await server.sendToolsCall('send_message', {
|
|
1282
|
+
channelId,
|
|
1283
|
+
content: `⚠️ I had to set aside ${rec.descriptor} — it tripped a content ` +
|
|
1284
|
+
`filter (${category}), so it's withheld from my context and I'm ` +
|
|
1285
|
+
`continuing without it. If it was important, please rephrase or re-send.`,
|
|
1286
|
+
});
|
|
1287
|
+
}
|
|
1288
|
+
catch (err) {
|
|
1289
|
+
console.error('[refusal-rewind] announce failed:', err instanceof Error ? err.message : err);
|
|
1290
|
+
}
|
|
1291
|
+
}
|
|
1292
|
+
/**
|
|
1293
|
+
* Conclude an active `/unstick` session: post the outcome (what was shed and
|
|
1294
|
+
* whether the model stopped refusing) to the channel the command came from,
|
|
1295
|
+
* then clear the session. Idempotent — a no-op if there's no session.
|
|
1296
|
+
*/
|
|
1297
|
+
finishUnstick(agentName, success, category) {
|
|
1298
|
+
const s = this.forcedRewind.get(agentName);
|
|
1299
|
+
if (!s)
|
|
1300
|
+
return;
|
|
1301
|
+
this.forcedRewind.delete(agentName);
|
|
1302
|
+
const n = s.removed.length;
|
|
1303
|
+
const list = n
|
|
1304
|
+
? '\n' + s.removed.map((r) => `• ${r.descriptor}`).join('\n')
|
|
1305
|
+
: '';
|
|
1306
|
+
const content = success
|
|
1307
|
+
? `🔧 Unstuck **${agentName}** — shed ${n} turn(s); the model responded.${list}`
|
|
1308
|
+
: `⚠️ Couldn't unstick **${agentName}** — still refusing after ${n} rewind(s)` +
|
|
1309
|
+
`${category ? ` (category=${category})` : ''}.${list}`;
|
|
1310
|
+
console.error(`[unstick] agent=${agentName} ${success ? 'succeeded' : 'gave up'} after ${n} rewind(s)`);
|
|
1311
|
+
if (s.channelId)
|
|
1312
|
+
void this.postToChannel(s.serverId, s.channelId, content);
|
|
1313
|
+
}
|
|
1314
|
+
/** Best-effort send_message to a raw channel via a named MCPL server. */
|
|
1315
|
+
async postToChannel(serverId, channelId, content) {
|
|
1316
|
+
try {
|
|
1317
|
+
const server = this.mcplServerRegistry?.getServer(serverId);
|
|
1318
|
+
if (!server)
|
|
1319
|
+
return;
|
|
1320
|
+
await server.sendToolsCall('send_message', { channelId, content });
|
|
1321
|
+
}
|
|
1322
|
+
catch (err) {
|
|
1323
|
+
console.error('[unstick] report post failed:', err instanceof Error ? err.message : err);
|
|
1324
|
+
}
|
|
1325
|
+
}
|
|
878
1326
|
/**
|
|
879
1327
|
* Handle a `host/command` request from an MCPL surface server (e.g. a
|
|
880
1328
|
* Discord slash command). Currently supports:
|
|
@@ -885,13 +1333,41 @@ export class AgentFramework {
|
|
|
885
1333
|
* `previewActivation` render (no inference, no Chronicle writes).
|
|
886
1334
|
*/
|
|
887
1335
|
async handleHostCommand(serverId, params) {
|
|
888
|
-
if (params.command !== 'undo' && params.command !== 'hide') {
|
|
1336
|
+
if (params.command !== 'undo' && params.command !== 'hide' && params.command !== 'unstick') {
|
|
889
1337
|
return { ok: false, error: `Unknown host command: ${String(params.command)}` };
|
|
890
1338
|
}
|
|
891
1339
|
const agentName = params.agentName ?? [...this.agents.keys()][0];
|
|
892
1340
|
if (!agentName || !this.agents.has(agentName)) {
|
|
893
1341
|
return { ok: false, error: `Unknown agent: ${String(agentName)}` };
|
|
894
1342
|
}
|
|
1343
|
+
// unstick: force the refusal-rewind loop on demand (even if the agent's
|
|
1344
|
+
// autoRewind toggle is off). Redacts the turn that fed the refusal and
|
|
1345
|
+
// re-runs, up to `cap` times, until the model stops refusing. Kicks the
|
|
1346
|
+
// loop and returns immediately; the outcome (what was shed + whether it
|
|
1347
|
+
// cleared) is posted to the channel when the chain resolves.
|
|
1348
|
+
if (params.command === 'unstick') {
|
|
1349
|
+
const agent = this.agents.get(agentName);
|
|
1350
|
+
if (agent.state.status !== 'idle') {
|
|
1351
|
+
return { ok: false, error: `Cannot unstick while agent is ${agent.state.status}` };
|
|
1352
|
+
}
|
|
1353
|
+
const cap = Math.max(1, Math.min(10, Math.floor(params.maxRewinds ?? agent.refusalHandling?.maxRewinds ?? 3)));
|
|
1354
|
+
this.forcedRewind.set(agentName, {
|
|
1355
|
+
remaining: cap,
|
|
1356
|
+
removed: [],
|
|
1357
|
+
serverId,
|
|
1358
|
+
channelId: params.channelId ?? '',
|
|
1359
|
+
});
|
|
1360
|
+
this.refusalRewinds.set(agentName, 0);
|
|
1361
|
+
this.pendingRequests.push({
|
|
1362
|
+
agentName,
|
|
1363
|
+
reason: 'unstick',
|
|
1364
|
+
source: 'framework',
|
|
1365
|
+
timestamp: Date.now(),
|
|
1366
|
+
});
|
|
1367
|
+
console.error(`[unstick] agent=${agentName} started cap=${cap} ` +
|
|
1368
|
+
`by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`);
|
|
1369
|
+
return { ok: true, started: true, cap };
|
|
1370
|
+
}
|
|
895
1371
|
// hide: redact a single message (or an inclusive range) from the active
|
|
896
1372
|
// branch, addressed by Discord message id. Unlike undo this is a
|
|
897
1373
|
// removal-in-place (chronicle redact), not a branch rewind.
|
|
@@ -1410,18 +1886,86 @@ export class AgentFramework {
|
|
|
1410
1886
|
}
|
|
1411
1887
|
this.saveTurnCheckpoints(agentName, checkpoints);
|
|
1412
1888
|
}
|
|
1889
|
+
/**
|
|
1890
|
+
* Checkpoints live in one Tree state (TURN_CHECKPOINTS_TREE_ID) keyed by agent
|
|
1891
|
+
* name, each key pointing at a JSON blob of that agent's ≤MAX_TURN_CHECKPOINTS
|
|
1892
|
+
* list. This kills two unbounded-in-agents-ever dimensions at once:
|
|
1893
|
+
*
|
|
1894
|
+
* - the original single Record<agentName, list> map was rewritten whole on
|
|
1895
|
+
* every turn of every agent and never evicted a departed subagent's key;
|
|
1896
|
+
* - the intermediate design (one state slot per agent) fixed the writes but
|
|
1897
|
+
* leaked a permanent chronicle state *registration* per spawn — chronicle
|
|
1898
|
+
* has no deregistration, and the state index is rewritten and fsynced on
|
|
1899
|
+
* every sync tick, so per-tick cost grew with fleet history anyway.
|
|
1900
|
+
*
|
|
1901
|
+
* A tree gives per-key O(entry) writes, real key removal (treeRemove), and
|
|
1902
|
+
* exactly one registration for the life of the store. The legacy map state
|
|
1903
|
+
* (TURN_CHECKPOINTS_ID) is kept as a read-only fallback for stores written
|
|
1904
|
+
* before the split; eviction tombstones legacy keys so a reused agent name
|
|
1905
|
+
* can't inherit a dead agent's checkpoints through the fallback.
|
|
1906
|
+
*/
|
|
1907
|
+
legacyCheckpointKeys = null;
|
|
1908
|
+
hasLegacyCheckpoints(agentName) {
|
|
1909
|
+
if (!this.legacyCheckpointKeys) {
|
|
1910
|
+
const legacy = this.store.getStateJson(TURN_CHECKPOINTS_ID);
|
|
1911
|
+
this.legacyCheckpointKeys = new Set(legacy && typeof legacy === 'object' ? Object.keys(legacy) : []);
|
|
1912
|
+
}
|
|
1913
|
+
return this.legacyCheckpointKeys.has(agentName);
|
|
1914
|
+
}
|
|
1413
1915
|
getTurnCheckpoints(agentName) {
|
|
1414
|
-
const
|
|
1415
|
-
if (
|
|
1916
|
+
const entry = this.store.treeGet(TURN_CHECKPOINTS_TREE_ID, agentName);
|
|
1917
|
+
if (entry) {
|
|
1918
|
+
const blob = this.store.getBlob(entry.blobHash);
|
|
1919
|
+
if (!blob)
|
|
1920
|
+
return [];
|
|
1921
|
+
try {
|
|
1922
|
+
const parsed = JSON.parse(blob.toString());
|
|
1923
|
+
return Array.isArray(parsed) ? parsed : [];
|
|
1924
|
+
}
|
|
1925
|
+
catch {
|
|
1926
|
+
return [];
|
|
1927
|
+
}
|
|
1928
|
+
}
|
|
1929
|
+
if (!this.hasLegacyCheckpoints(agentName))
|
|
1416
1930
|
return [];
|
|
1417
|
-
const
|
|
1418
|
-
|
|
1931
|
+
const legacy = this.store.getStateJson(TURN_CHECKPOINTS_ID);
|
|
1932
|
+
const list = legacy?.[agentName];
|
|
1933
|
+
return Array.isArray(list) ? [...list] : [];
|
|
1419
1934
|
}
|
|
1420
1935
|
saveTurnCheckpoints(agentName, checkpoints) {
|
|
1421
|
-
const
|
|
1422
|
-
const
|
|
1423
|
-
|
|
1424
|
-
|
|
1936
|
+
const bytes = Buffer.from(JSON.stringify(checkpoints));
|
|
1937
|
+
const blobHash = this.store.storeBlob(bytes, 'application/json');
|
|
1938
|
+
this.store.treeSet(TURN_CHECKPOINTS_TREE_ID, agentName, {
|
|
1939
|
+
blobHash,
|
|
1940
|
+
size: bytes.length,
|
|
1941
|
+
mode: 0o644,
|
|
1942
|
+
});
|
|
1943
|
+
}
|
|
1944
|
+
/**
|
|
1945
|
+
* Drop a departed agent's checkpoint tree key and turn/redo bookkeeping.
|
|
1946
|
+
* Without this, spawn-and-dispose agents leave a key (and in-memory map
|
|
1947
|
+
* entries) behind for the life of the store/session.
|
|
1948
|
+
*/
|
|
1949
|
+
evictTurnCheckpoints(agentName) {
|
|
1950
|
+
this.turnCounters.delete(agentName);
|
|
1951
|
+
this.redoStacks.delete(agentName);
|
|
1952
|
+
// Diagnostics maps are also keyed by agent name and never evicted —
|
|
1953
|
+
// spawn-and-dispose fleets would grow them for the session's life.
|
|
1954
|
+
this.staleWarnAt.delete(agentName);
|
|
1955
|
+
this.lastInferenceAt.delete(agentName);
|
|
1956
|
+
const entry = this.store.treeGet(TURN_CHECKPOINTS_TREE_ID, agentName);
|
|
1957
|
+
if (this.hasLegacyCheckpoints(agentName)) {
|
|
1958
|
+
// A bare treeRemove would resurrect the legacy map's list through the
|
|
1959
|
+
// fallback next time this name is reused — shadow it with an empty
|
|
1960
|
+
// tombstone instead. size <= 2 ⇔ the blob is already "[]".
|
|
1961
|
+
if (!entry || entry.size > 2) {
|
|
1962
|
+
this.saveTurnCheckpoints(agentName, []);
|
|
1963
|
+
}
|
|
1964
|
+
return;
|
|
1965
|
+
}
|
|
1966
|
+
if (entry) {
|
|
1967
|
+
this.store.treeRemove(TURN_CHECKPOINTS_TREE_ID, agentName);
|
|
1968
|
+
}
|
|
1425
1969
|
}
|
|
1426
1970
|
/**
|
|
1427
1971
|
* Run until the queue is empty and all agents are idle.
|
|
@@ -1429,6 +1973,9 @@ export class AgentFramework {
|
|
|
1429
1973
|
*/
|
|
1430
1974
|
async runUntilIdle() {
|
|
1431
1975
|
while (!this.queue.isEmpty ||
|
|
1976
|
+
// Direct inference requests (e.g. runEphemeralToCompletion) bypass the
|
|
1977
|
+
// event queue — without this the loop can exit before they're drained.
|
|
1978
|
+
this.pendingRequests.length > 0 ||
|
|
1432
1979
|
this.activeStreams.size > 0 ||
|
|
1433
1980
|
Array.from(this.agents.values()).some((a) => a.state.status !== 'idle')) {
|
|
1434
1981
|
await this.processNextEvent();
|
|
@@ -1722,6 +2269,9 @@ export class AgentFramework {
|
|
|
1722
2269
|
reason: 'mcpl:channel-incoming',
|
|
1723
2270
|
source: event.serverId,
|
|
1724
2271
|
timestamp: Date.now(),
|
|
2272
|
+
// Route this turn's auto-published speech back to THIS channel, not
|
|
2273
|
+
// the global most-recent-inbound locus (item-3 redux, trunk agents).
|
|
2274
|
+
channelId: event.channelId,
|
|
1725
2275
|
});
|
|
1726
2276
|
}
|
|
1727
2277
|
}
|
|
@@ -1800,6 +2350,9 @@ export class AgentFramework {
|
|
|
1800
2350
|
reason: 'mcpl:channel-incoming',
|
|
1801
2351
|
source: event.serverId,
|
|
1802
2352
|
timestamp: Date.now(),
|
|
2353
|
+
// A fork's home channel wins in routeSpeech regardless, but carry the
|
|
2354
|
+
// triggering channel too so the trunk/active path stays consistent.
|
|
2355
|
+
channelId: event.channelId,
|
|
1803
2356
|
});
|
|
1804
2357
|
}
|
|
1805
2358
|
}
|
|
@@ -1872,6 +2425,7 @@ export class AgentFramework {
|
|
|
1872
2425
|
this.agents.delete(agentName);
|
|
1873
2426
|
this.agentConfigs.delete(agentName);
|
|
1874
2427
|
this.conversationAgentHomes.delete(agentName);
|
|
2428
|
+
this.evictTurnCheckpoints(agentName);
|
|
1875
2429
|
this.emitTrace({
|
|
1876
2430
|
type: 'mcpl:conversation-disposed',
|
|
1877
2431
|
agentName,
|
|
@@ -1961,6 +2515,15 @@ export class AgentFramework {
|
|
|
1961
2515
|
};
|
|
1962
2516
|
const id = this.addMessage('user', event.content, metadata);
|
|
1963
2517
|
this.emitTrace({ type: 'message:added', messageId: id, source: 'mcpl:push-event' });
|
|
2518
|
+
// Some push events carry a channel of origin (Discord DMs arrive here, not
|
|
2519
|
+
// via channels/incoming, because discord-mcpl forwards them with the channel
|
|
2520
|
+
// closed). Register + open that channel so routeSpeech can resolve it, and
|
|
2521
|
+
// thread it onto the wake request so the reply routes back to it instead of
|
|
2522
|
+
// the global locus (item-3 redux, DM sub-case).
|
|
2523
|
+
const triggerChannel = this.derivePushEventChannel(event.origin);
|
|
2524
|
+
if (triggerChannel && this.channelRegistry) {
|
|
2525
|
+
this.channelRegistry.ensureChannelRegistered(event.serverId, triggerChannel.channelId, triggerChannel.label);
|
|
2526
|
+
}
|
|
1964
2527
|
if (event.triggerInference) {
|
|
1965
2528
|
// Default broadcast excludes conversation forks (channel-driven).
|
|
1966
2529
|
const targetAgents = event.targetAgents
|
|
@@ -1971,10 +2534,43 @@ export class AgentFramework {
|
|
|
1971
2534
|
reason: 'mcpl:push-event',
|
|
1972
2535
|
source: event.serverId,
|
|
1973
2536
|
timestamp: Date.now(),
|
|
2537
|
+
channelId: triggerChannel?.channelId,
|
|
1974
2538
|
});
|
|
1975
2539
|
}
|
|
1976
2540
|
}
|
|
1977
2541
|
}
|
|
2542
|
+
/**
|
|
2543
|
+
* Derive the MCPL composite channel id (the outbound routing locus) for a
|
|
2544
|
+
* push event from its server-defined `origin`, if it carries one.
|
|
2545
|
+
*
|
|
2546
|
+
* Discord DMs are the motivating case: discord-mcpl forwards them via
|
|
2547
|
+
* push/event (the DM channel is closed), so they never pass through
|
|
2548
|
+
* channels/incoming and their channel is never registered — leaving the
|
|
2549
|
+
* agent's reply with nowhere to route (item-3 redux, DM sub-case). Prefers an
|
|
2550
|
+
* explicit `origin.mcplChannelId` (a surface declaring its own composite id —
|
|
2551
|
+
* the surface-agnostic contract); otherwise reconstructs the Discord form
|
|
2552
|
+
* `discord:{guildId|dm}:{channelId}` from origin fields, matching
|
|
2553
|
+
* discord-mcpl's `mcplChannelId()` / `parseMcplChannelId()` convention so the
|
|
2554
|
+
* fix works even against a discord-mcpl build that predates `mcplChannelId`.
|
|
2555
|
+
* Returns undefined for push events with no channel provenance (heartbeats,
|
|
2556
|
+
* timers), which correctly keep the global fallback.
|
|
2557
|
+
*/
|
|
2558
|
+
derivePushEventChannel(origin) {
|
|
2559
|
+
if (!origin)
|
|
2560
|
+
return undefined;
|
|
2561
|
+
const label = typeof origin.channelName === 'string' ? origin.channelName : undefined;
|
|
2562
|
+
const explicit = origin.mcplChannelId;
|
|
2563
|
+
if (typeof explicit === 'string' && explicit) {
|
|
2564
|
+
return { channelId: explicit, label };
|
|
2565
|
+
}
|
|
2566
|
+
// Discord fallback: reconstruct the composite from origin parts. `guildId`
|
|
2567
|
+
// is null for a DM (→ 'dm'); a real guild id for a non-open guild channel.
|
|
2568
|
+
if (origin.source === 'discord' && typeof origin.channelId === 'string' && origin.channelId) {
|
|
2569
|
+
const guild = typeof origin.guildId === 'string' && origin.guildId ? origin.guildId : 'dm';
|
|
2570
|
+
return { channelId: `discord:${guild}:${origin.channelId}`, label };
|
|
2571
|
+
}
|
|
2572
|
+
return undefined;
|
|
2573
|
+
}
|
|
1978
2574
|
async processInferenceRequests() {
|
|
1979
2575
|
if (this.pendingRequests.length === 0) {
|
|
1980
2576
|
return;
|
|
@@ -2004,6 +2600,7 @@ export class AgentFramework {
|
|
|
2004
2600
|
requestCount: requests.length,
|
|
2005
2601
|
oldestRequestAge: now - oldest,
|
|
2006
2602
|
});
|
|
2603
|
+
console.error(`[inference-dropped] agent=${agentName} reason=agent_not_found requests=${requests.length}`);
|
|
2007
2604
|
continue;
|
|
2008
2605
|
}
|
|
2009
2606
|
// Skip if agent is busy (inferring, streaming, or waiting for tools)
|
|
@@ -2018,17 +2615,37 @@ export class AgentFramework {
|
|
|
2018
2615
|
requestCount: requests.length,
|
|
2019
2616
|
oldestRequestAge: now - oldest,
|
|
2020
2617
|
});
|
|
2618
|
+
// Loud (but throttled) note that requests are waiting on a busy agent.
|
|
2619
|
+
if ((this.staleWarnAt.get(agentName) ?? 0) < now - 60_000) {
|
|
2620
|
+
this.staleWarnAt.set(agentName, now);
|
|
2621
|
+
console.error(`[inference-stale] agent=${agentName} busy (${agent.state.status}) — ` +
|
|
2622
|
+
`${requests.length} request(s) waiting ${Math.round((now - oldest) / 1000)}s`);
|
|
2623
|
+
}
|
|
2021
2624
|
}
|
|
2022
2625
|
this.pendingRequests.push(...requests);
|
|
2023
2626
|
continue;
|
|
2024
2627
|
}
|
|
2025
2628
|
// Check policy
|
|
2026
2629
|
if (!this.inferencePolicy.shouldInfer(agentName, requests, state)) {
|
|
2630
|
+
// Loud drop: a queued request that dies here is otherwise invisible —
|
|
2631
|
+
// the 2026-07-09 mythos "not responding" diagnosis burned hours on
|
|
2632
|
+
// exactly this class of silent drop. One stderr line per drop.
|
|
2633
|
+
console.error(`[inference-dropped] agent=${agentName} reason=policy-skip ` +
|
|
2634
|
+
`requests=${requests.length} triggers=${requests.map((r) => r.reason).join(',')}`);
|
|
2027
2635
|
continue;
|
|
2028
2636
|
}
|
|
2029
2637
|
// Start streaming inference (non-blocking — driveStream runs in background)
|
|
2030
2638
|
const trigger = requests[0];
|
|
2031
|
-
|
|
2639
|
+
// Route this turn's auto-published speech to the channel that triggered
|
|
2640
|
+
// it (item-3 redux). A batched wake may carry several triggering channels
|
|
2641
|
+
// (messages arrived in >1 channel while the agent was busy/idle) — reply
|
|
2642
|
+
// in the MOST-RECENT one: it matches the legacy last-inbound semantics and
|
|
2643
|
+
// is the message the agent is most likely answering. The common case is
|
|
2644
|
+
// 1 message → 1 channel (unambiguous). Non-channel wakes carry no
|
|
2645
|
+
// channelId, leaving the field undefined → global fallback. `reduce`
|
|
2646
|
+
// keeps the last defined channelId across the (FIFO-ordered) batch.
|
|
2647
|
+
const triggerChannel = requests.reduce((acc, r) => r.channelId ?? acc, undefined);
|
|
2648
|
+
await this.startAgentStream(agent, { ...trigger, channelId: triggerChannel });
|
|
2032
2649
|
}
|
|
2033
2650
|
}
|
|
2034
2651
|
async startAgentStream(agent, trigger, attempt = 0) {
|
|
@@ -2037,22 +2654,34 @@ export class AgentFramework {
|
|
|
2037
2654
|
this.recordTurnCheckpoint(agent.name);
|
|
2038
2655
|
this.redoStacks.delete(agent.name); // new work invalidates redo
|
|
2039
2656
|
}
|
|
2657
|
+
// Establish this turn's outbound routing locus (item-3 redux). Set it to the
|
|
2658
|
+
// triggering channel for a channel/DM-triggered turn; clear it otherwise so
|
|
2659
|
+
// a heartbeat / no-trigger turn doesn't inherit a previous turn's channel and
|
|
2660
|
+
// instead falls back to the global default. A given agent runs one turn at a
|
|
2661
|
+
// time, so a set here is only read during THIS turn's routeSpeech /
|
|
2662
|
+
// buildChannelContext; retries re-run with the same trigger, re-setting it.
|
|
2663
|
+
if (trigger?.channelId) {
|
|
2664
|
+
this.activeTriggerChannels.set(agent.name, trigger.channelId);
|
|
2665
|
+
}
|
|
2666
|
+
else {
|
|
2667
|
+
this.activeTriggerChannels.delete(agent.name);
|
|
2668
|
+
}
|
|
2040
2669
|
this.emitTrace({ type: 'inference:started', agentName: agent.name });
|
|
2041
2670
|
this.eventGate?.onInferenceStarted(agent.name);
|
|
2671
|
+
this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), startedAt: Date.now() });
|
|
2042
2672
|
try {
|
|
2043
2673
|
const allTools = this.getAllTools();
|
|
2044
2674
|
const tools = allTools.filter((t) => agent.canUseTool(t.name));
|
|
2045
2675
|
// Gather context from modules (pull-based) and MCPL hooks (push-based)
|
|
2046
2676
|
// Both produce ContextInjection[] that get merged before inference.
|
|
2047
2677
|
let injections;
|
|
2048
|
-
// Module gatherContext (fail-open,
|
|
2049
|
-
//
|
|
2050
|
-
//
|
|
2051
|
-
//
|
|
2052
|
-
//
|
|
2053
|
-
// agent itself, or conversation forks will see cross-channel content.
|
|
2678
|
+
// Module gatherContext (fail-open, per-module timeout — the module's
|
|
2679
|
+
// contextTimeoutMs, else the registry default). Injections are
|
|
2680
|
+
// channel-scoped via scopeInjectionsForAgent, matching the MCPL hook
|
|
2681
|
+
// injections below, so conversation forks don't see cross-channel
|
|
2682
|
+
// content.
|
|
2054
2683
|
try {
|
|
2055
|
-
const moduleInjections = await this.moduleRegistry.gatherContext(agent.name);
|
|
2684
|
+
const moduleInjections = this.scopeInjectionsForAgent(agent.name, await this.moduleRegistry.gatherContext(agent.name));
|
|
2056
2685
|
if (moduleInjections.length > 0) {
|
|
2057
2686
|
injections = moduleInjections;
|
|
2058
2687
|
}
|
|
@@ -2096,6 +2725,13 @@ export class AgentFramework {
|
|
|
2096
2725
|
type: 'inference:exhausted',
|
|
2097
2726
|
agentName: agent.name,
|
|
2098
2727
|
error: err.message,
|
|
2728
|
+
// Drives the poison-history breaker. `retryable` is kept for
|
|
2729
|
+
// observability, but the breaker gates on `errorType` — only an
|
|
2730
|
+
// `invalid_request` (a 400-class rejection of the history itself)
|
|
2731
|
+
// means retrying the same context can never succeed. auth/abort/
|
|
2732
|
+
// context_length are also non-retryable but must NOT cost history.
|
|
2733
|
+
retryable: err instanceof MembraneError ? err.retryable : undefined,
|
|
2734
|
+
errorType: err instanceof MembraneError ? err.type : undefined,
|
|
2099
2735
|
});
|
|
2100
2736
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
2101
2737
|
if (action.emit) {
|
|
@@ -2334,7 +2970,73 @@ export class AgentFramework {
|
|
|
2334
2970
|
const category = stopDetails?.category ?? 'unknown';
|
|
2335
2971
|
console.error(`[inference-refusal] agent=${agent.name} category=${category}` +
|
|
2336
2972
|
(stopDetails?.explanation ? ` explanation=${stopDetails.explanation}` : ''));
|
|
2337
|
-
|
|
2973
|
+
// Rewind: excise the turn that fed the refusal, drop a
|
|
2974
|
+
// metadata-only marker, and retry — keeping the agent on its own
|
|
2975
|
+
// model instead of substituting a fallback. Driven either by an
|
|
2976
|
+
// admin `/unstick` (forced session) or the agent's autoRewind
|
|
2977
|
+
// config; both are bounded.
|
|
2978
|
+
let handledByRewind = false;
|
|
2979
|
+
const rh = agent.refusalHandling;
|
|
2980
|
+
const forced = this.forcedRewind.get(agent.name);
|
|
2981
|
+
// Shed exactly one more (newest, in-sequence) turn and keep the
|
|
2982
|
+
// single episode marker current. `budgetLeft` bounds the loop.
|
|
2983
|
+
const doRewind = (budgetLeft, onStep, onGiveUp) => {
|
|
2984
|
+
if (!budgetLeft) {
|
|
2985
|
+
onGiveUp();
|
|
2986
|
+
return;
|
|
2987
|
+
}
|
|
2988
|
+
const rec = this.shedNewestTurn(agent);
|
|
2989
|
+
if (!rec) {
|
|
2990
|
+
onGiveUp();
|
|
2991
|
+
return;
|
|
2992
|
+
}
|
|
2993
|
+
const count = this.updateRewindMarker(agent, category);
|
|
2994
|
+
onStep(count);
|
|
2995
|
+
if (forced)
|
|
2996
|
+
forced.removed.push(rec);
|
|
2997
|
+
this.pendingRequests.push({
|
|
2998
|
+
agentName: agent.name,
|
|
2999
|
+
reason: forced ? 'unstick-retry' : 'refusal-rewind-retry',
|
|
3000
|
+
source: 'framework',
|
|
3001
|
+
timestamp: Date.now(),
|
|
3002
|
+
});
|
|
3003
|
+
handledByRewind = true;
|
|
3004
|
+
};
|
|
3005
|
+
if (forced) {
|
|
3006
|
+
doRewind(forced.remaining > 0, (count) => {
|
|
3007
|
+
forced.remaining -= 1;
|
|
3008
|
+
console.error(`[unstick] agent=${agent.name} shed ${count} turn(s) ` +
|
|
3009
|
+
`(remaining ${forced.remaining})`);
|
|
3010
|
+
}, () => this.finishUnstick(agent.name, false, category));
|
|
3011
|
+
}
|
|
3012
|
+
else if (rh?.autoRewind) {
|
|
3013
|
+
const cap = Math.max(1, rh.maxRewinds ?? 3);
|
|
3014
|
+
const used = this.refusalRewinds.get(agent.name) ?? 0;
|
|
3015
|
+
doRewind(used < cap, (count) => {
|
|
3016
|
+
this.refusalRewinds.set(agent.name, used + 1);
|
|
3017
|
+
console.error(`[refusal-rewind] agent=${agent.name} shed ${count} turn(s) so far ` +
|
|
3018
|
+
`(cap ${cap})`);
|
|
3019
|
+
}, () => {
|
|
3020
|
+
console.error(`[refusal-rewind] agent=${agent.name} gave up: ` +
|
|
3021
|
+
`${used >= cap ? `cap ${cap} reached` : 'nothing left to shed'}`);
|
|
3022
|
+
this.refusalRewinds.set(agent.name, 0);
|
|
3023
|
+
this.rewindEpisode.delete(agent.name);
|
|
3024
|
+
});
|
|
3025
|
+
}
|
|
3026
|
+
if (!handledByRewind)
|
|
3027
|
+
void this.reactToRefusal(agent.name, category);
|
|
3028
|
+
}
|
|
3029
|
+
else {
|
|
3030
|
+
// A turn that completed WITHOUT a refusal ends the rewind episode:
|
|
3031
|
+
// the model responded. Leave the consolidated marker in place as
|
|
3032
|
+
// the durable record; just clear the per-episode counters.
|
|
3033
|
+
if (this.forcedRewind.has(agent.name)) {
|
|
3034
|
+
this.finishUnstick(agent.name, true);
|
|
3035
|
+
}
|
|
3036
|
+
if (this.refusalRewinds.get(agent.name)) {
|
|
3037
|
+
this.refusalRewinds.set(agent.name, 0);
|
|
3038
|
+
}
|
|
3039
|
+
this.rewindEpisode.delete(agent.name);
|
|
2338
3040
|
}
|
|
2339
3041
|
// Dispatch speech (and thoughts if any)
|
|
2340
3042
|
if (speechContent.length > 0 || thoughts.length > 0) {
|
|
@@ -2401,21 +3103,40 @@ export class AgentFramework {
|
|
|
2401
3103
|
.map((b) => b.name)
|
|
2402
3104
|
.filter((n) => typeof n === 'string');
|
|
2403
3105
|
const silenced = toolNames.some((n) => SILENCING.has(bare(n)));
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
3106
|
+
// Split the turn's content into ordered prose segments, broken at
|
|
3107
|
+
// each tool_use / tool_result boundary. `response.content` holds the
|
|
3108
|
+
// WHOLE turn's blocks in provider order — earlier tool rounds
|
|
3109
|
+
// included (see the trailing-slice note above, which slices exactly
|
|
3110
|
+
// because the full turn is present) — so a left-to-right walk
|
|
3111
|
+
// reconstructs the emission order. Interleaved prose
|
|
3112
|
+
// ("msgA → [tool] → msgB → [tool] → msgC") is then delivered as
|
|
3113
|
+
// separate messages IN ORDER instead of being collapsed into one
|
|
3114
|
+
// trailing post (item 4). The silencing rule stays turn-wide: an
|
|
3115
|
+
// explicit send / skip suppresses ALL auto-routed prose this turn,
|
|
3116
|
+
// preserving the double-post guard.
|
|
3117
|
+
const segments = splitProseSegments(response.content);
|
|
3118
|
+
if (silenced || segments.length === 0) {
|
|
2409
3119
|
console.error(`[routing] ${agent.name}: tool-call turn [${toolNames.join(', ') || 'none'}] -> NOT routed ` +
|
|
2410
3120
|
`(${silenced ? 'silencing tool / explicit send' : 'no prose'})`);
|
|
2411
3121
|
}
|
|
2412
3122
|
else {
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
3123
|
+
// Snapshot the locus ONCE. This loop dispatches AFTER the agent
|
|
3124
|
+
// is idle (see PR #32 note below), so a queued inbound from
|
|
3125
|
+
// another channel can start a NEW turn and overwrite the per-agent
|
|
3126
|
+
// triggering channel (`activeTriggerChannels`) BETWEEN segments —
|
|
3127
|
+
// splitting one turn's reply across channels (the residual
|
|
3128
|
+
// multi-segment race left by 83dedf8's per-agent locus). Pin every
|
|
3129
|
+
// segment of THIS turn to the channel resolved right now.
|
|
3130
|
+
const turnLocus = this.channelRegistry.resolveLocus(agent.name);
|
|
3131
|
+
console.error(`[routing] ${agent.name}: tool-call turn [${toolNames.join(', ')}] -> routing ${segments.length} prose segment(s) in order -> ${turnLocus ?? '(default)'}`);
|
|
3132
|
+
// Deliver sequentially (await each) so the segments land in order.
|
|
3133
|
+
for (const seg of segments) {
|
|
3134
|
+
try {
|
|
3135
|
+
await this.channelRegistry.routeSpeech(agent.name, seg, turnLocus);
|
|
3136
|
+
}
|
|
3137
|
+
catch (err) {
|
|
3138
|
+
console.error('speech routing failed:', err);
|
|
3139
|
+
}
|
|
2419
3140
|
}
|
|
2420
3141
|
}
|
|
2421
3142
|
}
|
|
@@ -2457,6 +3178,12 @@ export class AgentFramework {
|
|
|
2457
3178
|
type: 'inference:exhausted',
|
|
2458
3179
|
agentName: agent.name,
|
|
2459
3180
|
error: err.message,
|
|
3181
|
+
// Drives the poison-history breaker. `retryable` is kept for
|
|
3182
|
+
// observability, but the breaker gates on `errorType` — only an
|
|
3183
|
+
// `invalid_request` (a 400-class rejection of the history
|
|
3184
|
+
// itself) means retrying the same context can never succeed.
|
|
3185
|
+
retryable: err instanceof MembraneError ? err.retryable : undefined,
|
|
3186
|
+
errorType: err instanceof MembraneError ? err.type : undefined,
|
|
2460
3187
|
});
|
|
2461
3188
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
2462
3189
|
if (action.emit) {
|
|
@@ -2470,18 +3197,51 @@ export class AgentFramework {
|
|
|
2470
3197
|
// Only reset if this is still the active stream (a budget restart
|
|
2471
3198
|
// may have already started a new stream, bumping streamId)
|
|
2472
3199
|
if (agent.streamId === myStreamId) {
|
|
3200
|
+
const durationMs = Date.now() - startTime;
|
|
2473
3201
|
agent.reset();
|
|
2474
3202
|
this.emitTrace({
|
|
2475
3203
|
type: 'inference:exhausted',
|
|
2476
3204
|
agentName: agent.name,
|
|
2477
3205
|
error: `Stream aborted: ${reason}`,
|
|
2478
3206
|
});
|
|
3207
|
+
// Postmortem 2026-05-28 P2 #7: persist the abort to the
|
|
3208
|
+
// inference log so future investigations can attribute the
|
|
3209
|
+
// terminal cause without relying on live in-memory reducer
|
|
3210
|
+
// state. Without this, abort-terminated inferences are
|
|
3211
|
+
// invisible to forensic queries (only request-side telemetry
|
|
3212
|
+
// via llm-calls.jsonl shows them, and only by absence).
|
|
3213
|
+
this.logInference({
|
|
3214
|
+
timestamp: startTime,
|
|
3215
|
+
agentName: agent.name,
|
|
3216
|
+
requestId,
|
|
3217
|
+
success: false,
|
|
3218
|
+
error: `Stream aborted: ${reason}`,
|
|
3219
|
+
request: compiledRequest ?? { note: 'streaming request aborted' },
|
|
3220
|
+
durationMs,
|
|
3221
|
+
});
|
|
2479
3222
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
2480
3223
|
}
|
|
2481
3224
|
break;
|
|
2482
3225
|
}
|
|
2483
|
-
case 'usage':
|
|
3226
|
+
case 'usage': {
|
|
2484
3227
|
agent.lastStreamInputTokens = event.usage.inputTokens;
|
|
3228
|
+
// Closed-loop estimator calibration (2026-07-12). Sample the REAL
|
|
3229
|
+
// prefix size of THIS API call (fresh + cache write + cache read)
|
|
3230
|
+
// and hand it to the context strategy, which accepts exactly one
|
|
3231
|
+
// sample per compile (its arm-once gate) and rejects out-of-band
|
|
3232
|
+
// ratios. It must be sampled HERE, per call: `response.details.
|
|
3233
|
+
// usage` at turn completion is CUMULATIVE across the tool-use
|
|
3234
|
+
// loop (5 calls x ~160k reported as 884k), which is not a
|
|
3235
|
+
// window-shaped number and drove the multiplier to 2.37 before
|
|
3236
|
+
// the guards caught it.
|
|
3237
|
+
try {
|
|
3238
|
+
const realTotal = (event.usage.inputTokens ?? 0) +
|
|
3239
|
+
(event.usage.cacheCreationTokens ?? 0) +
|
|
3240
|
+
(event.usage.cacheReadTokens ?? 0);
|
|
3241
|
+
const strat = agent.getContextManager?.()?.getStrategy?.();
|
|
3242
|
+
strat?.reportRealInputTokens?.(realTotal);
|
|
3243
|
+
}
|
|
3244
|
+
catch { /* calibration is best-effort */ }
|
|
2485
3245
|
this.emitTrace({
|
|
2486
3246
|
type: 'inference:usage',
|
|
2487
3247
|
agentName: agent.name,
|
|
@@ -2493,6 +3253,7 @@ export class AgentFramework {
|
|
|
2493
3253
|
},
|
|
2494
3254
|
});
|
|
2495
3255
|
break;
|
|
3256
|
+
}
|
|
2496
3257
|
}
|
|
2497
3258
|
}
|
|
2498
3259
|
}
|
|
@@ -2500,6 +3261,7 @@ export class AgentFramework {
|
|
|
2500
3261
|
// Stream itself threw (unexpected) — no retry path here, so also emit
|
|
2501
3262
|
// inference:exhausted so ephemeral agent promises can settle.
|
|
2502
3263
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
3264
|
+
const durationMs = Date.now() - startTime;
|
|
2503
3265
|
this.emitTrace({
|
|
2504
3266
|
type: 'inference:failed',
|
|
2505
3267
|
agentName: agent.name,
|
|
@@ -2511,10 +3273,31 @@ export class AgentFramework {
|
|
|
2511
3273
|
agentName: agent.name,
|
|
2512
3274
|
error: err.message,
|
|
2513
3275
|
});
|
|
3276
|
+
// Postmortem 2026-05-28 P2 #7: catch-path failures (stream itself
|
|
3277
|
+
// threw) were previously only visible via in-memory trace listeners.
|
|
3278
|
+
// Persist to inference log for forensic attribution.
|
|
3279
|
+
this.logInference({
|
|
3280
|
+
timestamp: startTime,
|
|
3281
|
+
agentName: agent.name,
|
|
3282
|
+
requestId,
|
|
3283
|
+
success: false,
|
|
3284
|
+
error: `Stream threw: ${err.message}`,
|
|
3285
|
+
request: compiledRequest ?? { note: 'streaming request threw' },
|
|
3286
|
+
durationMs,
|
|
3287
|
+
});
|
|
2514
3288
|
agent.reset();
|
|
2515
3289
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
2516
3290
|
}
|
|
2517
3291
|
finally {
|
|
3292
|
+
// Clear the gate's inference flag on EVERY exit path (paired with the
|
|
3293
|
+
// onInferenceStarted in startAgentStream). The branch-level calls above
|
|
3294
|
+
// are kept but are best-effort; if any exit path bypassed them the agent
|
|
3295
|
+
// stayed stuck in the gate's `inferring` set, which permanently BUFFERS
|
|
3296
|
+
// all incoming events → the agent silently stops waking on messages
|
|
3297
|
+
// (typing still stops, compression still runs — matching the observed
|
|
3298
|
+
// wedge). onInferenceEnded is idempotent, so a redundant call is safe.
|
|
3299
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
3300
|
+
this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), endedAt: Date.now() });
|
|
2518
3301
|
// Stop the typing indicator on every exit path (complete, error,
|
|
2519
3302
|
// exhausted, abort) so it never sticks after the turn ends.
|
|
2520
3303
|
this.channelRegistry?.stopTyping();
|
|
@@ -2599,7 +3382,10 @@ export class AgentFramework {
|
|
|
2599
3382
|
if (blocks) {
|
|
2600
3383
|
return { toolUseId: callId, content: blocks, isError: false };
|
|
2601
3384
|
}
|
|
2602
|
-
|
|
3385
|
+
// JSON.stringify returns the VALUE undefined (not a string) for undefined
|
|
3386
|
+
// input — a module tool returning `{ success: true }` with no data would
|
|
3387
|
+
// otherwise make `content.length` below throw a TypeError mid-turn.
|
|
3388
|
+
let content = JSON.stringify(afResult.data) ?? '';
|
|
2603
3389
|
if (maxChars && content.length > maxChars) {
|
|
2604
3390
|
content = safeSlice(content, 0, maxChars)
|
|
2605
3391
|
+ '\n\n[truncated — original was ' + content.length + ' chars]';
|
|
@@ -2669,11 +3455,11 @@ export class AgentFramework {
|
|
|
2669
3455
|
entryToStore.response = { blobId };
|
|
2670
3456
|
}
|
|
2671
3457
|
}
|
|
2672
|
-
// Append
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
this.store.
|
|
3458
|
+
// Append one record per inference. A read-modify-write via setStateJson
|
|
3459
|
+
// would emit a whole-array Set — record size then grows with accumulated
|
|
3460
|
+
// history (O(n²) aggregate disk) and the append_log strategy registered
|
|
3461
|
+
// for this state never sees an append.
|
|
3462
|
+
this.store.appendToStateJson(INFERENCE_LOG_ID, entryToStore);
|
|
2677
3463
|
}
|
|
2678
3464
|
logProcessEvent(event, responses) {
|
|
2679
3465
|
const entry = {
|
|
@@ -2689,11 +3475,8 @@ export class AgentFramework {
|
|
|
2689
3475
|
const blobId = this.store.storeBlob(Buffer.from(responsesJson), 'application/json');
|
|
2690
3476
|
entryToStore.responses = { blobId };
|
|
2691
3477
|
}
|
|
2692
|
-
// Append
|
|
2693
|
-
|
|
2694
|
-
const entries = Array.isArray(data) ? data : [];
|
|
2695
|
-
entries.push(entryToStore);
|
|
2696
|
-
this.store.setStateJson(PROCESS_LOG_ID, entries);
|
|
3478
|
+
// Append one record per event — same rationale as logInference.
|
|
3479
|
+
this.store.appendToStateJson(PROCESS_LOG_ID, entryToStore);
|
|
2697
3480
|
}
|
|
2698
3481
|
/**
|
|
2699
3482
|
* Find the MCPL server for a tool call by checking against the prefix map.
|
|
@@ -2731,6 +3514,16 @@ export class AgentFramework {
|
|
|
2731
3514
|
this.dispatchSleepToolCall(agentName, enrichedCall);
|
|
2732
3515
|
return;
|
|
2733
3516
|
}
|
|
3517
|
+
// Route wake-rule tools (runtime gate.json policy add/remove)
|
|
3518
|
+
if ((enrichedCall.name === 'wake_add_rule' || enrichedCall.name === 'wake_remove_rule') && this.eventGate) {
|
|
3519
|
+
this.dispatchWakeRuleToolCall(agentName, enrichedCall);
|
|
3520
|
+
return;
|
|
3521
|
+
}
|
|
3522
|
+
// Route event_tags (tag/ontology discovery)
|
|
3523
|
+
if (enrichedCall.name === 'event_tags' && this.eventGate) {
|
|
3524
|
+
this.dispatchEventTagsToolCall(agentName, enrichedCall);
|
|
3525
|
+
return;
|
|
3526
|
+
}
|
|
2734
3527
|
// Route synthesized 'think' (private reasoning) and 'skip_reply' (deliberate
|
|
2735
3528
|
// stay-silent) tools — handled by the channel registry like the other
|
|
2736
3529
|
// synthesized channel tools, but they aren't `channel_`-prefixed so they
|
|
@@ -2880,15 +3673,19 @@ export class AgentFramework {
|
|
|
2880
3673
|
// without this the only durable record of a failed inference is a field in
|
|
2881
3674
|
// llm-calls.jsonl — invisible to operator, agent, and monitoring.
|
|
2882
3675
|
if (event.type === 'inference:exhausted') {
|
|
2883
|
-
this.noteInferenceExhausted(event.agentName ?? 'unknown', event.error ?? 'unknown error');
|
|
3676
|
+
this.noteInferenceExhausted(event.agentName ?? 'unknown', event.error ?? 'unknown error', event.retryable, event.errorType);
|
|
2884
3677
|
}
|
|
2885
3678
|
else if (event.type === 'inference:completed') {
|
|
2886
3679
|
// A successful response — even mid-turn between tool calls — proves the
|
|
2887
|
-
// agent isn't hard-down; clear its consecutive-failure streak
|
|
3680
|
+
// agent isn't hard-down; clear its consecutive-failure streak (and the
|
|
3681
|
+
// poison-history breaker's rewind budget).
|
|
2888
3682
|
const name = event.agentName;
|
|
2889
3683
|
if (name && this.consecutiveInferenceFailures.get(name)) {
|
|
2890
3684
|
this.consecutiveInferenceFailures.set(name, 0);
|
|
2891
3685
|
}
|
|
3686
|
+
if (name && this.exhaustionRewinds.get(name)) {
|
|
3687
|
+
this.exhaustionRewinds.set(name, 0);
|
|
3688
|
+
}
|
|
2892
3689
|
}
|
|
2893
3690
|
const traceEvent = {
|
|
2894
3691
|
...event,
|
|
@@ -2917,15 +3714,52 @@ export class AgentFramework {
|
|
|
2917
3714
|
* inference, so this never causes a retry/wake loop.
|
|
2918
3715
|
* 3. escalation — after N consecutive failures the agent is hard-down;
|
|
2919
3716
|
* log that loudly (a repeated identical failure is the textbook signal).
|
|
3717
|
+
* 4. breaker — when the failures are an `invalid_request` (a 400-class
|
|
3718
|
+
* rejection of the history itself, e.g. corrupted tool_use/tool_result
|
|
3719
|
+
* pairing or an oversized attachment), retrying the same context can
|
|
3720
|
+
* never succeed: every new push event wakes the agent onto the same
|
|
3721
|
+
* poisoned history forever. At the hard-down threshold, automatically
|
|
3722
|
+
* quarantine: shed the newest complete exchange (the same forced-rewind
|
|
3723
|
+
* primitive `/unstick` uses — shedNewestTurn never orphans a
|
|
3724
|
+
* tool_use/thinking block) and retry, bounded by the same rewind cap so
|
|
3725
|
+
* the breaker can never eat the whole history.
|
|
3726
|
+
*/
|
|
3727
|
+
/**
|
|
3728
|
+
* Live health snapshot for /healthz and doctor tooling: gate state, queued
|
|
3729
|
+
* work, per-agent status + last inference activity. Cheap and read-only.
|
|
2920
3730
|
*/
|
|
2921
|
-
|
|
3731
|
+
healthSnapshot() {
|
|
3732
|
+
return {
|
|
3733
|
+
at: new Date().toISOString(),
|
|
3734
|
+
uptimeSec: Math.round(process.uptime()),
|
|
3735
|
+
gate: this.eventGate?.inferenceDiagnostics() ?? null,
|
|
3736
|
+
pendingRequests: this.pendingRequests.length,
|
|
3737
|
+
activeStreams: [...this.activeStreams.keys()],
|
|
3738
|
+
agents: [...this.agents.entries()].map(([name, agent]) => ({
|
|
3739
|
+
name,
|
|
3740
|
+
status: agent.state.status,
|
|
3741
|
+
consecutiveInferenceFailures: this.consecutiveInferenceFailures.get(name) ?? 0,
|
|
3742
|
+
lastInference: this.lastInferenceAt.get(name) ?? null,
|
|
3743
|
+
})),
|
|
3744
|
+
};
|
|
3745
|
+
}
|
|
3746
|
+
noteInferenceExhausted(agentName, reason, retryable, errorType) {
|
|
2922
3747
|
const streak = (this.consecutiveInferenceFailures.get(agentName) ?? 0) + 1;
|
|
2923
3748
|
this.consecutiveInferenceFailures.set(agentName, streak);
|
|
3749
|
+
this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), failedAt: Date.now(), lastError: reason.slice(0, 300) });
|
|
2924
3750
|
// (1) Durable stderr line — works in headless/daemon mode with no client.
|
|
2925
3751
|
console.error(`[inference-failed] agent=${agentName} consecutive=${streak}: ${reason}`);
|
|
3752
|
+
// (1b) Machine-greppable durable record, independent of journald/unit log
|
|
3753
|
+
// redirects: logs/failures.log under the host's working directory. This is
|
|
3754
|
+
// what connectome-doctor reads.
|
|
3755
|
+
try {
|
|
3756
|
+
mkdirSync('logs', { recursive: true });
|
|
3757
|
+
appendFileSync('logs/failures.log', JSON.stringify({ at: new Date().toISOString(), agent: agentName, consecutive: streak, reason }) + '\n');
|
|
3758
|
+
}
|
|
3759
|
+
catch { /* best-effort */ }
|
|
2926
3760
|
// (2) Agent-facing chronicle marker (no inference triggered → no loop).
|
|
2927
3761
|
const agent = this.agents.get(agentName);
|
|
2928
|
-
if (agent) {
|
|
3762
|
+
if (agent && process.env.SUPPRESS_INFERENCE_FAILED_MARKER !== '1') {
|
|
2929
3763
|
try {
|
|
2930
3764
|
agent.getContextManager().addMessage('user', [{
|
|
2931
3765
|
type: 'text',
|
|
@@ -2944,8 +3778,71 @@ export class AgentFramework {
|
|
|
2944
3778
|
if (streak >= this.inferenceFailureEscalationThreshold) {
|
|
2945
3779
|
console.error(`[inference-hard-down] agent=${agentName} has FAILED ${streak} consecutive ` +
|
|
2946
3780
|
`inferences — it cannot complete a turn. Last reason: ${reason}`);
|
|
3781
|
+
// Optional ops alert: set CONNECTOME_OPS_WEBHOOK to a Discord webhook URL.
|
|
3782
|
+
const hook = process.env.CONNECTOME_OPS_WEBHOOK;
|
|
3783
|
+
if (hook) {
|
|
3784
|
+
fetch(hook, {
|
|
3785
|
+
method: 'POST',
|
|
3786
|
+
headers: { 'content-type': 'application/json' },
|
|
3787
|
+
body: JSON.stringify({
|
|
3788
|
+
content: `\u{1F6A8} **${agentName}** hard-down: ${streak} consecutive inference failures.\nLast reason: ${reason.slice(0, 500)}`,
|
|
3789
|
+
}),
|
|
3790
|
+
}).catch((err) => console.error('[inference-hard-down] webhook post failed:', err instanceof Error ? err.message : err));
|
|
3791
|
+
}
|
|
3792
|
+
// (4) Poison-history breaker: ONLY for `invalid_request` — a 400-class
|
|
3793
|
+
// rejection of the history itself. Deliberately NOT keyed on
|
|
3794
|
+
// `retryable === false`, which membrane also returns for auth (expired
|
|
3795
|
+
// key), abort (deliberate cancel), context_length (compression's job,
|
|
3796
|
+
// and shedding newest is the wrong direction), safety and unsupported —
|
|
3797
|
+
// none of which mean the history is poisoned, and all of which would
|
|
3798
|
+
// otherwise shed good exchanges and stamp a false "the API kept
|
|
3799
|
+
// rejecting your history" marker. `retryable` is kept only for the trace.
|
|
3800
|
+
void retryable;
|
|
3801
|
+
if (errorType === 'invalid_request' && agent) {
|
|
3802
|
+
this.quarantinePoisonedHistory(agent, reason);
|
|
3803
|
+
}
|
|
2947
3804
|
}
|
|
2948
3805
|
}
|
|
3806
|
+
/**
|
|
3807
|
+
* Automatic poison-history quarantine (the actual "breaker"). Sheds the
|
|
3808
|
+
* newest complete exchange from the agent's history — reusing the same
|
|
3809
|
+
* primitives as the refusal auto-rewind / `/unstick` (shedNewestTurn +
|
|
3810
|
+
* the single consolidated episode marker) — and queues a retry so the
|
|
3811
|
+
* rewound history is verified immediately instead of waiting for the next
|
|
3812
|
+
* push event to wake the agent onto the same poisoned context.
|
|
3813
|
+
*
|
|
3814
|
+
* Bounded: at most `refusalHandling.maxRewinds` (default 3, hard cap 10)
|
|
3815
|
+
* sheds per failure episode; the budget resets on any successful inference.
|
|
3816
|
+
* At the cap (or with nothing left to shed) it stops — the agent stays up
|
|
3817
|
+
* and hard-down logging continues, but no further history is consumed.
|
|
3818
|
+
*/
|
|
3819
|
+
quarantinePoisonedHistory(agent, reason) {
|
|
3820
|
+
const cap = Math.max(1, Math.min(10, agent.refusalHandling?.maxRewinds ?? 3));
|
|
3821
|
+
const used = this.exhaustionRewinds.get(agent.name) ?? 0;
|
|
3822
|
+
if (used >= cap) {
|
|
3823
|
+
console.error(`[inference-rewind] agent=${agent.name} rewind cap ${cap} reached — ` +
|
|
3824
|
+
`not shedding further history. Manual repair (/unstick or /undo) needed.`);
|
|
3825
|
+
return;
|
|
3826
|
+
}
|
|
3827
|
+
const rec = this.shedNewestTurn(agent);
|
|
3828
|
+
if (!rec) {
|
|
3829
|
+
console.error(`[inference-rewind] agent=${agent.name} has nothing left to shed — ` +
|
|
3830
|
+
`history is only system markers. Manual repair needed.`);
|
|
3831
|
+
return;
|
|
3832
|
+
}
|
|
3833
|
+
this.exhaustionRewinds.set(agent.name, used + 1);
|
|
3834
|
+
const count = this.updateRewindMarker(agent, truncateReason(reason), 'inference-failure');
|
|
3835
|
+
console.error(`[inference-rewind] agent=${agent.name} auto-quarantined ${rec.descriptor} ` +
|
|
3836
|
+
`(${count} turn(s) shed this episode, cap ${cap}) after non-retryable ` +
|
|
3837
|
+
`inference failures — retrying on the rewound history.`);
|
|
3838
|
+
// Retry immediately on the repaired history (bounded by the cap above).
|
|
3839
|
+
this.pendingRequests.push({
|
|
3840
|
+
agentName: agent.name,
|
|
3841
|
+
reason: 'inference-failure-rewind-retry',
|
|
3842
|
+
source: 'framework',
|
|
3843
|
+
timestamp: Date.now(),
|
|
3844
|
+
});
|
|
3845
|
+
}
|
|
2949
3846
|
// ==========================================================================
|
|
2950
3847
|
// MCPL subsystem wiring
|
|
2951
3848
|
// ==========================================================================
|
|
@@ -2986,6 +3883,21 @@ export class AgentFramework {
|
|
|
2986
3883
|
}
|
|
2987
3884
|
},
|
|
2988
3885
|
shouldTriggerInference: triggerFilter,
|
|
3886
|
+
// Route a conversation fork's plain-text speech to its HOME channel, not
|
|
3887
|
+
// the process-global most-recent-inbound locus (item 3). The trunk agent
|
|
3888
|
+
// has no home entry, so this returns undefined and routeSpeech falls back
|
|
3889
|
+
// to defaultPublishChannel. `conversationAgentHomes` is the permanent
|
|
3890
|
+
// spawn-time binding; `channelForAgent` is the router's live binding as a
|
|
3891
|
+
// belt-and-suspenders fallback.
|
|
3892
|
+
homeChannelResolver: (agentName) => this.conversationAgentHomes.get(agentName)
|
|
3893
|
+
?? this.conversationRouter?.channelForAgent(agentName),
|
|
3894
|
+
// Route a single TRUNK agent's plain-text speech to the channel that
|
|
3895
|
+
// triggered its CURRENT turn (item-3 redux). connectome-host runs every
|
|
3896
|
+
// agent as a trunk (it never exposes conversation forks), so without this
|
|
3897
|
+
// a reply falls back to the process-global most-recent-inbound locus and
|
|
3898
|
+
// a concurrent message in another channel hijacks it. Empty for
|
|
3899
|
+
// heartbeat / no-trigger turns → correct global fallback.
|
|
3900
|
+
activeChannelResolver: (agentName) => this.activeTriggerChannels.get(agentName),
|
|
2989
3901
|
// A text-only turn whose speech couldn't be delivered must not vanish
|
|
2990
3902
|
// silently: record a `[discord-send-failed]` marker in chronicle so the
|
|
2991
3903
|
// agent sees, on her next turn, that her reply never reached the human.
|
|
@@ -3004,8 +3916,9 @@ export class AgentFramework {
|
|
|
3004
3916
|
}
|
|
3005
3917
|
},
|
|
3006
3918
|
});
|
|
3007
|
-
// Host capabilities advertised during the MCP handshake
|
|
3008
|
-
|
|
3919
|
+
// Host capabilities advertised during the MCP handshake — stored so
|
|
3920
|
+
// servers can also be connected later at runtime (connectMcplServer).
|
|
3921
|
+
this.mcplHostCapabilities = {
|
|
3009
3922
|
version: '0.4',
|
|
3010
3923
|
pushEvents: true,
|
|
3011
3924
|
contextHooks: {
|
|
@@ -3016,56 +3929,193 @@ export class AgentFramework {
|
|
|
3016
3929
|
};
|
|
3017
3930
|
for (const config of serverConfigs) {
|
|
3018
3931
|
try {
|
|
3019
|
-
|
|
3020
|
-
// registers channels — handleRegister fires during the handshake.
|
|
3021
|
-
if (config.channelSubscription !== undefined && this.channelRegistry) {
|
|
3022
|
-
this.channelRegistry.setSubscriptionPolicy(config.id, config.channelSubscription);
|
|
3023
|
-
}
|
|
3024
|
-
const connection = await this.mcplServerRegistry.addServer(config, hostCapabilities);
|
|
3025
|
-
// Wire event listeners then flush any events that arrived during the
|
|
3026
|
-
// handshake window (between setupMessageRouting and now).
|
|
3027
|
-
this.wireMcplEvents(connection);
|
|
3028
|
-
connection.ready();
|
|
3029
|
-
// Initialize feature sets if server advertises MCPL capabilities
|
|
3030
|
-
if (connection.capabilities) {
|
|
3031
|
-
const updateParams = this.featureSetManager.initializeServer(config.id, connection.capabilities, {
|
|
3032
|
-
enabledFeatureSets: config.enabledFeatureSets,
|
|
3033
|
-
disabledFeatureSets: config.disabledFeatureSets,
|
|
3034
|
-
});
|
|
3035
|
-
// Inform server which feature sets are enabled/disabled
|
|
3036
|
-
if (updateParams.enabled?.length || updateParams.disabled?.length) {
|
|
3037
|
-
connection.sendFeatureSetsUpdate(updateParams);
|
|
3038
|
-
}
|
|
3039
|
-
// Configure scope whitelist/blacklist patterns
|
|
3040
|
-
if (config.scopes) {
|
|
3041
|
-
this.scopeManager.configureAll(config.scopes);
|
|
3042
|
-
}
|
|
3043
|
-
// Register stateful feature sets with checkpoint manager (Step 8)
|
|
3044
|
-
if (this.checkpointManager) {
|
|
3045
|
-
const declared = this.featureSetManager.getDeclaredFeatureSets(config.id);
|
|
3046
|
-
if (declared) {
|
|
3047
|
-
for (const [fsName, fsDecl] of Object.entries(declared)) {
|
|
3048
|
-
if (fsDecl.rollback || fsDecl.hostState) {
|
|
3049
|
-
this.checkpointManager.registerFeatureSet(config.id, fsName, {
|
|
3050
|
-
hostState: fsDecl.hostState ?? false,
|
|
3051
|
-
rollback: fsDecl.rollback ?? false,
|
|
3052
|
-
});
|
|
3053
|
-
}
|
|
3054
|
-
}
|
|
3055
|
-
}
|
|
3056
|
-
}
|
|
3057
|
-
}
|
|
3058
|
-
this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` });
|
|
3932
|
+
await this.connectMcplServerInternal(config);
|
|
3059
3933
|
}
|
|
3060
3934
|
catch (error) {
|
|
3061
3935
|
// Fail-open: log and continue with remaining servers
|
|
3062
3936
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
3063
3937
|
console.error(`Failed to connect MCPL server "${config.id}":`, err.message);
|
|
3938
|
+
// Reaching this catch means no stub was created, so nothing will
|
|
3939
|
+
// retry — except in the edge case where addServer succeeded (with a
|
|
3940
|
+
// reconnecting stub) and a later setup step threw; config.reconnect
|
|
3941
|
+
// is the best cheap approximation of that.
|
|
3942
|
+
this.emitTrace({
|
|
3943
|
+
type: 'mcpl:server-connect-failed',
|
|
3944
|
+
serverId: config.id,
|
|
3945
|
+
error: err.message,
|
|
3946
|
+
attempt: 0,
|
|
3947
|
+
willRetry: config.reconnect === true,
|
|
3948
|
+
});
|
|
3064
3949
|
}
|
|
3065
3950
|
}
|
|
3066
3951
|
// Discover tools from all connected servers
|
|
3067
3952
|
await this.refreshMcplTools();
|
|
3068
3953
|
}
|
|
3954
|
+
/**
|
|
3955
|
+
* Connect a single MCPL server: register routing entries, open the
|
|
3956
|
+
* connection, wire events, and initialize feature sets / scopes /
|
|
3957
|
+
* checkpoints. Shared by startup (initializeMcpl) and the runtime
|
|
3958
|
+
* lifecycle API (connectMcplServer). Throws on connection failure.
|
|
3959
|
+
*/
|
|
3960
|
+
async connectMcplServerInternal(config) {
|
|
3961
|
+
if (!this.mcplServerRegistry || !this.mcplHostCapabilities) {
|
|
3962
|
+
throw new Error('MCPL subsystem is not initialized');
|
|
3963
|
+
}
|
|
3964
|
+
// Register prefix + config for tool dispatch routing (idempotent with
|
|
3965
|
+
// the pre-registration pass in initializeMcpl).
|
|
3966
|
+
const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
|
|
3967
|
+
this.mcplPrefixMap.set(prefix, config.id);
|
|
3968
|
+
this.mcplServerConfigs.set(config.id, config);
|
|
3969
|
+
// Record per-server channel subscription policy before the server
|
|
3970
|
+
// registers channels — handleRegister fires during the handshake.
|
|
3971
|
+
if (config.channelSubscription !== undefined && this.channelRegistry) {
|
|
3972
|
+
this.channelRegistry.setSubscriptionPolicy(config.id, config.channelSubscription);
|
|
3973
|
+
}
|
|
3974
|
+
const connection = await this.mcplServerRegistry.addServer(config, this.mcplHostCapabilities);
|
|
3975
|
+
// Wire event listeners then flush any events that arrived during the
|
|
3976
|
+
// handshake window (between setupMessageRouting and now).
|
|
3977
|
+
this.wireMcplEvents(connection);
|
|
3978
|
+
connection.ready();
|
|
3979
|
+
// Initialize feature sets if server advertises MCPL capabilities
|
|
3980
|
+
this.registerMcplServerFeatures(config, connection);
|
|
3981
|
+
this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` });
|
|
3982
|
+
}
|
|
3983
|
+
/**
|
|
3984
|
+
* (Re-)establish a server's host-side MCPL registration: feature-set state
|
|
3985
|
+
* (plus the featureSets/update sent to the server), scope patterns, and
|
|
3986
|
+
* stateful-feature-set checkpoint registration.
|
|
3987
|
+
*
|
|
3988
|
+
* Called on initial connect (connectMcplServerInternal) AND after every
|
|
3989
|
+
* auto-reconnect. The 'close' handler drops the feature-set registration, so
|
|
3990
|
+
* without the reconnect re-run FeatureSetManager.validateInbound would throw
|
|
3991
|
+
* "Unknown server" forever — every push event and inference request from the
|
|
3992
|
+
* revived server silently rejected until a full host restart.
|
|
3993
|
+
*
|
|
3994
|
+
* CheckpointManager.registerFeatureSet is idempotent (no-op for a key that
|
|
3995
|
+
* already has a tree), so checkpoint trees preserved across a transient
|
|
3996
|
+
* disconnect are resumed, not reset.
|
|
3997
|
+
*/
|
|
3998
|
+
registerMcplServerFeatures(config, connection) {
|
|
3999
|
+
if (!connection.capabilities)
|
|
4000
|
+
return;
|
|
4001
|
+
const updateParams = this.featureSetManager.initializeServer(config.id, connection.capabilities, {
|
|
4002
|
+
enabledFeatureSets: config.enabledFeatureSets,
|
|
4003
|
+
disabledFeatureSets: config.disabledFeatureSets,
|
|
4004
|
+
});
|
|
4005
|
+
// Inform server which feature sets are enabled/disabled
|
|
4006
|
+
if (updateParams.enabled?.length || updateParams.disabled?.length) {
|
|
4007
|
+
connection.sendFeatureSetsUpdate(updateParams);
|
|
4008
|
+
}
|
|
4009
|
+
// Configure scope whitelist/blacklist patterns
|
|
4010
|
+
if (config.scopes) {
|
|
4011
|
+
this.scopeManager.configureAll(config.scopes);
|
|
4012
|
+
}
|
|
4013
|
+
// Register stateful feature sets with checkpoint manager (Step 8)
|
|
4014
|
+
if (this.checkpointManager) {
|
|
4015
|
+
const declared = this.featureSetManager.getDeclaredFeatureSets(config.id);
|
|
4016
|
+
if (declared) {
|
|
4017
|
+
for (const [fsName, fsDecl] of Object.entries(declared)) {
|
|
4018
|
+
if (fsDecl.rollback || fsDecl.hostState) {
|
|
4019
|
+
this.checkpointManager.registerFeatureSet(config.id, fsName, {
|
|
4020
|
+
hostState: fsDecl.hostState ?? false,
|
|
4021
|
+
rollback: fsDecl.rollback ?? false,
|
|
4022
|
+
});
|
|
4023
|
+
}
|
|
4024
|
+
}
|
|
4025
|
+
}
|
|
4026
|
+
}
|
|
4027
|
+
}
|
|
4028
|
+
// ==========================================================================
|
|
4029
|
+
// Runtime MCPL server lifecycle (agent-facing hot deploy/restart/unload)
|
|
4030
|
+
// ==========================================================================
|
|
4031
|
+
/**
|
|
4032
|
+
* Connect a new MCPL server at runtime. Refreshes the tool list and
|
|
4033
|
+
* notifies the agent of newly available tools. Throws if the MCPL
|
|
4034
|
+
* subsystem is not initialized, the id is already connected, or the
|
|
4035
|
+
* connection/handshake fails.
|
|
4036
|
+
*/
|
|
4037
|
+
async connectMcplServer(config) {
|
|
4038
|
+
// Lazily bring up the MCPL subsystem — a framework that started with zero
|
|
4039
|
+
// configured servers can still deploy its first one at runtime.
|
|
4040
|
+
if (!this.mcplServerRegistry || !this.mcplHostCapabilities) {
|
|
4041
|
+
await this.initializeMcpl([], this.mcplInferenceRoutingConfig ?? undefined);
|
|
4042
|
+
}
|
|
4043
|
+
// Same collision rules create() enforces at startup: the tool prefix must
|
|
4044
|
+
// not shadow a module name or another server's prefix.
|
|
4045
|
+
const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
|
|
4046
|
+
if (this.moduleRegistry.getAllModules().some(m => m.name === prefix)) {
|
|
4047
|
+
throw new Error(`MCPL server "${config.id}" toolPrefix "${prefix}" collides with module "${prefix}"`);
|
|
4048
|
+
}
|
|
4049
|
+
const prefixOwner = this.mcplPrefixMap.get(prefix);
|
|
4050
|
+
if (prefixOwner && prefixOwner !== config.id) {
|
|
4051
|
+
throw new Error(`MCPL server "${config.id}" toolPrefix "${prefix}" collides with server "${prefixOwner}"`);
|
|
4052
|
+
}
|
|
4053
|
+
const oldToolNames = new Set(this.mcplTools.map(t => t.name));
|
|
4054
|
+
await this.connectMcplServerInternal(config);
|
|
4055
|
+
await this.refreshMcplTools();
|
|
4056
|
+
this.emitMcplToolDiff(oldToolNames, config.id);
|
|
4057
|
+
}
|
|
4058
|
+
/**
|
|
4059
|
+
* Disconnect an MCPL server at runtime: close the connection, destroy its
|
|
4060
|
+
* feature-set and checkpoint state (permanent removal — unlike a transient
|
|
4061
|
+
* transport close, which preserves checkpoints for the reconnect), remove
|
|
4062
|
+
* its channels from the registry, drop routing entries, and refresh tools.
|
|
4063
|
+
* No-op-ish if the server is not connected (still clears routing state).
|
|
4064
|
+
*/
|
|
4065
|
+
async disconnectMcplServer(id) {
|
|
4066
|
+
if (!this.mcplServerRegistry) {
|
|
4067
|
+
throw new Error('MCPL subsystem is not initialized');
|
|
4068
|
+
}
|
|
4069
|
+
const config = this.mcplServerConfigs.get(id);
|
|
4070
|
+
const oldToolNames = new Set(this.mcplTools.map(t => t.name));
|
|
4071
|
+
await this.mcplServerRegistry.removeServer(id);
|
|
4072
|
+
this.channelRegistry?.removeServer(id);
|
|
4073
|
+
// Permanent removal: also destroy feature-set and checkpoint state
|
|
4074
|
+
// explicitly. The 'close' handler usually does this, but a connection that
|
|
4075
|
+
// already transiently closed (reconnect pending) emits no second 'close'
|
|
4076
|
+
// from close(), and the transient path deliberately preserves checkpoints.
|
|
4077
|
+
this.featureSetManager?.removeServer(id);
|
|
4078
|
+
this.checkpointManager?.removeServer(id);
|
|
4079
|
+
const prefix = config?.toolPrefix ?? `mcpl--${id}`;
|
|
4080
|
+
this.mcplPrefixMap.delete(prefix);
|
|
4081
|
+
this.mcplServerConfigs.delete(id);
|
|
4082
|
+
await this.refreshMcplTools();
|
|
4083
|
+
this.emitMcplToolDiff(oldToolNames, id);
|
|
4084
|
+
}
|
|
4085
|
+
/**
|
|
4086
|
+
* Restart an MCPL server at runtime: disconnect, then reconnect with the
|
|
4087
|
+
* same (or an updated) config. This actually respawns a stdio child —
|
|
4088
|
+
* unlike `reconnect: true`, which only retries after transport-level
|
|
4089
|
+
* failures. Throws if the server has no stored config and none is given.
|
|
4090
|
+
*/
|
|
4091
|
+
async restartMcplServer(id, newConfig) {
|
|
4092
|
+
const config = newConfig ?? this.mcplServerConfigs.get(id);
|
|
4093
|
+
if (!config) {
|
|
4094
|
+
throw new Error(`MCPL server "${id}" is not configured`);
|
|
4095
|
+
}
|
|
4096
|
+
await this.disconnectMcplServer(id);
|
|
4097
|
+
await this.connectMcplServer(config);
|
|
4098
|
+
}
|
|
4099
|
+
/**
|
|
4100
|
+
* List configured MCPL servers with live connection status and the number
|
|
4101
|
+
* of tools each currently contributes.
|
|
4102
|
+
*/
|
|
4103
|
+
listMcplServers() {
|
|
4104
|
+
const result = [];
|
|
4105
|
+
for (const [id, config] of this.mcplServerConfigs) {
|
|
4106
|
+
const prefix = config.toolPrefix ?? `mcpl--${id}`;
|
|
4107
|
+
const connection = this.mcplServerRegistry?.getServer(id) ?? null;
|
|
4108
|
+
result.push({
|
|
4109
|
+
id,
|
|
4110
|
+
connected: connection?.isConnected ?? false,
|
|
4111
|
+
toolPrefix: prefix,
|
|
4112
|
+
toolCount: this.mcplTools.filter(t => t.name.startsWith(`${prefix}--`)).length,
|
|
4113
|
+
command: config.command,
|
|
4114
|
+
url: config.url,
|
|
4115
|
+
});
|
|
4116
|
+
}
|
|
4117
|
+
return result;
|
|
4118
|
+
}
|
|
3069
4119
|
/**
|
|
3070
4120
|
* Wire event listeners on an MCPL server connection.
|
|
3071
4121
|
* Push events and inference requests are deferred to Steps 6/7.
|
|
@@ -3136,14 +4186,95 @@ export class AgentFramework {
|
|
|
3136
4186
|
connection.on('tools-list-changed', () => {
|
|
3137
4187
|
this.handleToolsListChanged(connection.id);
|
|
3138
4188
|
});
|
|
3139
|
-
//
|
|
3140
|
-
|
|
4189
|
+
// Re-establish full server registration on reconnect. The 'close' handler
|
|
4190
|
+
// removed the feature-set state, so pushes / inference requests from the
|
|
4191
|
+
// revived server would otherwise be rejected with "Unknown server" until a
|
|
4192
|
+
// host restart. Re-runs the same init as the initial connect path (the
|
|
4193
|
+
// fresh handshake refreshed connection.capabilities); checkpoint trees
|
|
4194
|
+
// preserved across the transient close are resumed by idempotent
|
|
4195
|
+
// registration. Then refresh tools (server may have different tools).
|
|
4196
|
+
connection.on('reconnect', (info) => {
|
|
4197
|
+
try {
|
|
4198
|
+
const config = this.mcplServerConfigs.get(connection.id);
|
|
4199
|
+
if (config) {
|
|
4200
|
+
this.registerMcplServerFeatures(config, connection);
|
|
4201
|
+
}
|
|
4202
|
+
}
|
|
4203
|
+
catch (error) {
|
|
4204
|
+
console.error(`MCPL server "${connection.id}" re-registration after reconnect failed:`, error instanceof Error ? error.message : error);
|
|
4205
|
+
}
|
|
3141
4206
|
this.handleToolsListChanged(connection.id);
|
|
4207
|
+
this.emitTrace({
|
|
4208
|
+
type: 'mcpl:server-reconnected',
|
|
4209
|
+
serverId: connection.id,
|
|
4210
|
+
attempts: info?.attempts ?? 0,
|
|
4211
|
+
});
|
|
4212
|
+
// Mirror the module:removed emitted on 'close' so module-lifecycle
|
|
4213
|
+
// consumers see the server come back, not just vanish.
|
|
4214
|
+
this.emitTrace({ type: 'module:added', moduleName: `mcpl:${connection.id}` });
|
|
4215
|
+
});
|
|
4216
|
+
// Surface connect/reconnect failures. Before these traces existed the
|
|
4217
|
+
// only receipt was a console.error on the host's own stderr — invisible
|
|
4218
|
+
// unless someone ssh'd in and read the process log.
|
|
4219
|
+
connection.on('connect-failed', (params) => {
|
|
4220
|
+
this.emitTrace({
|
|
4221
|
+
type: 'mcpl:server-connect-failed',
|
|
4222
|
+
serverId: connection.id,
|
|
4223
|
+
error: params.error,
|
|
4224
|
+
attempt: params.attempt,
|
|
4225
|
+
willRetry: connection.willReconnect,
|
|
4226
|
+
});
|
|
4227
|
+
});
|
|
4228
|
+
connection.on('reconnect-failed', (params) => {
|
|
4229
|
+
this.emitTrace({
|
|
4230
|
+
type: 'mcpl:server-connect-failed',
|
|
4231
|
+
serverId: connection.id,
|
|
4232
|
+
error: params.error,
|
|
4233
|
+
attempt: params.attempt,
|
|
4234
|
+
willRetry: connection.willReconnect,
|
|
4235
|
+
});
|
|
3142
4236
|
});
|
|
3143
|
-
//
|
|
3144
|
-
|
|
4237
|
+
// A late response to a tools/call that already timed out, carrying
|
|
4238
|
+
// stateful data the server advanced to. The host can't re-inject it (the
|
|
4239
|
+
// dispatch context is gone), but surfacing it makes the checkpoint-tree
|
|
4240
|
+
// divergence greppable instead of a silent drift into stale state.
|
|
4241
|
+
connection.on('orphaned-response', (info) => {
|
|
4242
|
+
this.emitTrace({
|
|
4243
|
+
type: 'mcpl:orphaned-response',
|
|
4244
|
+
serverId: connection.id,
|
|
4245
|
+
responseId: info.id,
|
|
4246
|
+
hadState: info.hadState,
|
|
4247
|
+
hadCheckpoint: info.hadCheckpoint,
|
|
4248
|
+
});
|
|
4249
|
+
});
|
|
4250
|
+
// Connection-level errors (e.g. child process 'error' after spawn).
|
|
4251
|
+
// Attaching this listener also keeps an unhandled EventEmitter 'error'
|
|
4252
|
+
// from crashing the host process.
|
|
4253
|
+
connection.on('error', (err) => {
|
|
4254
|
+
this.emitTrace({
|
|
4255
|
+
type: 'mcpl:server-error',
|
|
4256
|
+
serverId: connection.id,
|
|
4257
|
+
error: err.message,
|
|
4258
|
+
});
|
|
4259
|
+
});
|
|
4260
|
+
// Cleanup on disconnect. Feature-set state is always dropped (the server
|
|
4261
|
+
// can't be validated while down; reconnect re-registers it). Checkpoint
|
|
4262
|
+
// trees, however, are durable state: CheckpointManager.removeServer deletes
|
|
4263
|
+
// them AND persists the deletion to Chronicle. The close handler must
|
|
4264
|
+
// therefore NEVER destroy checkpoints — `willReconnect` is false on a clean
|
|
4265
|
+
// stop() too (reconnectEnabled is cleared before 'close' fires), so gating
|
|
4266
|
+
// on it would erase every checkpoint tree on an ordinary host restart while
|
|
4267
|
+
// a SIGKILL preserves them. Permanent removal is owned solely by
|
|
4268
|
+
// disconnectMcplServer, which deletes the trees explicitly.
|
|
4269
|
+
connection.on('close', (code, signal) => {
|
|
3145
4270
|
this.featureSetManager?.removeServer(connection.id);
|
|
3146
|
-
this.
|
|
4271
|
+
this.emitTrace({
|
|
4272
|
+
type: 'mcpl:server-closed',
|
|
4273
|
+
serverId: connection.id,
|
|
4274
|
+
code: code ?? null,
|
|
4275
|
+
signal: signal ?? null,
|
|
4276
|
+
willReconnect: connection.willReconnect,
|
|
4277
|
+
});
|
|
3147
4278
|
this.emitTrace({ type: 'module:removed', moduleName: `mcpl:${connection.id}` });
|
|
3148
4279
|
});
|
|
3149
4280
|
}
|
|
@@ -3262,15 +4393,21 @@ export class AgentFramework {
|
|
|
3262
4393
|
this.emitTrace({ type: 'tool:started', module: `mcpl:${serverId}`, tool: toolName, callId: call.id, input: call.input });
|
|
3263
4394
|
const startTime = Date.now();
|
|
3264
4395
|
const args = (call.input && typeof call.input === 'object') ? call.input : {};
|
|
3265
|
-
// Build state params for stateful tools (Step 8)
|
|
4396
|
+
// Build state params for stateful tools (Step 8).
|
|
4397
|
+
// Host-managed state is single-valued per server, so inject the host-managed
|
|
4398
|
+
// set's `state` for any call; otherwise fall back to a server-managed set's
|
|
4399
|
+
// opaque `checkpoint`. Never blindly pick "first stateful" — when a server
|
|
4400
|
+
// mixes host-managed (e.g. a feed) and server-managed (e.g. post/undo) sets,
|
|
4401
|
+
// that misattributes the feed's state.
|
|
3266
4402
|
let stateParams;
|
|
3267
4403
|
if (this.checkpointManager) {
|
|
3268
|
-
const
|
|
3269
|
-
if (
|
|
3270
|
-
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
|
|
4404
|
+
const hostFs = this.checkpointManager.getHostManagedFeatureSet(serverId);
|
|
4405
|
+
if (hostFs) {
|
|
4406
|
+
stateParams = { state: this.checkpointManager.getCurrentState(serverId, hostFs) };
|
|
4407
|
+
}
|
|
4408
|
+
else {
|
|
4409
|
+
const fs = this.checkpointManager.getStatefulFeatureSet(serverId);
|
|
4410
|
+
if (fs) {
|
|
3274
4411
|
const cp = this.checkpointManager.getCurrentCheckpoint(serverId, fs);
|
|
3275
4412
|
if (cp)
|
|
3276
4413
|
stateParams = { checkpoint: cp };
|
|
@@ -3281,10 +4418,17 @@ export class AgentFramework {
|
|
|
3281
4418
|
.then((result) => {
|
|
3282
4419
|
const durationMs = Date.now() - startTime;
|
|
3283
4420
|
this.emitTrace({ type: 'tool:completed', module: `mcpl:${serverId}`, tool: toolName, callId: call.id, durationMs });
|
|
3284
|
-
// Record checkpoint from stateful tool response (Step 8)
|
|
4421
|
+
// Record checkpoint from stateful tool response (Step 8).
|
|
4422
|
+
// Attribute to the set the server tagged (result.state.featureSet) first;
|
|
4423
|
+
// else the host-managed set (host-managed checkpoints carry data/patch);
|
|
4424
|
+
// else the single stateful set. Avoids recording a feed checkpoint onto a
|
|
4425
|
+
// server-managed set just because it registered first.
|
|
3285
4426
|
if (result.state && this.checkpointManager) {
|
|
3286
|
-
const
|
|
3287
|
-
|
|
4427
|
+
const tagged = result.state.featureSet;
|
|
4428
|
+
const fs = tagged
|
|
4429
|
+
?? this.checkpointManager.getHostManagedFeatureSet(serverId)
|
|
4430
|
+
?? this.checkpointManager.getStatefulFeatureSet(serverId);
|
|
4431
|
+
if (fs && this.checkpointManager.isStateful(serverId, fs)) {
|
|
3288
4432
|
this.checkpointManager.recordCheckpoint(serverId, fs, result.state);
|
|
3289
4433
|
}
|
|
3290
4434
|
}
|
|
@@ -3398,6 +4542,84 @@ export class AgentFramework {
|
|
|
3398
4542
|
});
|
|
3399
4543
|
});
|
|
3400
4544
|
}
|
|
4545
|
+
/**
|
|
4546
|
+
* Handle the synthesized `wake_add_rule` / `wake_remove_rule` tools: mutate
|
|
4547
|
+
* the hot-reloaded gate.json at runtime. Validation lives in the EventGate
|
|
4548
|
+
* (same path as gate.json load), so an invalid rule is surfaced as a tool
|
|
4549
|
+
* error and nothing is written.
|
|
4550
|
+
*/
|
|
4551
|
+
dispatchWakeRuleToolCall(agentName, call) {
|
|
4552
|
+
this.emitTrace({ type: 'tool:started', module: 'gate', tool: call.name, callId: call.id, input: call.input });
|
|
4553
|
+
const finish = (result) => {
|
|
4554
|
+
this.emitTrace({
|
|
4555
|
+
type: result.isError ? 'tool:failed' : 'tool:completed',
|
|
4556
|
+
module: 'gate', tool: call.name, callId: call.id, durationMs: 0,
|
|
4557
|
+
...(result.isError ? { error: result.error } : {}),
|
|
4558
|
+
});
|
|
4559
|
+
this.pushEvent({ type: 'tool-result', callId: call.id, agentName, moduleName: 'gate', result });
|
|
4560
|
+
};
|
|
4561
|
+
try {
|
|
4562
|
+
if (call.name === 'wake_remove_rule') {
|
|
4563
|
+
const input = (call.input ?? {});
|
|
4564
|
+
if (typeof input.name !== 'string' || !input.name) {
|
|
4565
|
+
finish({ success: false, error: 'wake_remove_rule: `name` (string) is required', isError: true });
|
|
4566
|
+
return;
|
|
4567
|
+
}
|
|
4568
|
+
const removed = this.removeGatePolicy(input.name);
|
|
4569
|
+
finish({
|
|
4570
|
+
success: true,
|
|
4571
|
+
data: { removed, name: input.name, policies: this.getGatePolicyNames() },
|
|
4572
|
+
});
|
|
4573
|
+
return;
|
|
4574
|
+
}
|
|
4575
|
+
// wake_add_rule — assemble the canonical behavior from the typed fields
|
|
4576
|
+
// (exactly one), then let the gate's own validator do the authoritative
|
|
4577
|
+
// range/shape checks.
|
|
4578
|
+
const input = (call.input ?? {});
|
|
4579
|
+
const behaviorSources = [
|
|
4580
|
+
input.behavior !== undefined ? 'behavior' : null,
|
|
4581
|
+
input.debounceMs !== undefined ? 'debounceMs' : null,
|
|
4582
|
+
input.rateLimit !== undefined ? 'rateLimit' : null,
|
|
4583
|
+
input.passiveSample !== undefined ? 'passiveSample' : null,
|
|
4584
|
+
].filter((s) => s !== null);
|
|
4585
|
+
if (behaviorSources.length === 0) {
|
|
4586
|
+
finish({
|
|
4587
|
+
success: false,
|
|
4588
|
+
error: 'wake_add_rule: specify exactly one behavior — `behavior` ("always"/"defer"/"skip"), `debounceMs`, `rateLimit`, or `passiveSample`.',
|
|
4589
|
+
isError: true,
|
|
4590
|
+
});
|
|
4591
|
+
return;
|
|
4592
|
+
}
|
|
4593
|
+
if (behaviorSources.length > 1) {
|
|
4594
|
+
finish({
|
|
4595
|
+
success: false,
|
|
4596
|
+
error: `wake_add_rule: give only one behavior, got [${behaviorSources.join(', ')}].`,
|
|
4597
|
+
isError: true,
|
|
4598
|
+
});
|
|
4599
|
+
return;
|
|
4600
|
+
}
|
|
4601
|
+
const behavior = input.debounceMs !== undefined ? { debounce: input.debounceMs }
|
|
4602
|
+
: input.rateLimit !== undefined ? { rate_limit: input.rateLimit }
|
|
4603
|
+
: input.passiveSample !== undefined ? { passive_sample: input.passiveSample }
|
|
4604
|
+
: input.behavior;
|
|
4605
|
+
const position = input.position === 'prepend' ? 'prepend' : 'append';
|
|
4606
|
+
const rawPolicy = {
|
|
4607
|
+
name: input.name,
|
|
4608
|
+
match: input.match ?? {},
|
|
4609
|
+
behavior,
|
|
4610
|
+
...(input.resets !== undefined ? { resets: input.resets } : {}),
|
|
4611
|
+
};
|
|
4612
|
+
const policy = this.addGatePolicy(rawPolicy, { position });
|
|
4613
|
+
finish({
|
|
4614
|
+
success: true,
|
|
4615
|
+
data: { added: policy.name, policies: this.getGatePolicyNames() },
|
|
4616
|
+
});
|
|
4617
|
+
}
|
|
4618
|
+
catch (err) {
|
|
4619
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
4620
|
+
finish({ success: false, error: msg, isError: true });
|
|
4621
|
+
}
|
|
4622
|
+
}
|
|
3401
4623
|
/**
|
|
3402
4624
|
* Handle the synthesized `sleep` / `wake` tools. `sleep` arms the gate's
|
|
3403
4625
|
* suppression window, optionally announces in the sticky channel, and ends
|
|
@@ -3421,7 +4643,7 @@ export class AgentFramework {
|
|
|
3421
4643
|
finish({ success: false, error: 'sleep: `seconds` must be a positive number', isError: true });
|
|
3422
4644
|
return;
|
|
3423
4645
|
}
|
|
3424
|
-
const { until } = gate.setSleep(seconds, input.message);
|
|
4646
|
+
const { until } = gate.setSleep(seconds, input.message, agentName);
|
|
3425
4647
|
const announce = input.announce !== false; // default true
|
|
3426
4648
|
const human = seconds >= 3600 ? `${(seconds / 3600).toFixed(1)}h`
|
|
3427
4649
|
: seconds >= 60 ? `${Math.round(seconds / 60)}m`
|
|
@@ -3437,6 +4659,42 @@ export class AgentFramework {
|
|
|
3437
4659
|
// endTurn: the agent stops here and goes idle for the duration.
|
|
3438
4660
|
finish({ success: true, data: { sleepingFor: human, until }, endTurn: true });
|
|
3439
4661
|
}
|
|
4662
|
+
/** Aggregate the event-tag vocabulary: reserved chat:* core + each connected
|
|
4663
|
+
* server's declared tag ontology + gate.js status. */
|
|
4664
|
+
buildEventTagsResult() {
|
|
4665
|
+
const servers = {};
|
|
4666
|
+
for (const conn of this.mcplServerRegistry?.getAllServers() ?? []) {
|
|
4667
|
+
const declared = this.featureSetManager?.getDeclaredFeatureSets(conn.id) ?? {};
|
|
4668
|
+
const sets = {};
|
|
4669
|
+
for (const [name, decl] of Object.entries(declared)) {
|
|
4670
|
+
if (decl.tagOntology)
|
|
4671
|
+
sets[name] = decl.tagOntology;
|
|
4672
|
+
}
|
|
4673
|
+
if (Object.keys(sets).length > 0)
|
|
4674
|
+
servers[conn.id] = sets;
|
|
4675
|
+
}
|
|
4676
|
+
return {
|
|
4677
|
+
core: AgentFramework.CHAT_CORE_TAGS,
|
|
4678
|
+
servers,
|
|
4679
|
+
gateScript: this.eventGate?.getStatus().script ?? null,
|
|
4680
|
+
hint: 'Use these in gate.json policies (match.tagsAny / tagsAll / tagsNone) or ' +
|
|
4681
|
+
'in gate.js. Unknown/undeclared tags are tolerated (open ontologies).',
|
|
4682
|
+
};
|
|
4683
|
+
}
|
|
4684
|
+
dispatchEventTagsToolCall(agentName, call) {
|
|
4685
|
+
this.emitTrace({ type: 'tool:started', module: 'gate', tool: call.name, callId: call.id, input: call.input });
|
|
4686
|
+
let result;
|
|
4687
|
+
try {
|
|
4688
|
+
result = { success: true, data: this.buildEventTagsResult() };
|
|
4689
|
+
this.emitTrace({ type: 'tool:completed', module: 'gate', tool: call.name, callId: call.id, durationMs: 0 });
|
|
4690
|
+
}
|
|
4691
|
+
catch (error) {
|
|
4692
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
4693
|
+
this.emitTrace({ type: 'tool:failed', module: 'gate', tool: call.name, callId: call.id, error: err.message });
|
|
4694
|
+
result = { success: false, error: err.message, isError: true };
|
|
4695
|
+
}
|
|
4696
|
+
this.pushEvent({ type: 'tool-result', callId: call.id, agentName, moduleName: 'gate', result });
|
|
4697
|
+
}
|
|
3440
4698
|
dispatchGateToolCall(agentName, call) {
|
|
3441
4699
|
this.emitTrace({ type: 'tool:started', module: 'gate', tool: call.name, callId: call.id, input: call.input });
|
|
3442
4700
|
const startTime = Date.now();
|
|
@@ -3468,7 +4726,13 @@ export class AgentFramework {
|
|
|
3468
4726
|
const inferenceId = `${agent.name}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
3469
4727
|
return {
|
|
3470
4728
|
inferenceId,
|
|
3471
|
-
|
|
4729
|
+
// Conversation identity = the agent (a trunk agent IS its own conversation;
|
|
4730
|
+
// forks get their own agent). The turn's channel LOCUS — the "proper
|
|
4731
|
+
// conversation tracking" this once flagged as a TODO — is now tracked
|
|
4732
|
+
// per-agent in `activeTriggerChannels` and surfaced to the agent via
|
|
4733
|
+
// buildChannelContext (channels.defaultOutgoing) below, so the agent is
|
|
4734
|
+
// told the same channel its speech will route to (item-3 redux).
|
|
4735
|
+
conversationId: agent.name,
|
|
3472
4736
|
turnIndex: 0, // Simplified; needs per-conversation counter TODO
|
|
3473
4737
|
userMessage: null, // Could extract from trigger context
|
|
3474
4738
|
model: {
|
|
@@ -3477,7 +4741,7 @@ export class AgentFramework {
|
|
|
3477
4741
|
contextWindow: 200000,
|
|
3478
4742
|
capabilities: ['tools'],
|
|
3479
4743
|
},
|
|
3480
|
-
channels: this.channelRegistry?.buildChannelContext(),
|
|
4744
|
+
channels: this.channelRegistry?.buildChannelContext(agent.name),
|
|
3481
4745
|
};
|
|
3482
4746
|
}
|
|
3483
4747
|
}
|