@animalabs/connectome-host 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/.env.example +20 -0
  2. package/.github/workflows/publish.yml +65 -0
  3. package/ARCHITECTURE.md +355 -0
  4. package/CHANGELOG.md +30 -0
  5. package/HEADLESS-FLEET-PLAN.md +330 -0
  6. package/README.md +189 -0
  7. package/UNIFIED-TREE-PLAN.md +242 -0
  8. package/bun.lock +288 -0
  9. package/docs/AGENT-MEMORY-GUIDE.md +214 -0
  10. package/docs/AGENT-ONBOARDING.md +541 -0
  11. package/docs/ATTENTION-AND-GATING.md +120 -0
  12. package/docs/DEPLOYMENTS.md +130 -0
  13. package/docs/DEV-ENVIRONMENT.md +215 -0
  14. package/docs/LIBRARY-PIPELINE.md +286 -0
  15. package/docs/LOCUS-ROUTING-DESIGN.md +115 -0
  16. package/docs/claudeai-evacuation.md +228 -0
  17. package/docs/debug-context-api.md +208 -0
  18. package/docs/webui-deployment.md +219 -0
  19. package/package.json +33 -0
  20. package/recipes/SETUP.md +308 -0
  21. package/recipes/TRIUMVIRATE-SETUP.md +467 -0
  22. package/recipes/claude-export-revive.json +19 -0
  23. package/recipes/clerk.json +157 -0
  24. package/recipes/knowledge-miner.json +174 -0
  25. package/recipes/knowledge-reviewer.json +76 -0
  26. package/recipes/mcpl-editor-test.json +38 -0
  27. package/recipes/prompts/transplant-addendum.md +17 -0
  28. package/recipes/triumvirate.json +63 -0
  29. package/recipes/webui-fleet-test.json +27 -0
  30. package/recipes/webui-test.json +16 -0
  31. package/recipes/zulip-miner.json +87 -0
  32. package/scripts/connectome-doctor +373 -0
  33. package/scripts/evacuator.ts +956 -0
  34. package/scripts/import-claudeai-export.ts +747 -0
  35. package/scripts/lib/line-reader.ts +53 -0
  36. package/scripts/test-historical-thinking.ts +148 -0
  37. package/scripts/warmup-session.ts +336 -0
  38. package/src/agent-name.ts +58 -0
  39. package/src/commands.ts +1074 -0
  40. package/src/headless.ts +528 -0
  41. package/src/index.ts +867 -0
  42. package/src/logging-adapter.ts +208 -0
  43. package/src/mcpl-config.ts +199 -0
  44. package/src/modules/activity-module.ts +270 -0
  45. package/src/modules/channel-mode-module.ts +260 -0
  46. package/src/modules/fleet-module.ts +1705 -0
  47. package/src/modules/fleet-types.ts +225 -0
  48. package/src/modules/lessons-module.ts +465 -0
  49. package/src/modules/mcpl-admin-module.ts +345 -0
  50. package/src/modules/retrieval-module.ts +327 -0
  51. package/src/modules/settings-module.ts +143 -0
  52. package/src/modules/subagent-module.ts +2074 -0
  53. package/src/modules/subscription-gc-module.ts +322 -0
  54. package/src/modules/time-module.ts +85 -0
  55. package/src/modules/tui-module.ts +76 -0
  56. package/src/modules/web-ui-curve-page.ts +249 -0
  57. package/src/modules/web-ui-module.ts +2595 -0
  58. package/src/recipe.ts +1003 -0
  59. package/src/session-manager.ts +278 -0
  60. package/src/state/agent-tree-reducer.ts +431 -0
  61. package/src/state/fleet-tree-aggregator.ts +195 -0
  62. package/src/strategies/frontdesk-strategy.ts +364 -0
  63. package/src/synesthete.ts +68 -0
  64. package/src/tui.ts +2200 -0
  65. package/src/types/bun-ffi.d.ts +6 -0
  66. package/src/web/protocol.ts +648 -0
  67. package/test/agent-name.test.ts +62 -0
  68. package/test/agent-tree-fleet-launch.test.ts +64 -0
  69. package/test/agent-tree-reducer-parity.test.ts +194 -0
  70. package/test/agent-tree-reducer.test.ts +443 -0
  71. package/test/agent-tree-rollup.test.ts +109 -0
  72. package/test/autobio-progress-snapshot.test.ts +40 -0
  73. package/test/claudeai-export-importer.test.ts +386 -0
  74. package/test/evacuator.test.ts +332 -0
  75. package/test/fleet-commands.test.ts +244 -0
  76. package/test/fleet-no-subfleets.test.ts +147 -0
  77. package/test/fleet-orchestration.test.ts +353 -0
  78. package/test/fleet-recipe.test.ts +124 -0
  79. package/test/fleet-route.test.ts +44 -0
  80. package/test/fleet-smoke.test.ts +479 -0
  81. package/test/fleet-subscribe-union-e2e.test.ts +123 -0
  82. package/test/fleet-subscribe-union.test.ts +85 -0
  83. package/test/fleet-tree-aggregator-e2e.test.ts +110 -0
  84. package/test/fleet-tree-aggregator.test.ts +215 -0
  85. package/test/frontdesk-strategy.test.ts +326 -0
  86. package/test/headless-describe.test.ts +182 -0
  87. package/test/headless-smoke.test.ts +190 -0
  88. package/test/logging-adapter.test.ts +87 -0
  89. package/test/mcpl-admin-module.test.ts +213 -0
  90. package/test/mcpl-agent-overlay.test.ts +126 -0
  91. package/test/mock-headless-child.ts +133 -0
  92. package/test/recipe-cache-ttl.test.ts +37 -0
  93. package/test/recipe-env.test.ts +157 -0
  94. package/test/recipe-path-resolution.test.ts +170 -0
  95. package/test/subagent-async-timeout.test.ts +381 -0
  96. package/test/subagent-fork-materialise.test.ts +241 -0
  97. package/test/subagent-peek-scoping.test.ts +180 -0
  98. package/test/subagent-peek-zombie.test.ts +148 -0
  99. package/test/subagent-reaper-in-flight.test.ts +229 -0
  100. package/test/subscription-gc-module.test.ts +136 -0
  101. package/test/warmup-handoff.test.ts +150 -0
  102. package/test/web-ui-bind.test.ts +51 -0
  103. package/test/web-ui-module.test.ts +246 -0
  104. package/test/web-ui-protocol.test.ts +0 -0
  105. package/tsconfig.json +14 -0
  106. package/web/bun.lock +357 -0
  107. package/web/index.html +12 -0
  108. package/web/package.json +24 -0
  109. package/web/src/App.tsx +1931 -0
  110. package/web/src/Context.tsx +158 -0
  111. package/web/src/ContextDocument.tsx +150 -0
  112. package/web/src/Files.tsx +271 -0
  113. package/web/src/Lessons.tsx +164 -0
  114. package/web/src/Mcpl.tsx +310 -0
  115. package/web/src/Stream.tsx +119 -0
  116. package/web/src/TreeSidebar.tsx +283 -0
  117. package/web/src/Usage.tsx +182 -0
  118. package/web/src/main.tsx +7 -0
  119. package/web/src/styles.css +26 -0
  120. package/web/src/tree.ts +268 -0
  121. package/web/src/wire.ts +120 -0
  122. package/web/tsconfig.json +21 -0
  123. package/web/vite.config.ts +32 -0
@@ -0,0 +1,345 @@
1
+ /**
2
+ * McplAdminModule — lets the agent deploy, restart, and unload its own MCPL
3
+ * servers at runtime, without a host restart.
4
+ *
5
+ * Tools:
6
+ * - mcpl_list → configured servers + live connection status
7
+ * - mcpl_deploy {id, ...} → add/update a server and hot-connect it
8
+ * - mcpl_restart {id} → kill + respawn a server (picks up rebuilt dist)
9
+ * - mcpl_unload {id} → disconnect a server and remove its tools
10
+ *
11
+ * Persistence model (agent overlay):
12
+ * Agent deployments are written to `mcpl-servers.agent.json` (cwd), which
13
+ * index.ts merges over the recipe/file server list at startup — so agent
14
+ * deployments survive host restarts without touching human-owned recipe
15
+ * files or mcpl-servers.json. Unloading a recipe/file server writes a
16
+ * `{disabled: true}` tombstone to the overlay; unloading an agent-deployed
17
+ * server just deletes its overlay entry.
18
+ *
19
+ * Security: enabling this module in a recipe grants the agent the ability to
20
+ * spawn arbitrary commands as the host user (mcpl_deploy). Recipe opt-in
21
+ * (`modules: { mcplAdmin: true }`) is the permission gate.
22
+ */
23
+
24
+ import type {
25
+ Module,
26
+ ModuleContext,
27
+ ProcessEvent,
28
+ ProcessState,
29
+ EventResponse,
30
+ ToolDefinition,
31
+ ToolCall,
32
+ ToolResult,
33
+ AgentFramework,
34
+ McplServerConfig,
35
+ } from '@animalabs/agent-framework';
36
+ import {
37
+ DEFAULT_CONFIG_PATH,
38
+ DEFAULT_AGENT_OVERLAY_PATH,
39
+ readMcplServersFile,
40
+ readAgentOverlay,
41
+ saveAgentOverlay,
42
+ resolveOverlayEntry,
43
+ type AgentOverlayEntry,
44
+ } from '../mcpl-config.js';
45
+
46
+ export interface McplAdminModuleConfig {
47
+ /** Path to the agent overlay file. Default: `mcpl-servers.agent.json` in cwd. */
48
+ overlayPath?: string;
49
+ /** Path to the human-owned server config file (read-only here). */
50
+ configPath?: string;
51
+ }
52
+
53
+ function ok(text: string): ToolResult {
54
+ return { success: true, data: text };
55
+ }
56
+
57
+ function fail(text: string): ToolResult {
58
+ return { success: false, error: text, isError: true };
59
+ }
60
+
61
+ export class McplAdminModule implements Module {
62
+ readonly name = 'mcpl-admin';
63
+
64
+ private framework: AgentFramework | null = null;
65
+ private overlayPath: string;
66
+ private configPath: string;
67
+
68
+ constructor(config?: McplAdminModuleConfig) {
69
+ this.overlayPath = config?.overlayPath ?? DEFAULT_AGENT_OVERLAY_PATH;
70
+ this.configPath = config?.configPath ?? DEFAULT_CONFIG_PATH;
71
+ }
72
+
73
+ /** Post-creation wiring (called from index.ts, mirrors ActivityModule.setFramework). */
74
+ setFramework(framework: AgentFramework): void {
75
+ this.framework = framework;
76
+ }
77
+
78
+ async start(_ctx: ModuleContext): Promise<void> {}
79
+
80
+ async stop(): Promise<void> {
81
+ this.framework = null;
82
+ }
83
+
84
+ getTools(): ToolDefinition[] {
85
+ return [
86
+ {
87
+ name: 'mcpl_list',
88
+ description:
89
+ 'List all MCPL servers: id, live connection status, tool count, command/url, ' +
90
+ 'and where each is defined (recipe/file vs your own agent overlay).',
91
+ inputSchema: { type: 'object', properties: {} },
92
+ },
93
+ {
94
+ name: 'mcpl_deploy',
95
+ description:
96
+ 'Deploy an MCPL server: persist it to your agent overlay (survives host ' +
97
+ 'restarts) and hot-connect it now — its tools become available immediately. ' +
98
+ 'If a server with this id is already running it is restarted with the new ' +
99
+ 'config. Provide either `command` (stdio, spawned as the host user) or `url` ' +
100
+ '(WebSocket). Relative ./ args resolve against the host working directory.',
101
+ inputSchema: {
102
+ type: 'object',
103
+ properties: {
104
+ id: { type: 'string', description: 'Unique server id (also the default tool prefix: mcpl--<id>).' },
105
+ command: { type: 'string', description: 'Executable to spawn (stdio transport). Mutually exclusive with url.' },
106
+ args: { type: 'array', items: { type: 'string' }, description: 'Arguments for the command.' },
107
+ env: { type: 'object', description: 'Environment variables for the spawned process.' },
108
+ url: { type: 'string', description: 'WebSocket URL (websocket transport). Mutually exclusive with command.' },
109
+ token: { type: 'string', description: 'Bearer token for WebSocket auth.' },
110
+ toolPrefix: { type: 'string', description: 'Tool namespace prefix. Default: mcpl--<id>.' },
111
+ reconnect: { type: 'boolean', description: 'Auto-reconnect on transport failure (default false). Note: does NOT respawn a crashed child — use mcpl_restart for that.' },
112
+ channelSubscription: {
113
+ type: 'string',
114
+ enum: ['auto', 'manual'],
115
+ description: "Channel auto-open policy: 'auto' (default) opens every channel the server registers; 'manual' opens none. For an allow-list, use channelAllowlist instead.",
116
+ },
117
+ channelAllowlist: {
118
+ type: 'array',
119
+ items: { type: 'string' },
120
+ description: 'Allow-list of channel ids to auto-open (overrides channelSubscription).',
121
+ },
122
+ enabledFeatureSets: { type: 'array', items: { type: 'string' } },
123
+ disabledFeatureSets: { type: 'array', items: { type: 'string' } },
124
+ enabledTools: { type: 'array', items: { type: 'string' }, description: 'Tool allow-list (bare names, * wildcard).' },
125
+ disabledTools: { type: 'array', items: { type: 'string' }, description: 'Tool deny-list; wins over enabledTools.' },
126
+ },
127
+ required: ['id'],
128
+ },
129
+ },
130
+ {
131
+ name: 'mcpl_restart',
132
+ description:
133
+ 'Restart an MCPL server: kill the process and respawn it with its current ' +
134
+ 'config. Use after rebuilding a server\'s dist, or to recover a crashed ' +
135
+ 'server (reconnect:true does not respawn dead children — this does). ' +
136
+ 'CAUTION: restarting the server that carries your active conversation ' +
137
+ '(e.g. discord) briefly interrupts your own message delivery; it reconnects ' +
138
+ 'within a few seconds.',
139
+ inputSchema: {
140
+ type: 'object',
141
+ properties: {
142
+ id: { type: 'string', description: 'Server id to restart.' },
143
+ },
144
+ required: ['id'],
145
+ },
146
+ },
147
+ {
148
+ name: 'mcpl_unload',
149
+ description:
150
+ 'Unload an MCPL server: disconnect it and remove its tools from your ' +
151
+ 'toolset. By default this persists (an unloaded recipe server stays ' +
152
+ 'unloaded after host restarts; an agent-deployed server is deleted from ' +
153
+ 'your overlay). Pass persist:false to unload for this session only. ' +
154
+ 'WARNING: unloading the server that carries your conversation (e.g. ' +
155
+ 'discord) cuts your own communication channel — you would need another ' +
156
+ 'route (or a human) to get it back.',
157
+ inputSchema: {
158
+ type: 'object',
159
+ properties: {
160
+ id: { type: 'string', description: 'Server id to unload.' },
161
+ persist: { type: 'boolean', description: 'Persist across host restarts (default true).' },
162
+ },
163
+ required: ['id'],
164
+ },
165
+ },
166
+ ];
167
+ }
168
+
169
+ async handleToolCall(call: ToolCall): Promise<ToolResult> {
170
+ const input = (call.input ?? {}) as Record<string, unknown>;
171
+ try {
172
+ switch (call.name) {
173
+ case 'mcpl_list':
174
+ return this.handleList();
175
+ case 'mcpl_deploy':
176
+ return await this.handleDeploy(input);
177
+ case 'mcpl_restart':
178
+ return await this.handleRestart(input);
179
+ case 'mcpl_unload':
180
+ return await this.handleUnload(input);
181
+ default:
182
+ return fail(`Unknown tool: ${call.name}`);
183
+ }
184
+ } catch (error) {
185
+ const err = error instanceof Error ? error : new Error(String(error));
186
+ return fail(`${call.name} failed: ${err.message}`);
187
+ }
188
+ }
189
+
190
+ async onProcess(_event: ProcessEvent, _state: ProcessState): Promise<EventResponse> {
191
+ return {};
192
+ }
193
+
194
+ // --------------------------------------------------------------------------
195
+ // Handlers
196
+ // --------------------------------------------------------------------------
197
+
198
+ private requireFramework(): AgentFramework {
199
+ if (!this.framework) {
200
+ throw new Error('mcpl-admin module is not wired to the framework yet');
201
+ }
202
+ return this.framework;
203
+ }
204
+
205
+ private handleList(): ToolResult {
206
+ const framework = this.requireFramework();
207
+ const live = framework.listMcplServers();
208
+ const overlay = readAgentOverlay(this.overlayPath);
209
+ const fileServers = readMcplServersFile(this.configPath);
210
+
211
+ const lines: string[] = [];
212
+ for (const s of live) {
213
+ const source = overlay[s.id] && !overlay[s.id]!.disabled
214
+ ? 'agent-overlay'
215
+ : s.id in fileServers ? 'file/recipe' : 'recipe';
216
+ const target = s.command ?? s.url ?? '?';
217
+ lines.push(
218
+ `${s.id}: ${s.connected ? 'CONNECTED' : 'DISCONNECTED'} — ${s.toolCount} tools, ` +
219
+ `prefix=${s.toolPrefix}, source=${source}, ${target}`,
220
+ );
221
+ }
222
+
223
+ // Tombstoned / overlay-only entries that aren't currently loaded
224
+ const liveIds = new Set(live.map(s => s.id));
225
+ for (const [id, entry] of Object.entries(overlay)) {
226
+ if (entry.disabled) {
227
+ lines.push(`${id}: UNLOADED (tombstoned in your overlay — redeploy with mcpl_deploy to restore)`);
228
+ } else if (!liveIds.has(id)) {
229
+ lines.push(`${id}: NOT LOADED (in your overlay but not connected — try mcpl_deploy again)`);
230
+ }
231
+ }
232
+
233
+ if (lines.length === 0) {
234
+ return ok('No MCPL servers configured. Use mcpl_deploy to add one.');
235
+ }
236
+ return ok(`MCPL servers (${lines.length}):\n` + lines.map(l => ` ${l}`).join('\n'));
237
+ }
238
+
239
+ private async handleDeploy(input: Record<string, unknown>): Promise<ToolResult> {
240
+ const framework = this.requireFramework();
241
+ const id = typeof input.id === 'string' ? input.id.trim() : '';
242
+ if (!id) return fail('mcpl_deploy requires a non-empty string `id`.');
243
+ if (!/^[a-zA-Z0-9_-]+$/.test(id)) {
244
+ return fail('`id` must match [a-zA-Z0-9_-]+ (it becomes part of tool names).');
245
+ }
246
+
247
+ const command = typeof input.command === 'string' ? input.command : undefined;
248
+ const url = typeof input.url === 'string' ? input.url : undefined;
249
+ if (!command && !url) return fail('mcpl_deploy requires either `command` (stdio) or `url` (websocket).');
250
+ if (command && url) return fail('`command` and `url` are mutually exclusive.');
251
+
252
+ // Build the overlay entry from recognized fields only.
253
+ const entry: AgentOverlayEntry = {};
254
+ if (command) entry.command = command;
255
+ if (url) { entry.url = url; entry.transport = 'websocket'; }
256
+ if (Array.isArray(input.args)) entry.args = input.args.map(String);
257
+ if (input.env && typeof input.env === 'object') entry.env = input.env as Record<string, string>;
258
+ if (typeof input.token === 'string') entry.token = input.token;
259
+ if (typeof input.toolPrefix === 'string') entry.toolPrefix = input.toolPrefix;
260
+ if (typeof input.reconnect === 'boolean') entry.reconnect = input.reconnect;
261
+ if (Array.isArray(input.channelAllowlist)) {
262
+ entry.channelSubscription = input.channelAllowlist.map(String);
263
+ } else if (input.channelSubscription === 'auto' || input.channelSubscription === 'manual') {
264
+ entry.channelSubscription = input.channelSubscription;
265
+ }
266
+ if (Array.isArray(input.enabledFeatureSets)) entry.enabledFeatureSets = input.enabledFeatureSets.map(String);
267
+ if (Array.isArray(input.disabledFeatureSets)) entry.disabledFeatureSets = input.disabledFeatureSets.map(String);
268
+ if (Array.isArray(input.enabledTools)) entry.enabledTools = input.enabledTools.map(String);
269
+ if (Array.isArray(input.disabledTools)) entry.disabledTools = input.disabledTools.map(String);
270
+
271
+ // Persist to the overlay first — a connect failure still leaves the entry
272
+ // in place so the agent can fix the server and mcpl_restart it.
273
+ const overlay = readAgentOverlay(this.overlayPath);
274
+ overlay[id] = entry;
275
+ saveAgentOverlay(this.overlayPath, overlay);
276
+
277
+ const config = resolveOverlayEntry(id, entry, this.overlayPath) as unknown as McplServerConfig;
278
+
279
+ const alreadyLoaded = framework.listMcplServers().some(s => s.id === id);
280
+ try {
281
+ if (alreadyLoaded) {
282
+ await framework.restartMcplServer(id, config);
283
+ } else {
284
+ await framework.connectMcplServer(config);
285
+ }
286
+ } catch (error) {
287
+ const err = error instanceof Error ? error : new Error(String(error));
288
+ return fail(
289
+ `Server "${id}" was saved to your overlay but failed to connect: ${err.message}. ` +
290
+ 'Fix the server (check command/path/build) and run mcpl_restart, or mcpl_unload to remove it.',
291
+ );
292
+ }
293
+
294
+ const status = framework.listMcplServers().find(s => s.id === id);
295
+ return ok(
296
+ `${alreadyLoaded ? 'Redeployed' : 'Deployed'} server "${id}" — connected, ` +
297
+ `${status?.toolCount ?? 0} tools under prefix ${status?.toolPrefix ?? `mcpl--${id}`}. ` +
298
+ 'Persisted to your agent overlay (survives host restarts).',
299
+ );
300
+ }
301
+
302
+ private async handleRestart(input: Record<string, unknown>): Promise<ToolResult> {
303
+ const framework = this.requireFramework();
304
+ const id = typeof input.id === 'string' ? input.id.trim() : '';
305
+ if (!id) return fail('mcpl_restart requires `id`.');
306
+
307
+ await framework.restartMcplServer(id);
308
+ const status = framework.listMcplServers().find(s => s.id === id);
309
+ return ok(
310
+ `Restarted server "${id}" — ${status?.connected ? 'connected' : 'NOT connected'}, ` +
311
+ `${status?.toolCount ?? 0} tools.`,
312
+ );
313
+ }
314
+
315
+ private async handleUnload(input: Record<string, unknown>): Promise<ToolResult> {
316
+ const framework = this.requireFramework();
317
+ const id = typeof input.id === 'string' ? input.id.trim() : '';
318
+ if (!id) return fail('mcpl_unload requires `id`.');
319
+ const persist = input.persist !== false;
320
+
321
+ const known = framework.listMcplServers().some(s => s.id === id);
322
+ const overlay = readAgentOverlay(this.overlayPath);
323
+ if (!known && !(id in overlay)) {
324
+ return fail(`Server "${id}" is not loaded and not in your overlay.`);
325
+ }
326
+
327
+ await framework.disconnectMcplServer(id);
328
+
329
+ let persistNote = 'Session-only: it will load again on the next host restart.';
330
+ if (persist) {
331
+ if (overlay[id] && !overlay[id]!.disabled) {
332
+ // Agent-deployed server: forget it entirely.
333
+ delete overlay[id];
334
+ persistNote = 'Removed from your agent overlay.';
335
+ } else {
336
+ // Recipe/file server: tombstone it so it stays unloaded across restarts.
337
+ overlay[id] = { disabled: true };
338
+ persistNote = 'Tombstoned in your overlay — it stays unloaded across host restarts; redeploy with mcpl_deploy to restore.';
339
+ }
340
+ saveAgentOverlay(this.overlayPath, overlay);
341
+ }
342
+
343
+ return ok(`Unloaded server "${id}" — its tools are gone from your toolset. ${persistNote}`);
344
+ }
345
+ }
@@ -0,0 +1,327 @@
1
+ /**
2
+ * RetrievalModule — LLM-as-retriever for semantic memory.
3
+ *
4
+ * Three-step retrieval pipeline running in gatherContext():
5
+ * 1. Flag concepts (Haiku): identify concepts being discussed that might
6
+ * benefit from background knowledge
7
+ * 2. Mechanical query: keyword-match against LessonsModule
8
+ * 3. Validate relevance (Haiku): filter to actually relevant lessons
9
+ *
10
+ * Steps 1 and 3 use cheap Haiku calls (~$0.001 each).
11
+ * Results are cached to avoid redundant calls on unchanged context.
12
+ */
13
+
14
+ import type {
15
+ Module,
16
+ ModuleContext,
17
+ ProcessState,
18
+ ProcessEvent,
19
+ EventResponse,
20
+ ToolDefinition,
21
+ ToolCall,
22
+ ToolResult,
23
+ } from '@animalabs/agent-framework';
24
+ import type { Membrane, NormalizedRequest } from '@animalabs/membrane';
25
+ import type { ContextInjection } from '@animalabs/context-manager';
26
+ import type { LessonsModule, Lesson } from './lessons-module.js';
27
+
28
+ // ---------------------------------------------------------------------------
29
+ // Configuration
30
+ // ---------------------------------------------------------------------------
31
+
32
+ export interface RetrievalModuleConfig {
33
+ /** Membrane instance for Haiku calls */
34
+ membrane: Membrane;
35
+ /** Model to use for retrieval calls (default: claude-haiku-4-5-20251001) */
36
+ retrievalModel?: string;
37
+ /** Max lessons to inject (default: 5) */
38
+ maxInjectedLessons?: number;
39
+ /** Minimum lesson confidence for injection (default: 0.3) */
40
+ minConfidence?: number;
41
+ }
42
+
43
+ // ---------------------------------------------------------------------------
44
+ // Prompts
45
+ // ---------------------------------------------------------------------------
46
+
47
+ const CONCEPT_FLAG_PROMPT = `You are a knowledge retrieval assistant. Given the recent conversation below, identify concepts, entities, topics, or themes that might benefit from background knowledge or lessons learned from previous research.
48
+
49
+ Return ONLY a JSON array of keyword strings, nothing else. Example: ["RFC process", "authentication", "team lead"]
50
+
51
+ If nothing would benefit from background knowledge, return an empty array: []`;
52
+
53
+ const RELEVANCE_VALIDATION_PROMPT = `You are a relevance filter. Given the current conversation context and a set of candidate knowledge lessons, determine which lessons are actually relevant to the current discussion.
54
+
55
+ Return ONLY a JSON array of lesson IDs that are relevant, nothing else. Example: ["a1b2c3d4", "e5f6g7h8"]
56
+
57
+ If none are relevant, return an empty array: []`;
58
+
59
+ // ---------------------------------------------------------------------------
60
+ // Module
61
+ // ---------------------------------------------------------------------------
62
+
63
+ export class RetrievalModule implements Module {
64
+ readonly name = 'retrieval';
65
+
66
+ private ctx: ModuleContext | null = null;
67
+ private config: RetrievalModuleConfig;
68
+ private lastContextHash = '';
69
+ private cachedInjections: ContextInjection[] = [];
70
+
71
+ constructor(config: RetrievalModuleConfig) {
72
+ this.config = config;
73
+ }
74
+
75
+ async start(ctx: ModuleContext): Promise<void> {
76
+ this.ctx = ctx;
77
+ }
78
+
79
+ async stop(): Promise<void> {
80
+ this.ctx = null;
81
+ }
82
+
83
+ getTools(): ToolDefinition[] {
84
+ // RetrievalModule is passive — no tools, only gatherContext
85
+ return [];
86
+ }
87
+
88
+ async handleToolCall(_call: ToolCall): Promise<ToolResult> {
89
+ return { success: false, error: 'RetrievalModule has no tools', isError: true };
90
+ }
91
+
92
+ async onProcess(_event: ProcessEvent, _state: ProcessState): Promise<EventResponse> {
93
+ return {};
94
+ }
95
+
96
+ /**
97
+ * Run the 3-step retrieval pipeline before each inference.
98
+ */
99
+ async gatherContext(_agentName: string): Promise<ContextInjection[]> {
100
+ if (!this.ctx) return [];
101
+
102
+ // Get the LessonsModule to query
103
+ const lessonsModule = this.ctx.getModule<LessonsModule>('lessons');
104
+ if (!lessonsModule) return [];
105
+
106
+ const lessons = lessonsModule.getLessons().filter(
107
+ l => !l.deprecated && l.confidence >= (this.config.minConfidence ?? 0.3)
108
+ );
109
+ if (lessons.length === 0) return [];
110
+
111
+ // Get recent context for concept extraction
112
+ const recentMessages = this.getRecentContext();
113
+ if (!recentMessages) return [];
114
+
115
+ // Check cache: if context hasn't changed, reuse cached results
116
+ const contextHash = this.hashContext(recentMessages);
117
+ if (contextHash === this.lastContextHash && this.cachedInjections.length > 0) {
118
+ return this.cachedInjections;
119
+ }
120
+
121
+ try {
122
+ // Step 1: Flag concepts (Haiku call)
123
+ const concepts = await this.flagConcepts(recentMessages);
124
+ if (concepts.length === 0) {
125
+ this.lastContextHash = contextHash;
126
+ this.cachedInjections = [];
127
+ return [];
128
+ }
129
+
130
+ // Step 2: Mechanical query (keyword matching)
131
+ const candidates = this.queryCandidates(concepts, lessons);
132
+ if (candidates.length === 0) {
133
+ this.lastContextHash = contextHash;
134
+ this.cachedInjections = [];
135
+ return [];
136
+ }
137
+
138
+ // Step 3: Validate relevance (Haiku call)
139
+ const relevant = await this.validateRelevance(recentMessages, candidates);
140
+
141
+ // Build injection
142
+ const maxLessons = this.config.maxInjectedLessons ?? 5;
143
+ const injected = relevant.slice(0, maxLessons);
144
+
145
+ if (injected.length === 0) {
146
+ this.lastContextHash = contextHash;
147
+ this.cachedInjections = [];
148
+ return [];
149
+ }
150
+
151
+ const text = injected
152
+ .map(l => `- [${(l.confidence * 100).toFixed(0)}%] ${l.content} (tags: ${l.tags.join(', ')})`)
153
+ .join('\n');
154
+
155
+ const injections: ContextInjection[] = [{
156
+ namespace: 'retrieval',
157
+ position: 'system',
158
+ content: [{ type: 'text', text: `## Retrieved Knowledge\n${text}` }],
159
+ }];
160
+
161
+ this.lastContextHash = contextHash;
162
+ this.cachedInjections = injections;
163
+ return injections;
164
+ } catch (err) {
165
+ // Fail open — don't block inference if retrieval fails
166
+ console.error('RetrievalModule: retrieval failed:', err);
167
+ return [];
168
+ }
169
+ }
170
+
171
+ // =========================================================================
172
+ // Pipeline Steps
173
+ // =========================================================================
174
+
175
+ /**
176
+ * Step 1: Use Haiku to identify concepts that might benefit from background knowledge.
177
+ */
178
+ private async flagConcepts(recentContext: string): Promise<string[]> {
179
+ const model = this.config.retrievalModel ?? 'claude-haiku-4-5-20251001';
180
+
181
+ const request: NormalizedRequest = {
182
+ messages: [
183
+ {
184
+ participant: 'user',
185
+ content: [{ type: 'text', text: `Recent conversation:\n${recentContext}` }],
186
+ },
187
+ ],
188
+ system: CONCEPT_FLAG_PROMPT,
189
+ config: { model, maxTokens: 500, temperature: 0 },
190
+ };
191
+
192
+ const response = await this.config.membrane.complete(request);
193
+ const text = response.content
194
+ .filter((b): b is { type: 'text'; text: string } => b.type === 'text')
195
+ .map(b => b.text)
196
+ .join('');
197
+
198
+ try {
199
+ const parsed = JSON.parse(text);
200
+ if (Array.isArray(parsed)) {
201
+ return parsed.filter((s): s is string => typeof s === 'string');
202
+ }
203
+ } catch {
204
+ // Try to extract from markdown code block
205
+ const match = text.match(/\[([^\]]*)\]/);
206
+ if (match) {
207
+ try {
208
+ return JSON.parse(`[${match[1]}]`);
209
+ } catch { /* fall through */ }
210
+ }
211
+ }
212
+ return [];
213
+ }
214
+
215
+ /**
216
+ * Step 2: Mechanical keyword matching against lessons.
217
+ */
218
+ private queryCandidates(concepts: string[], lessons: Lesson[]): Lesson[] {
219
+ const seen = new Set<string>();
220
+ const candidates: Lesson[] = [];
221
+
222
+ for (const concept of concepts) {
223
+ const keywords = concept.toLowerCase().split(/\s+/);
224
+
225
+ for (const lesson of lessons) {
226
+ if (seen.has(lesson.id)) continue;
227
+
228
+ const text = lesson.content.toLowerCase();
229
+ const tags = lesson.tags.map(t => t.toLowerCase());
230
+
231
+ // Match if any keyword appears in content or tags
232
+ const matches = keywords.some(kw =>
233
+ text.includes(kw) || tags.some(tag => tag.includes(kw))
234
+ );
235
+
236
+ if (matches) {
237
+ seen.add(lesson.id);
238
+ candidates.push(lesson);
239
+ }
240
+ }
241
+ }
242
+
243
+ // Sort by confidence
244
+ candidates.sort((a, b) => b.confidence - a.confidence);
245
+ return candidates.slice(0, 20); // Cap at 20 candidates for validation
246
+ }
247
+
248
+ /**
249
+ * Step 3: Use Haiku to validate which candidates are actually relevant.
250
+ */
251
+ private async validateRelevance(recentContext: string, candidates: Lesson[]): Promise<Lesson[]> {
252
+ if (candidates.length <= 3) {
253
+ // If only a few candidates, skip validation — they're probably all relevant
254
+ return candidates;
255
+ }
256
+
257
+ const model = this.config.retrievalModel ?? 'claude-haiku-4-5-20251001';
258
+
259
+ const candidateList = candidates.map(l =>
260
+ `[${l.id}] (${l.confidence.toFixed(2)}) ${l.content}`
261
+ ).join('\n');
262
+
263
+ const request: NormalizedRequest = {
264
+ messages: [
265
+ {
266
+ participant: 'user',
267
+ content: [{ type: 'text', text: `Current conversation:\n${recentContext}\n\nCandidate lessons:\n${candidateList}` }],
268
+ },
269
+ ],
270
+ system: RELEVANCE_VALIDATION_PROMPT,
271
+ config: { model, maxTokens: 500, temperature: 0 },
272
+ };
273
+
274
+ const response = await this.config.membrane.complete(request);
275
+ const text = response.content
276
+ .filter((b): b is { type: 'text'; text: string } => b.type === 'text')
277
+ .map(b => b.text)
278
+ .join('');
279
+
280
+ try {
281
+ const parsed = JSON.parse(text);
282
+ if (Array.isArray(parsed)) {
283
+ const relevantIds = new Set(parsed.filter((s): s is string => typeof s === 'string'));
284
+ return candidates.filter(l => relevantIds.has(l.id));
285
+ }
286
+ } catch {
287
+ // On parse failure, return top 5 by confidence (fail open)
288
+ return candidates.slice(0, 5);
289
+ }
290
+ return candidates.slice(0, 5);
291
+ }
292
+
293
+ // =========================================================================
294
+ // Helpers
295
+ // =========================================================================
296
+
297
+ private getRecentContext(): string | null {
298
+ if (!this.ctx) return null;
299
+
300
+ // Get the last few messages from the conversation
301
+ const { messages } = this.ctx.queryMessages({});
302
+ if (messages.length === 0) return null;
303
+
304
+ // Take the last 10 messages for context
305
+ const recent = messages.slice(-10);
306
+ return recent
307
+ .map(m => {
308
+ const text = m.content
309
+ .filter((b): b is { type: 'text'; text: string } => b.type === 'text')
310
+ .map(b => b.text)
311
+ .join('\n');
312
+ return `${m.participant}: ${text}`;
313
+ })
314
+ .join('\n\n');
315
+ }
316
+
317
+ private hashContext(text: string): string {
318
+ // Simple hash for cache invalidation
319
+ let hash = 0;
320
+ for (let i = 0; i < text.length; i++) {
321
+ const chr = text.charCodeAt(i);
322
+ hash = ((hash << 5) - hash) + chr;
323
+ hash |= 0;
324
+ }
325
+ return hash.toString(36);
326
+ }
327
+ }