heurchain 1.4.1 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/server.js DELETED
@@ -1,1364 +0,0 @@
1
- const express = require("express");
2
- const { createClient } = require("redis");
3
- const cors = require("cors");
4
- const fs = require("fs").promises;
5
- const path = require("path");
6
- const { McpServer } = require("@modelcontextprotocol/sdk/server/mcp.js");
7
- const { SSEServerTransport } = require("@modelcontextprotocol/sdk/server/sse.js");
8
- const { z } = require("zod");
9
-
10
- const app = express();
11
- app.use(cors());
12
-
13
- const redis = createClient({ url: process.env.REDIS_URL || "redis://localhost:6379" });
14
- redis.on("error", (err) => console.error("Redis error:", err));
15
-
16
- const OBSIDIAN_VAULT_PATH = path.resolve(process.env.OBSIDIAN_VAULT_PATH || "/opt/obsidian-vault");
17
- const VAULT_ROOT = OBSIDIAN_VAULT_PATH + path.sep;
18
-
19
- const BUFFER_CAPACITY = parseInt(process.env.BUFFER_CAPACITY, 10) || 50;
20
- const BUFFER_FLUSH_MS = parseInt(process.env.BUFFER_FLUSH_MS, 10) || 5 * 60 * 1000; // 5 min
21
- const SESSION_CONTEXT_TOKENS = parseInt(process.env.SESSION_CONTEXT_TOKENS, 10) || 400;
22
- const PROC_KEY_RE = /^[a-zA-Z0-9_\-\.]+$/; // safe proc key segment
23
-
24
- // M2: Ollama-first compressor provider chain
25
- const COMPRESSOR_PROVIDER = process.env.COMPRESSOR_PROVIDER || (process.env.OLLAMA_BASE_URL ? "ollama" : process.env.ANTHROPIC_API_KEY ? "anthropic" : "none");
26
- const COMPRESSOR_OLLAMA_MODEL = process.env.COMPRESSOR_OLLAMA_MODEL || "qwen2.5:1.5b";
27
- const COMPRESSOR_ANTHROPIC_MODEL = process.env.COMPRESSOR_MODEL || "claude-haiku-4-5-20251001";
28
- const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL || "http://localhost:11434";
29
- const COMPRESSOR_API_KEY = process.env.ANTHROPIC_API_KEY || process.env.COMPRESSOR_API_KEY || null;
30
-
31
- // M3: Proactive memory briefing worker
32
- const BRIEFING_ENABLED = process.env.BRIEFING_ENABLED !== "false";
33
- const BRIEFING_INTERVAL_MS = parseInt(process.env.BRIEFING_INTERVAL_MS, 10) || 24 * 60 * 60 * 1000;
34
- const BRIEFING_MAX_AGE_MS = parseInt(process.env.BRIEFING_MAX_AGE_HOURS, 10) * 60 * 60 * 1000 || 6 * 60 * 60 * 1000;
35
- const BRIEFING_MAX_ITEMS = parseInt(process.env.BRIEFING_MAX_ITEMS, 10) || 5;
36
- const BRIEFING_INSTRUCTIONS = process.env.BRIEFING_INSTRUCTIONS || "";
37
-
38
- // node-redis scanIterator yields pages (arrays) in some versions and individual
39
- // keys in others. scanKeys normalizes to individual key strings in all cases.
40
- async function* scanKeys(pattern, type) {
41
- const opts = { MATCH: pattern, COUNT: 200 };
42
- if (type) opts.TYPE = type;
43
- for await (const item of redis.scanIterator(opts)) {
44
- if (Array.isArray(item)) { for (const k of item) yield k; }
45
- else yield item;
46
- }
47
- }
48
-
49
- // ---------------------------------------------------------------------------
50
- // Auth middleware — set MCP_AUTH_TOKEN env var to enforce bearer token auth.
51
- // If unset, access is unrestricted (warn loudly).
52
- // ---------------------------------------------------------------------------
53
-
54
- const MCP_AUTH_TOKEN = process.env.MCP_AUTH_TOKEN || "";
55
- if (!MCP_AUTH_TOKEN) {
56
- console.warn("⚠️ SECURITY WARNING: MCP_AUTH_TOKEN is not set. All endpoints are unauthenticated. Set this env var to a secret token.");
57
- }
58
-
59
- function requireAuth(req, res, next) {
60
- if (!MCP_AUTH_TOKEN) return next(); // auth disabled — allow (warned at startup)
61
- const auth = req.headers.authorization || "";
62
- if (!auth.startsWith("Bearer ") || auth.slice(7) !== MCP_AUTH_TOKEN) {
63
- return res.status(401).json({ error: "Unauthorized" });
64
- }
65
- next();
66
- }
67
-
68
- // ---------------------------------------------------------------------------
69
- // Key validation & path resolution — #1 path traversal fix
70
- // ---------------------------------------------------------------------------
71
-
72
- const SAFE_KEY_SEGMENT = /^[a-zA-Z0-9_\-\.]+$/;
73
-
74
- function validateKey(key) {
75
- if (typeof key !== "string" || key.length === 0 || key.length > 512)
76
- throw new Error("invalid key: must be a non-empty string ≤ 512 chars");
77
- if (/[\\/\x00]/.test(key))
78
- throw new Error("invalid key: contains illegal characters");
79
- const segments = key.split(":");
80
- for (const seg of segments) {
81
- if (seg === "" || seg === "." || seg === "..")
82
- throw new Error("invalid key: empty or dot segment");
83
- if (!SAFE_KEY_SEGMENT.test(seg))
84
- throw new Error(`invalid key: segment '${seg}' contains illegal characters`);
85
- }
86
- }
87
-
88
- function keyToObsidianPath(key) {
89
- validateKey(key);
90
- const relative = key.replace(/:/g, path.sep) + ".md";
91
- const resolved = path.resolve(OBSIDIAN_VAULT_PATH, relative);
92
- if (resolved !== OBSIDIAN_VAULT_PATH && !resolved.startsWith(VAULT_ROOT))
93
- throw new Error("invalid key: path escapes vault");
94
- return resolved;
95
- }
96
-
97
- // ---------------------------------------------------------------------------
98
- // BM25 Index — fixes: drop tokens array (#13), O(N²) rebuild (#5),
99
- // pending-ops queue to survive rebuild (#4), tier validation (#15)
100
- // ---------------------------------------------------------------------------
101
-
102
- const STOPWORDS = new Set([
103
- "a","an","the","and","or","but","in","on","at","to","for","of","with",
104
- "by","from","as","is","was","are","were","be","been","being","have",
105
- "has","had","do","does","did","will","would","could","should","may",
106
- "might","shall","can","not","no","nor","so","yet","both","either",
107
- "than","then","when","where","which","who","that","this","these","those",
108
- "it","its","i","me","my","we","our","you","your","he","she","they","them",
109
- "his","her","their","what","how","all","each","any","more","also","just",
110
- "if","up","out","about","into","through","over","after","before","between"
111
- ]);
112
-
113
- const VALID_TIERS = new Set(["all", "longterm", "working"]);
114
-
115
- const NOISE_KEY_PATTERNS = [/session-summary/, /agent-end/, /heartbeat/];
116
- const NOISE_CONTENT_PATTERNS = [
117
- /^\d+ user turns,\s*\d+ total messages/i,
118
- /^openclaw session on [a-z0-9-]+:/i,
119
- /^session ended\./i
120
- ];
121
-
122
- function isSessionNoise(key, content) {
123
- if (NOISE_KEY_PATTERNS.some(p => p.test(key))) return true;
124
- return NOISE_CONTENT_PATTERNS.some(p => p.test((content || "").trim()));
125
- }
126
-
127
- class BM25Index {
128
- constructor(k1 = 1.5, b = 0.75) {
129
- this.k1 = k1;
130
- this.b = b;
131
- // doc entry: { tf: Map<term,count>, dl: number, content: string, tier: string, updatedAt: string|null }
132
- this.docs = new Map();
133
- this.df = new Map(); // term -> document frequency
134
- this.avgLen = 0;
135
- this.N = 0;
136
- }
137
-
138
- tokenize(text) {
139
- return (text || "")
140
- .toLowerCase()
141
- .replace(/[^a-z0-9\s]/g, " ")
142
- .split(/\s+/)
143
- .filter(t => t.length > 1 && !STOPWORDS.has(t));
144
- }
145
-
146
- // Single-doc add — used for incremental updates; calls _recalcStats each time
147
- add(key, content, tier, updatedAt) {
148
- if (this.docs.has(key)) this._removeFromIndex(key);
149
- const tokens = this.tokenize(content);
150
- const tf = new Map();
151
- for (const t of tokens) tf.set(t, (tf.get(t) || 0) + 1);
152
- this.docs.set(key, { tf, dl: tokens.length, content, tier, updatedAt });
153
- for (const term of tf.keys()) this.df.set(term, (this.df.get(term) || 0) + 1);
154
- this._recalcStats();
155
- }
156
-
157
- // Bulk add — O(N log N) instead of O(N²); calls _recalcStats once at end
158
- bulkAdd(entries) {
159
- for (const { key, content, tier, updatedAt } of entries) {
160
- if (this.docs.has(key)) this._removeFromIndex(key);
161
- const tokens = this.tokenize(content);
162
- const tf = new Map();
163
- for (const t of tokens) tf.set(t, (tf.get(t) || 0) + 1);
164
- this.docs.set(key, { tf, dl: tokens.length, content, tier, updatedAt });
165
- for (const term of tf.keys()) this.df.set(term, (this.df.get(term) || 0) + 1);
166
- }
167
- this._recalcStats();
168
- }
169
-
170
- remove(key) {
171
- if (!this.docs.has(key)) return;
172
- this._removeFromIndex(key);
173
- this._recalcStats();
174
- }
175
-
176
- _removeFromIndex(key) {
177
- const doc = this.docs.get(key);
178
- if (!doc) return;
179
- for (const term of doc.tf.keys()) {
180
- const c = this.df.get(term) || 0;
181
- if (c <= 1) this.df.delete(term);
182
- else this.df.set(term, c - 1);
183
- }
184
- this.docs.delete(key);
185
- }
186
-
187
- _recalcStats() {
188
- this.N = this.docs.size;
189
- if (this.N === 0) { this.avgLen = 0; return; }
190
- let total = 0;
191
- for (const doc of this.docs.values()) total += doc.dl;
192
- this.avgLen = total / this.N;
193
- }
194
-
195
- _scoreDoc(queryTokens, docKey) {
196
- const doc = this.docs.get(docKey);
197
- if (!doc) return 0;
198
- const lenNorm = this.avgLen > 0 ? doc.dl / this.avgLen : 1;
199
- let score = 0;
200
- for (const term of queryTokens) {
201
- const tf = doc.tf.get(term) || 0;
202
- if (tf === 0) continue;
203
- const df = this.df.get(term) || 0;
204
- const idf = Math.log((this.N - df + 0.5) / (df + 0.5) + 1);
205
- const tfn = (tf * (this.k1 + 1)) / (tf + this.k1 * (1 - this.b + this.b * lenNorm));
206
- score += idf * tfn;
207
- }
208
- return score;
209
- }
210
-
211
- search(query, { limit = 10, tier = "all", minScore = 0, noiseFilter = true, namespace = null } = {}) {
212
- const safeTier = VALID_TIERS.has(tier) ? tier : "all";
213
- const safeLimit = Math.max(1, Math.min(Number.isFinite(limit) ? Math.floor(limit) : 10, 50));
214
- const queryTokens = this.tokenize(query);
215
- if (queryTokens.length === 0) return [];
216
- const results = [];
217
- for (const [key, doc] of this.docs.entries()) {
218
- if (safeTier !== "all" && doc.tier !== safeTier) continue;
219
- if (namespace && !key.startsWith(namespace + ":")) continue;
220
- if (noiseFilter && isSessionNoise(key, doc.content)) continue;
221
- const score = this._scoreDoc(queryTokens, key);
222
- if (score > minScore) results.push({ key, score, tier: doc.tier, content: doc.content, updated_at: doc.updatedAt });
223
- }
224
- return results.sort((a, b) => b.score - a.score).slice(0, safeLimit);
225
- }
226
-
227
- get size() { return this.N; }
228
- }
229
-
230
- const bm25 = new BM25Index();
231
-
232
- // ---------------------------------------------------------------------------
233
- // Session buffer — ambient turn capture without explicit agent store calls.
234
- // Agents POST to /api/buffer fire-and-forget; buffer auto-flushes to Redis.
235
- // ---------------------------------------------------------------------------
236
-
237
- const sessionBuffers = new Map(); // session_id -> [{ role, content, agent, ts }]
238
-
239
- async function flushBuffer(sessionId) {
240
- const buf = sessionBuffers.get(sessionId);
241
- if (!buf || buf.length === 0) return;
242
- const key = `session:${sessionId}:buffer`;
243
- try {
244
- const existing = await redis.get(key);
245
- const prev = existing ? JSON.parse(existing) : [];
246
- await redis.setEx(key, 7 * 24 * 60 * 60, JSON.stringify([...prev, ...buf]));
247
- sessionBuffers.set(sessionId, []);
248
- } catch (e) {
249
- console.error(`Buffer flush error session ${sessionId}:`, e.message);
250
- }
251
- }
252
-
253
- function bufferAppend(sessionId, agentName, role, content) {
254
- if (!sessionBuffers.has(sessionId)) sessionBuffers.set(sessionId, []);
255
- const buf = sessionBuffers.get(sessionId);
256
- buf.push({ role: role || "user", content, agent: agentName || "unknown", ts: new Date().toISOString() });
257
- if (buf.length >= BUFFER_CAPACITY) flushBuffer(sessionId).catch(() => {});
258
- }
259
-
260
- // ---------------------------------------------------------------------------
261
- // Procedural memory — stable agent preferences/style knobs, no TTL/decay.
262
- // Redis namespace: proc:agent:{name}:{key}
263
- // ---------------------------------------------------------------------------
264
-
265
- function procKey(agentName, key) {
266
- if (!PROC_KEY_RE.test(agentName)) throw new Error("invalid agent_name");
267
- if (!PROC_KEY_RE.test(key)) throw new Error("invalid proc key");
268
- return `proc:agent:${agentName}:${key}`;
269
- }
270
-
271
- async function procGet(agentName) {
272
- const prefix = `proc:agent:${agentName}:`;
273
- const result = {};
274
- for await (const k of scanKeys(`${prefix}*`, "string")) {
275
- const val = await redis.get(k);
276
- if (!val) continue;
277
- const shortKey = k.slice(prefix.length);
278
- try { result[shortKey] = JSON.parse(val); } catch { result[shortKey] = val; }
279
- }
280
- return result;
281
- }
282
-
283
- // ---------------------------------------------------------------------------
284
- // M2: Ollama-first compressor provider chain
285
- // ---------------------------------------------------------------------------
286
-
287
- async function runCompressor(content, task = "cue") {
288
- const truncated = String(content).slice(0, 1200);
289
-
290
- const PROMPTS = {
291
- cue: `Compress this note into 5-15 tokens that would uniquely retrieve it later.\nPrioritize: product codes, proper nouns, specific numbers, technical terms.\nAvoid: generic descriptions, function words, obvious categories.\nOutput only the compressed tokens, nothing else.\n\nNote:\n${truncated}\n\nCompressed:`,
292
- briefing: `Based on these memory entries, generate 3-5 specific memory reminders for an AI agent starting a new session.\nEach reminder must be concrete and specific (not generic).\nReturn ONLY a JSON array of short strings, max 15 words each. No explanation.\n\nEntries:\n${truncated}\n\nJSON array:`,
293
- };
294
-
295
- const prompt = PROMPTS[task] || PROMPTS.cue;
296
-
297
- if (COMPRESSOR_PROVIDER === "ollama") {
298
- try {
299
- const res = await fetch(`${OLLAMA_BASE_URL}/api/generate`, {
300
- method: "POST",
301
- headers: { "Content-Type": "application/json" },
302
- body: JSON.stringify({ model: COMPRESSOR_OLLAMA_MODEL, prompt, stream: false, options: { temperature: 0.1 } }),
303
- signal: AbortSignal.timeout(30000),
304
- });
305
- if (!res.ok) throw new Error(`Ollama ${res.status}`);
306
- const j = await res.json();
307
- return j.response?.trim() ?? null;
308
- } catch (e) {
309
- console.error("[compressor] ollama failed:", e.message);
310
- return null;
311
- }
312
- }
313
-
314
- if (COMPRESSOR_PROVIDER === "anthropic" && COMPRESSOR_API_KEY) {
315
- try {
316
- const { default: Anthropic } = await import("@anthropic-ai/sdk");
317
- const client = new Anthropic({ apiKey: COMPRESSOR_API_KEY });
318
- const msg = await client.messages.create({
319
- model: COMPRESSOR_ANTHROPIC_MODEL,
320
- max_tokens: 128,
321
- temperature: 0.1,
322
- messages: [{ role: "user", content: prompt }],
323
- });
324
- return msg.content[0]?.text?.trim() ?? null;
325
- } catch (e) {
326
- console.error("[compressor] anthropic failed:", e.message);
327
- return null;
328
- }
329
- }
330
-
331
- return null; // COMPRESSOR_PROVIDER=none or no credentials
332
- }
333
-
334
- // ---------------------------------------------------------------------------
335
- // M3: Proactive memory briefing worker
336
- // ---------------------------------------------------------------------------
337
-
338
- function briefingKey(agentName) { return `briefing:${agentName}:latest`; }
339
- function briefingTsKey(agentName) { return `briefing:${agentName}:generated_at`; }
340
-
341
- async function discoverAgents() {
342
- const agents = new Set();
343
- for await (const k of scanKeys("proc:agent:*")) {
344
- // key pattern: proc:agent:{name}:{field}
345
- const parts = k.split(":");
346
- if (parts.length >= 3) agents.add(parts[2]);
347
- }
348
- return [...agents];
349
- }
350
-
351
- async function generateBriefing(agentName) {
352
- // Check opt-out
353
- const disabled = await redis.get(procKey(agentName, "briefing_disabled"));
354
- if (disabled === "true") return null;
355
-
356
- // Collect proc entries
357
- const procEntries = [];
358
- for await (const k of scanKeys(`proc:agent:${agentName}:*`)) {
359
- const field = k.split(":").slice(3).join(":");
360
- // Skip internal briefing control keys
361
- if (["briefing_disabled", "briefing_interval_ms", "briefing_instructions"].includes(field)) continue;
362
- const val = await redis.get(k);
363
- if (val) procEntries.push(`${field}: ${val}`);
364
- }
365
-
366
- // Collect last session buffer (most recent flushed session)
367
- const sessionTurns = [];
368
- const sessionKeys = [];
369
- for await (const k of scanKeys("session:*:buffer")) sessionKeys.push(k);
370
- if (sessionKeys.length) {
371
- // Sort by key name descending to get most recent
372
- sessionKeys.sort().reverse();
373
- try {
374
- const raw = await redis.get(sessionKeys[0]);
375
- const turns = JSON.parse(raw || "[]");
376
- for (const t of turns.slice(-6)) {
377
- if (t.content) sessionTurns.push(String(t.content).slice(0, 120));
378
- }
379
- } catch {}
380
- }
381
-
382
- const allEntries = [...procEntries, ...sessionTurns];
383
- if (!allEntries.length) return null;
384
-
385
- // Format proc entries and recent session turns directly as HeurChain items.
386
- // The receiving agent decides whether to surface them, query deeper, or synthesize.
387
- const items = [];
388
- for (const entry of procEntries.slice(0, 3)) {
389
- items.push(`HeurChain: ${entry}`);
390
- }
391
- for (const turn of sessionTurns.slice(0, 2)) {
392
- items.push(`HeurChain: ${turn.slice(0, 100)}`);
393
- }
394
- return items.slice(0, BRIEFING_MAX_ITEMS);
395
- }
396
-
397
- let isBriefing = false;
398
-
399
- async function runProactiveBriefings() {
400
- if (!BRIEFING_ENABLED || isBriefing) return;
401
- isBriefing = true;
402
- try {
403
- const agents = await discoverAgents();
404
- for (const agentName of agents) {
405
- try {
406
- // Check per-agent interval override
407
- const intervalOverride = await redis.get(procKey(agentName, "briefing_interval_ms"));
408
- const agentInterval = intervalOverride ? parseInt(intervalOverride, 10) : BRIEFING_INTERVAL_MS;
409
-
410
- // Check if briefing is still fresh for this agent's interval
411
- const tsRaw = await redis.get(briefingTsKey(agentName));
412
- if (tsRaw) {
413
- const age = Date.now() - parseInt(tsRaw, 10);
414
- if (age < agentInterval) continue; // not yet due
415
- }
416
-
417
- const items = await generateBriefing(agentName);
418
- if (items && items.length) {
419
- await redis.setEx(briefingKey(agentName), 24 * 60 * 60, JSON.stringify(items));
420
- await redis.set(briefingTsKey(agentName), String(Date.now()));
421
- console.log(`[briefing] agent=${agentName} items=${items.length}`);
422
-
423
- // Optional webhook push — agent decides what to do with the reminders
424
- const webhookUrl = await redis.get(procKey(agentName, "briefing_webhook_url"));
425
- if (webhookUrl) {
426
- fetch(webhookUrl, {
427
- method: "POST",
428
- headers: { "Content-Type": "application/json" },
429
- body: JSON.stringify({
430
- agent: agentName,
431
- timestamp: Date.now(),
432
- items,
433
- }),
434
- signal: AbortSignal.timeout(10000),
435
- }).catch(e => console.error(`[briefing] webhook failed for ${agentName}:`, e.message));
436
- }
437
- }
438
- } catch (e) {
439
- console.error(`[briefing] failed for agent ${agentName}:`, e.message);
440
- }
441
- }
442
- } finally {
443
- isBriefing = false;
444
- }
445
- }
446
-
447
- // Pending-ops queue: writes/removes that occur during a rebuild are applied
448
- // after the swap so they aren't silently discarded — fix #4
449
- let isBuilding = false;
450
- const pendingOps = [];
451
-
452
- function indexAdd(key, content, tier, updatedAt) {
453
- bm25.add(key, content, tier, updatedAt); // update live index for immediate search
454
- if (isBuilding) pendingOps.push({ op: "add", key, content, tier, updatedAt });
455
- }
456
-
457
- function indexRemove(key) {
458
- bm25.remove(key);
459
- if (isBuilding) pendingOps.push({ op: "remove", key });
460
- }
461
-
462
- // ---------------------------------------------------------------------------
463
- // Index build
464
- // ---------------------------------------------------------------------------
465
-
466
- async function buildIndex() {
467
- if (isBuilding) return; // skip if already running
468
- isBuilding = true;
469
- pendingOps.length = 0;
470
-
471
- const entries = [];
472
-
473
- // Scan Redis — scanKeys normalizes page/individual yields from scanIterator
474
- try {
475
- for await (const key of scanKeys("*", "string")) {
476
- try {
477
- const val = await redis.get(key);
478
- if (val) entries.push({ key, content: val, tier: "longterm", updatedAt: null });
479
- } catch (e) {
480
- console.warn(`Index build — skipping key '${key}': ${e.message}`);
481
- }
482
- }
483
- } catch (e) {
484
- console.error("Index build — Redis scan error:", e.message);
485
- }
486
-
487
- // Walk Obsidian vault
488
- async function walk(dir) {
489
- let items;
490
- try { items = await fs.readdir(dir, { withFileTypes: true }); }
491
- catch (e) { if (e.code === "ENOENT") return; throw e; }
492
- for (const entry of items) {
493
- const fullPath = path.join(dir, entry.name);
494
- if (entry.isDirectory()) { await walk(fullPath); continue; }
495
- if (!entry.name.endsWith(".md")) continue;
496
- try {
497
- const content = await fs.readFile(fullPath, "utf-8");
498
- const stat = await fs.stat(fullPath);
499
- const relative = path.relative(OBSIDIAN_VAULT_PATH, fullPath);
500
- const key = relative.replace(/\//g, ":").replace(/\\/, ":").replace(/\.md$/, "");
501
- entries.push({ key, content, tier: "working", updatedAt: stat.mtime.toISOString() });
502
- } catch (e) {
503
- console.error("Index build — vault read error:", fullPath, e.message);
504
- }
505
- }
506
- }
507
- await walk(OBSIDIAN_VAULT_PATH);
508
-
509
- // Build fresh index in bulk — O(N log N), no event-loop stall per doc
510
- const fresh = new BM25Index();
511
- fresh.bulkAdd(entries);
512
-
513
- // Atomic field swap
514
- bm25.docs = fresh.docs;
515
- bm25.df = fresh.df;
516
- bm25.avgLen = fresh.avgLen;
517
- bm25.N = fresh.N;
518
-
519
- // Re-apply writes/removes that occurred during the build
520
- for (const op of pendingOps) {
521
- if (op.op === "add") bm25.add(op.key, op.content, op.tier, op.updatedAt);
522
- else if (op.op === "remove") bm25.remove(op.key);
523
- }
524
- pendingOps.length = 0;
525
- isBuilding = false;
526
-
527
- console.log(`BM25 index built: ${bm25.size} docs (${entries.length} total entries scanned)`);
528
- }
529
-
530
- // ---------------------------------------------------------------------------
531
- // Obsidian vault helpers
532
- // ---------------------------------------------------------------------------
533
-
534
- async function saveToObsidian(key, content, metadata) {
535
- const filePath = keyToObsidianPath(key); // throws on bad key
536
- await fs.mkdir(path.dirname(filePath), { recursive: true });
537
- let output = content;
538
- if (metadata && Object.keys(metadata).length > 0) {
539
- const lines = ["---"];
540
- for (const [k, v] of Object.entries(metadata))
541
- lines.push(`${k}: ${Array.isArray(v) ? `[${v.join(", ")}]` : v}`);
542
- lines.push("---", "");
543
- output = lines.join("\n") + content;
544
- }
545
- await fs.writeFile(filePath, output, "utf-8");
546
- return filePath;
547
- }
548
-
549
- async function readFromObsidian(key) {
550
- const filePath = keyToObsidianPath(key);
551
- try { return await fs.readFile(filePath, "utf-8"); }
552
- catch (e) { if (e.code === "ENOENT") return null; throw e; }
553
- }
554
-
555
- async function deleteFromObsidian(key) {
556
- const filePath = keyToObsidianPath(key);
557
- try { await fs.unlink(filePath); return true; }
558
- catch (e) { if (e.code === "ENOENT") return false; throw e; }
559
- }
560
-
561
- async function searchObsidian(query, namespace) {
562
- const results = [];
563
- async function walk(dir) {
564
- let items;
565
- try { items = await fs.readdir(dir, { withFileTypes: true }); }
566
- catch (e) { if (e.code === "ENOENT") return; throw e; }
567
- for (const entry of items) {
568
- const fullPath = path.join(dir, entry.name);
569
- if (entry.isDirectory()) { await walk(fullPath); continue; }
570
- if (!entry.name.endsWith(".md")) continue;
571
- const content = await fs.readFile(fullPath, "utf-8");
572
- if (!content.toLowerCase().includes(query.toLowerCase())) continue;
573
- const relative = path.relative(OBSIDIAN_VAULT_PATH, fullPath);
574
- const key = relative.replace(/[\\/]/g, ":").replace(/\.md$/, "");
575
- if (namespace && !key.startsWith(namespace + ":")) continue;
576
- results.push({ key, content, path: relative });
577
- }
578
- }
579
- await walk(OBSIDIAN_VAULT_PATH);
580
- return results;
581
- }
582
-
583
- async function listObsidian(prefix) {
584
- const results = [];
585
- async function walk(dir) {
586
- let items;
587
- try { items = await fs.readdir(dir, { withFileTypes: true }); }
588
- catch (e) { if (e.code === "ENOENT") return; throw e; }
589
- for (const entry of items) {
590
- const fullPath = path.join(dir, entry.name);
591
- if (entry.isDirectory()) { await walk(fullPath); continue; }
592
- if (!entry.name.endsWith(".md")) continue;
593
- const relative = path.relative(OBSIDIAN_VAULT_PATH, fullPath);
594
- const key = relative.replace(/[\\/]/g, ":").replace(/\.md$/, "");
595
- if (!prefix || key.startsWith(prefix)) results.push({ key, path: relative });
596
- }
597
- }
598
- await walk(OBSIDIAN_VAULT_PATH);
599
- return results;
600
- }
601
-
602
- // ---------------------------------------------------------------------------
603
- // MCP server factory
604
- // ---------------------------------------------------------------------------
605
-
606
- function createMcpServerInstance(profile = "full") {
607
- const server = new McpServer({ name: "heurchain-mcp", version: "1.4.0" });
608
-
609
- // --- Cache Tools ---
610
-
611
- server.tool("cache_set", "Store a key-value pair in Redis (longterm tier). Optionally scope to a namespace.", {
612
- key: z.string().describe("Cache key"),
613
- value: z.string().describe("Value to cache"),
614
- ttl: z.number().optional().describe("Time-to-live in seconds"),
615
- namespace: z.string().optional().describe("Optional namespace prefix, e.g. 'agent-hermes'. Key becomes namespace:key.")
616
- }, async ({ key, value, ttl, namespace }) => {
617
- const fullKey = namespace ? `${namespace}:${key}` : key;
618
- try { validateKey(fullKey); } catch (e) {
619
- return { content: [{ type: "text", text: JSON.stringify({ cached: false, error: e.message }) }] };
620
- }
621
- if (ttl) await redis.setEx(fullKey, ttl, value);
622
- else await redis.set(fullKey, value);
623
- indexAdd(fullKey, value, "longterm", new Date().toISOString());
624
- return { content: [{ type: "text", text: JSON.stringify({ cached: true, key: fullKey }) }] };
625
- });
626
-
627
- server.tool("cache_get", "Retrieve a cached value from Redis by key", {
628
- key: z.string().describe("Cache key to retrieve"),
629
- namespace: z.string().optional().describe("Namespace prefix used when storing")
630
- }, async ({ key, namespace }) => {
631
- const fullKey = namespace ? `${namespace}:${key}` : key;
632
- const val = await redis.get(fullKey);
633
- if (val) return { content: [{ type: "text", text: val }] };
634
- return { content: [{ type: "text", text: JSON.stringify({ hit: false }) }] };
635
- });
636
-
637
- server.tool("cache_delete", "Delete a cached key from Redis", {
638
- key: z.string().describe("Cache key to delete"),
639
- namespace: z.string().optional().describe("Namespace prefix used when storing")
640
- }, async ({ key, namespace }) => {
641
- const fullKey = namespace ? `${namespace}:${key}` : key;
642
- const deleted = await redis.del(fullKey);
643
- indexRemove(fullKey);
644
- return { content: [{ type: "text", text: JSON.stringify({ deleted: deleted > 0, key: fullKey }) }] };
645
- });
646
-
647
- server.tool("redis_stats", "Get Redis server statistics and memory info", {}, async () => {
648
- const info = await redis.info("stats");
649
- return { content: [{ type: "text", text: info }] };
650
- });
651
-
652
- server.tool("health_check", "Check the health of the MCP server, Redis, and Obsidian vault", {}, async () => {
653
- const redisOk = redis.isReady;
654
- let obsidianOk = false;
655
- try { await fs.access(OBSIDIAN_VAULT_PATH); obsidianOk = true; } catch {}
656
- return { content: [{ type: "text", text: JSON.stringify({
657
- status: (redisOk && obsidianOk) ? "healthy" : "degraded",
658
- redis: redisOk, obsidian: obsidianOk,
659
- vault_path: OBSIDIAN_VAULT_PATH,
660
- bm25_docs: bm25.size,
661
- uptime: process.uptime()
662
- }) }] };
663
- });
664
-
665
- // --- Obsidian Vault Tools ---
666
-
667
- server.tool("obsidian_write_note", "Write a markdown note to the Obsidian vault (working tier). Key colons become folder separators.", {
668
- key: z.string().describe("Note key (e.g. doc:proxmox:setup)"),
669
- content: z.string().describe("Markdown body"),
670
- metadata: z.record(z.string()).optional().describe("Optional frontmatter metadata"),
671
- namespace: z.string().optional().describe("Optional namespace prefix")
672
- }, async ({ key, content, metadata, namespace }) => {
673
- const fullKey = namespace ? `${namespace}:${key}` : key;
674
- try {
675
- const filePath = await saveToObsidian(fullKey, content, metadata);
676
- indexAdd(fullKey, content, "working", new Date().toISOString());
677
- return { content: [{ type: "text", text: JSON.stringify({ stored: true, key: fullKey, path: filePath, tier: "working" }) }] };
678
- } catch (e) {
679
- console.error(`obsidian_write_note error for key '${fullKey}':`, e);
680
- return { content: [{ type: "text", text: JSON.stringify({ stored: false, key: fullKey, error: e.message.startsWith("invalid key") ? e.message : "write failed" }) }] };
681
- }
682
- });
683
-
684
- server.tool("obsidian_read_note", "Read a markdown note from the Obsidian vault by key", {
685
- key: z.string().describe("Note key"),
686
- namespace: z.string().optional().describe("Namespace prefix used when storing")
687
- }, async ({ key, namespace }) => {
688
- const fullKey = namespace ? `${namespace}:${key}` : key;
689
- try {
690
- const content = await readFromObsidian(fullKey);
691
- if (content === null) return { content: [{ type: "text", text: JSON.stringify({ found: false, key: fullKey }) }] };
692
- return { content: [{ type: "text", text: JSON.stringify({ found: true, key: fullKey, content }) }] };
693
- } catch (e) {
694
- console.error(`obsidian_read_note error for key '${fullKey}':`, e);
695
- return { content: [{ type: "text", text: JSON.stringify({ found: false, key: fullKey, error: e.message.startsWith("invalid key") ? e.message : "read failed" }) }] };
696
- }
697
- });
698
-
699
- server.tool("obsidian_delete_note", "Delete a markdown note from the Obsidian vault by key", {
700
- key: z.string().describe("Note key to delete"),
701
- namespace: z.string().optional().describe("Namespace prefix used when storing")
702
- }, async ({ key, namespace }) => {
703
- const fullKey = namespace ? `${namespace}:${key}` : key;
704
- try {
705
- const deleted = await deleteFromObsidian(fullKey);
706
- indexRemove(fullKey);
707
- return { content: [{ type: "text", text: JSON.stringify({ deleted, key: fullKey }) }] };
708
- } catch (e) {
709
- console.error(`obsidian_delete_note error for key '${fullKey}':`, e);
710
- return { content: [{ type: "text", text: JSON.stringify({ deleted: false, key: fullKey, error: e.message.startsWith("invalid key") ? e.message : "delete failed" }) }] };
711
- }
712
- });
713
-
714
- server.tool("obsidian_search_notes", "Search markdown notes in the Obsidian vault by keyword (substring, vault only). Prefer heurchain_search for ranked results across all tiers.", {
715
- query: z.string().describe("Search term"),
716
- limit: z.number().optional().describe("Max results (1–100, default 10)").default(10),
717
- namespace: z.string().optional().describe("Restrict to a namespace prefix")
718
- }, async ({ query, limit, namespace }) => {
719
- const safeLimit = Math.max(1, Math.min(Number.isFinite(limit) ? Math.floor(limit) : 10, 100));
720
- try {
721
- const results = await searchObsidian(query, namespace || null);
722
- return { content: [{ type: "text", text: JSON.stringify({
723
- count: results.length,
724
- results: results.slice(0, safeLimit).map(r => ({ key: r.key, preview: r.content.substring(0, 200) }))
725
- }) }] };
726
- } catch (e) {
727
- console.error("obsidian_search_notes error:", e);
728
- return { content: [{ type: "text", text: JSON.stringify({ error: "search failed" }) }] };
729
- }
730
- });
731
-
732
- server.tool("obsidian_list_notes", "List all markdown notes in the Obsidian vault, optionally filtered by prefix", {
733
- prefix: z.string().optional().describe("Key prefix filter, e.g. doc:proxmox:")
734
- }, async ({ prefix }) => {
735
- try {
736
- const results = await listObsidian(prefix);
737
- return { content: [{ type: "text", text: JSON.stringify({ count: results.length, notes: results }) }] };
738
- } catch (e) {
739
- console.error("obsidian_list_notes error:", e);
740
- return { content: [{ type: "text", text: JSON.stringify({ error: "list failed" }) }] };
741
- }
742
- });
743
-
744
- // --- HeurChain BM25 Search ---
745
-
746
- server.tool("heurchain_search",
747
- "Search the full knowledge base using HeurChain's native BM25 ranked index. Covers all Redis keys (longterm) and Obsidian vault files (working). Preferred over obsidian_search_notes.", {
748
- query: z.string().describe("Search query"),
749
- limit: z.number().optional().describe("Max results (default 10, max 50)").default(10),
750
- tier: z.string().optional().describe("'all' | 'longterm' | 'working' (default: all)").default("all"),
751
- minScore: z.number().optional().describe("Minimum BM25 score (default 0; try 0.5 to reduce noise)").default(0),
752
- noiseFilter: z.boolean().optional().describe("Skip low-signal session metadata (default true)").default(true),
753
- namespace: z.string().optional().describe("Restrict to a namespace, e.g. 'agent-hermes'")
754
- }, async ({ query, limit, tier, minScore, noiseFilter, namespace }) => {
755
- try {
756
- const results = bm25.search(query, {
757
- limit: Math.min(limit, 50),
758
- tier,
759
- minScore,
760
- noiseFilter,
761
- namespace: namespace || null
762
- });
763
- return { content: [{ type: "text", text: JSON.stringify({
764
- count: results.length,
765
- index_size: bm25.size,
766
- results: results.map(r => ({
767
- key: r.key,
768
- score: Math.round(r.score * 1000) / 1000,
769
- tier: r.tier,
770
- preview: (r.content || "").substring(0, 300),
771
- updated_at: r.updated_at
772
- }))
773
- }) }] };
774
- } catch (e) {
775
- console.error("heurchain_search error:", e);
776
- return { content: [{ type: "text", text: JSON.stringify({ error: "search failed" }) }] };
777
- }
778
- });
779
-
780
- // --- heurchain_export (Glacier Sync) ---
781
-
782
- server.tool("heurchain_export",
783
- "Export Redis longterm-tier entries to Obsidian vault markdown files. Makes the Redis knowledge base human-readable. Idempotent — safe to run repeatedly.", {
784
- prefix: z.string().optional().describe("Only export keys matching this prefix (e.g. 'doc:'). Defaults to all keys."),
785
- overwrite: z.boolean().optional().describe("Overwrite existing vault files (default false — skip if file exists)").default(false)
786
- }, async ({ prefix, overwrite }) => {
787
- let exported = 0, skipped = 0, errors = 0;
788
- try {
789
- for await (const key of scanKeys(prefix ? `${prefix}*` : "*", "string")) {
790
- // skip keys that would fail path validation
791
- try { validateKey(key); } catch { skipped++; continue; }
792
- const val = await redis.get(key);
793
- if (!val) continue;
794
- const filePath = keyToObsidianPath(key);
795
- if (!overwrite) {
796
- try { await fs.access(filePath); skipped++; continue; } catch {}
797
- }
798
- try {
799
- await fs.mkdir(path.dirname(filePath), { recursive: true });
800
- // avoid double-frontmatter if the Redis value already starts with ---
801
- const hasFrontmatter = val.trimStart().startsWith("---");
802
- const header = hasFrontmatter
803
- ? `<!-- exported: key=${key} at ${new Date().toISOString()} -->\n\n`
804
- : `---\nkey: ${key}\nexported_at: ${new Date().toISOString()}\nsource: redis-longterm\n---\n\n`;
805
- await fs.writeFile(filePath, header + val, "utf-8");
806
- indexAdd(key, val, "longterm", new Date().toISOString());
807
- exported++;
808
- } catch (e) {
809
- console.error(`heurchain_export: failed to write '${key}':`, e.message);
810
- errors++;
811
- }
812
- }
813
- return { content: [{ type: "text", text: JSON.stringify({ exported, skipped, errors, vault_path: OBSIDIAN_VAULT_PATH }) }] };
814
- } catch (e) {
815
- console.error("heurchain_export error:", e);
816
- return { content: [{ type: "text", text: JSON.stringify({ error: "export failed", exported, skipped, errors }) }] };
817
- }
818
- });
819
-
820
- // --- Procedural Memory Tools ---
821
-
822
- server.tool("proc_set",
823
- "Store a stable agent preference or style knob in procedural memory. These persist indefinitely across sessions — no TTL, no decay. Use for learned behaviors, coding style, user preferences.", {
824
- agent_name: z.string().describe("Agent identifier (e.g. 'claude-code', 'hermes')"),
825
- key: z.string().describe("Preference name (alphanumeric, hyphens, underscores — e.g. 'coding_style')"),
826
- value: z.string().describe("Preference value")
827
- }, async ({ agent_name, key, value }) => {
828
- try {
829
- const rk = procKey(agent_name, key);
830
- await redis.set(rk, value);
831
- return { content: [{ type: "text", text: JSON.stringify({ stored: true, key: rk }) }] };
832
- } catch (e) {
833
- return { content: [{ type: "text", text: JSON.stringify({ stored: false, error: e.message }) }] };
834
- }
835
- });
836
-
837
- server.tool("proc_get",
838
- "Read all procedural memory knobs for an agent as compact JSON. Call once at session start and inject into system prompt — zero tool calls needed during the session.", {
839
- agent_name: z.string().describe("Agent identifier")
840
- }, async ({ agent_name }) => {
841
- try {
842
- const result = await procGet(agent_name);
843
- return { content: [{ type: "text", text: JSON.stringify(result) }] };
844
- } catch (e) {
845
- return { content: [{ type: "text", text: JSON.stringify({ error: e.message }) }] };
846
- }
847
- });
848
-
849
- // --- Infrastructure Tools (full profile only) ---
850
- if (profile === "full") {
851
-
852
- // --- Prometheus Monitoring Tools ---
853
-
854
- server.tool("prometheus_get_targets", "Returns the status of all targets Prometheus is currently scraping", {}, async () => {
855
- try {
856
- const response = await fetch("http://prometheus:9090/api/v1/targets");
857
- if (!response.ok) return { content: [{ type: "text", text: JSON.stringify({ error: `HTTP ${response.status}` }) }] };
858
- const data = await response.json();
859
- const targets = data.data.activeTargets.map(t => ({
860
- job: t.labels.job, instance: t.labels.instance, health: t.health, last_scrape: t.lastScrape
861
- }));
862
- return { content: [{ type: "text", text: JSON.stringify({ targets }) }] };
863
- } catch (e) {
864
- return { content: [{ type: "text", text: JSON.stringify({ error: "request failed" }) }] };
865
- }
866
- });
867
-
868
- server.tool("prometheus_get_alerts", "Returns any currently firing or pending Prometheus alerts", {}, async () => {
869
- try {
870
- const response = await fetch("http://prometheus:9090/api/v1/alerts");
871
- if (!response.ok) return { content: [{ type: "text", text: JSON.stringify({ error: `HTTP ${response.status}` }) }] };
872
- const data = await response.json();
873
- return { content: [{ type: "text", text: JSON.stringify({ alerts: data.data.alerts }) }] };
874
- } catch (e) {
875
- return { content: [{ type: "text", text: JSON.stringify({ error: "request failed" }) }] };
876
- }
877
- });
878
-
879
- server.tool("prometheus_query", "Execute a PromQL query against Prometheus", {
880
- promql_query: z.string().describe("The PromQL query string to execute")
881
- }, async ({ promql_query }) => {
882
- try {
883
- const response = await fetch(`http://prometheus:9090/api/v1/query?query=${encodeURIComponent(promql_query)}`);
884
- if (!response.ok) return { content: [{ type: "text", text: JSON.stringify({ error: `HTTP ${response.status}` }) }] };
885
- const data = await response.json();
886
- return { content: [{ type: "text", text: JSON.stringify(data.data) }] };
887
- } catch (e) {
888
- return { content: [{ type: "text", text: JSON.stringify({ error: "request failed" }) }] };
889
- }
890
- });
891
-
892
- // --- Grafana Monitoring Tools — fail closed if creds not configured (#6) ---
893
-
894
- const grafanaUser = process.env.GRAFANA_USER;
895
- const grafanaPass = process.env.GRAFANA_PASSWORD;
896
- const grafanaAuth = (grafanaUser && grafanaPass)
897
- ? "Basic " + Buffer.from(`${grafanaUser}:${grafanaPass}`).toString("base64")
898
- : null;
899
-
900
- function grafanaHeaders() {
901
- if (!grafanaAuth) throw new Error("Grafana credentials not configured (GRAFANA_USER / GRAFANA_PASSWORD not set)");
902
- return { "Authorization": grafanaAuth };
903
- }
904
-
905
- server.tool("grafana_get_health", "Check the health status of the Grafana server", {}, async () => {
906
- try {
907
- const response = await fetch("http://grafana:3000/api/health", { headers: grafanaHeaders() });
908
- if (!response.ok) return { content: [{ type: "text", text: JSON.stringify({ error: `HTTP ${response.status}` }) }] };
909
- return { content: [{ type: "text", text: JSON.stringify(await response.json()) }] };
910
- } catch (e) {
911
- return { content: [{ type: "text", text: JSON.stringify({ error: e.message.includes("credentials") ? e.message : "request failed" }) }] };
912
- }
913
- });
914
-
915
- server.tool("grafana_list_dashboards", "List all available dashboards in Grafana", {}, async () => {
916
- try {
917
- const response = await fetch("http://grafana:3000/api/search?type=dash-db", { headers: grafanaHeaders() });
918
- if (!response.ok) return { content: [{ type: "text", text: JSON.stringify({ error: `HTTP ${response.status}` }) }] };
919
- const dashboards = await response.json();
920
- return { content: [{ type: "text", text: JSON.stringify({ dashboards: dashboards.map(d => ({ title: d.title, url: d.url })) }) }] };
921
- } catch (e) {
922
- return { content: [{ type: "text", text: JSON.stringify({ error: e.message.includes("credentials") ? e.message : "request failed" }) }] };
923
- }
924
- });
925
-
926
- // --- Proxmox Documentation Tools ---
927
-
928
- async function searchDocs(keyPattern, contentFilter) {
929
- const keys = [];
930
- for await (const key of scanKeys(keyPattern, "string")) keys.push(key);
931
- if (keys.length === 0) return [];
932
- const results = [];
933
- for (const key of keys) {
934
- const val = await redis.get(key);
935
- if (val && (!contentFilter || val.toLowerCase().includes(contentFilter.toLowerCase())))
936
- results.push({ key, content: val });
937
- }
938
- return results;
939
- }
940
-
941
- server.tool("proxmox_get_cluster_status", "Retrieve the Proxmox cluster status document from the knowledge base", {}, async () => {
942
- const results = await searchDocs("doc:proxmox:cluster-status*", null);
943
- if (!results.length) return { content: [{ type: "text", text: JSON.stringify({ error: "No cluster status document found. Store one with key 'doc:proxmox:cluster-status' using cache_set." }) }] };
944
- return { content: [{ type: "text", text: results[0].content }] };
945
- });
946
-
947
- server.tool("proxmox_list_nodes", "List all Proxmox nodes documented in the knowledge base", {}, async () => {
948
- const results = await searchDocs("doc:proxmox:node:*", null);
949
- if (!results.length) return { content: [{ type: "text", text: JSON.stringify({ error: "No node documents found." }) }] };
950
- return { content: [{ type: "text", text: JSON.stringify({ nodes: results.map(r => ({ key: r.key, title: r.key.split(":").pop() })) }) }] };
951
- });
952
-
953
- server.tool("proxmox_find_vm", "Search for a specific VM or container by ID or name in the knowledge base", {
954
- vm_id_or_name: z.string().describe("VM ID or name to search for")
955
- }, async ({ vm_id_or_name }) => {
956
- let results = await searchDocs("doc:proxmox:vm:*", vm_id_or_name);
957
- if (!results.length) results = await searchDocs("doc:proxmox:ct:*", vm_id_or_name);
958
- if (!results.length) results = await searchDocs("doc:proxmox:*", vm_id_or_name);
959
- if (!results.length) return { content: [{ type: "text", text: JSON.stringify({ error: `No VM or CT found matching "${vm_id_or_name}".` }) }] };
960
- return { content: [{ type: "text", text: results[0].content }] };
961
- });
962
-
963
- // --- Ceph Documentation Tools ---
964
-
965
- server.tool("ceph_get_health_status", "Retrieve the latest Ceph cluster health status from the knowledge base", {}, async () => {
966
- const results = await searchDocs("doc:ceph:cluster-status*", null);
967
- if (!results.length) return { content: [{ type: "text", text: JSON.stringify({ error: "No Ceph status document found." }) }] };
968
- return { content: [{ type: "text", text: results[0].content }] };
969
- });
970
-
971
- server.tool("ceph_list_osd_notes", "List all Ceph OSD documentation notes from the knowledge base", {}, async () => {
972
- const results = await searchDocs("doc:ceph:osd:*", null);
973
- if (!results.length) return { content: [{ type: "text", text: JSON.stringify({ error: "No OSD documents found." }) }] };
974
- return { content: [{ type: "text", text: JSON.stringify({ osd_notes: results.map(r => r.key) }) }] };
975
- });
976
-
977
- // --- Network Documentation Tools ---
978
-
979
- server.tool("network_search_docs", "Search across network and infrastructure documentation in the knowledge base", {
980
- search_query: z.string().describe("Search query to find relevant documentation")
981
- }, async ({ search_query }) => {
982
- const results = [...await searchDocs("doc:network:*", search_query), ...await searchDocs("doc:infrastructure:*", search_query)];
983
- if (!results.length) return { content: [{ type: "text", text: JSON.stringify({ status: "No relevant documents found." }) }] };
984
- return { content: [{ type: "text", text: JSON.stringify({ results: results.slice(0, 5).map(r => ({ key: r.key, preview: r.content.substring(0, 200) })) }) }] };
985
- });
986
-
987
- server.tool("network_get_runbook", "Retrieve a specific network runbook by topic from the knowledge base", {
988
- topic: z.string().describe("Runbook topic to search for")
989
- }, async ({ topic }) => {
990
- const results = await searchDocs("doc:runbook:*", topic);
991
- if (!results.length) return { content: [{ type: "text", text: JSON.stringify({ error: `No runbook found for topic: "${topic}".` }) }] };
992
- return { content: [{ type: "text", text: results[0].content }] };
993
- });
994
-
995
- // --- User Context Tools ---
996
-
997
- const USER_HISTORY_TTL = 90 * 24 * 60 * 60;
998
-
999
- server.tool("user_context_add_entry", "Add an entry to a user's daily interaction history log", {
1000
- user_id: z.string().describe("User identifier"),
1001
- content: z.string().describe("Content to log")
1002
- }, async ({ user_id, content }) => {
1003
- const today = new Date().toISOString().split("T")[0];
1004
- const key = `doc:users:${user_id}:history:${today}`;
1005
- let existing = await redis.get(key);
1006
- if (!existing) existing = `## User History for ${user_id} - ${today}\n\n`;
1007
- const entry = `**[${new Date().toLocaleTimeString()}]**\n${content}\n\n---\n\n`;
1008
- const updated = existing + entry;
1009
- await redis.setEx(key, USER_HISTORY_TTL, updated);
1010
- indexAdd(key, updated, "longterm", new Date().toISOString());
1011
- return { content: [{ type: "text", text: JSON.stringify({ success: true, key }) }] };
1012
- });
1013
-
1014
- server.tool("user_context_get_history", "Retrieve the full interaction history for a user on a specific date", {
1015
- user_id: z.string().describe("User identifier"),
1016
- date: z.string().describe("Date in YYYY-MM-DD format")
1017
- }, async ({ user_id, date }) => {
1018
- const key = `doc:users:${user_id}:history:${date}`;
1019
- const val = await redis.get(key);
1020
- if (!val) return { content: [{ type: "text", text: JSON.stringify({ status: "Not Found", message: `No history for user '${user_id}' on '${date}'.` }) }] };
1021
- return { content: [{ type: "text", text: JSON.stringify({ user_id, date, history: val }) }] };
1022
- });
1023
-
1024
- server.tool("user_context_search_history", "Search across all of a user's history documents by keyword", {
1025
- user_id: z.string().describe("User identifier"),
1026
- query: z.string().describe("Search query")
1027
- }, async ({ user_id, query }) => {
1028
- const results = await searchDocs(`doc:users:${user_id}:history:*`, query);
1029
- if (!results.length) return { content: [{ type: "text", text: JSON.stringify({ status: "No matches found." }) }] };
1030
- return { content: [{ type: "text", text: JSON.stringify({
1031
- results: results.map(r => ({ date: r.key.split(":").pop(), match_preview: r.content.substring(0, 200) + "..." }))
1032
- }) }] };
1033
- });
1034
-
1035
- } // end profile === "full"
1036
-
1037
- return server;
1038
- }
1039
-
1040
- // ---------------------------------------------------------------------------
1041
- // SSE transport — fix #3: max session cap + cleanup on error/close
1042
- // ---------------------------------------------------------------------------
1043
-
1044
- const MAX_SESSIONS = 50;
1045
- const transports = {};
1046
-
1047
- app.get("/sse", requireAuth, async (req, res) => {
1048
- if (Object.keys(transports).length >= MAX_SESSIONS) {
1049
- return res.status(503).json({ error: "Max concurrent sessions reached" });
1050
- }
1051
- const profile = req.query.profile === "slim" ? "slim" : "full";
1052
- const transport = new SSEServerTransport("/messages", res);
1053
- const mcpServer = createMcpServerInstance(profile);
1054
- const sessionId = transport.sessionId;
1055
- transports[sessionId] = { transport, server: mcpServer };
1056
-
1057
- const cleanup = () => {
1058
- if (!transports[sessionId]) return;
1059
- mcpServer.close().catch(() => {});
1060
- delete transports[sessionId];
1061
- };
1062
-
1063
- res.on("close", cleanup);
1064
- req.on("close", cleanup);
1065
-
1066
- try {
1067
- await mcpServer.connect(transport);
1068
- } catch (e) {
1069
- cleanup();
1070
- console.error("SSE connect error:", e.message);
1071
- if (!res.headersSent) res.status(500).json({ error: "Connection failed" });
1072
- }
1073
- });
1074
-
1075
- app.post("/messages", requireAuth, async (req, res) => {
1076
- const sessionId = req.query.sessionId;
1077
- const entry = transports[sessionId];
1078
- if (entry) await entry.transport.handlePostMessage(req, res);
1079
- else res.status(400).json({ error: "No transport found for sessionId" });
1080
- });
1081
-
1082
- // ---------------------------------------------------------------------------
1083
- // REST API (backwards compatibility) — all routes require auth except /health
1084
- // ---------------------------------------------------------------------------
1085
-
1086
- const jsonParser = express.json();
1087
-
1088
- app.get("/health", async (req, res) => {
1089
- const redisOk = redis.isReady;
1090
- let obsidianOk = false;
1091
- try { await fs.access(OBSIDIAN_VAULT_PATH); obsidianOk = true; } catch {}
1092
- res.json({
1093
- status: (redisOk && obsidianOk) ? "healthy" : "degraded",
1094
- redis: redisOk, obsidian: obsidianOk,
1095
- vault_path: OBSIDIAN_VAULT_PATH,
1096
- bm25_docs: bm25.size,
1097
- uptime: process.uptime()
1098
- });
1099
- });
1100
-
1101
- app.get("/api/tools", requireAuth, (req, res) => {
1102
- const profile = req.query.profile === "slim" ? "slim" : "full";
1103
- const core = [
1104
- "cache_set", "cache_get", "cache_delete", "redis_stats", "health_check",
1105
- "obsidian_write_note", "obsidian_read_note", "obsidian_delete_note",
1106
- "obsidian_search_notes", "obsidian_list_notes",
1107
- "heurchain_search", "heurchain_export",
1108
- "proc_set", "proc_get"
1109
- ];
1110
- const infra = [
1111
- "prometheus_get_targets", "prometheus_get_alerts", "prometheus_query",
1112
- "grafana_get_health", "grafana_list_dashboards",
1113
- "proxmox_get_cluster_status", "proxmox_list_nodes", "proxmox_find_vm",
1114
- "ceph_get_health_status", "ceph_list_osd_notes",
1115
- "network_search_docs", "network_get_runbook",
1116
- "user_context_add_entry", "user_context_get_history", "user_context_search_history"
1117
- ];
1118
- res.json({
1119
- profile,
1120
- tools: profile === "slim" ? core : [...core, ...infra],
1121
- rest: ["/api/search", "/api/buffer", "/api/proc", "/api/session-context"]
1122
- });
1123
- });
1124
-
1125
- app.post("/api/cache", requireAuth, jsonParser, async (req, res) => {
1126
- const { key, value, ttl } = req.body;
1127
- if (ttl) await redis.setEx(key, ttl, JSON.stringify(value));
1128
- else await redis.set(key, JSON.stringify(value));
1129
- res.json({ cached: true });
1130
- });
1131
-
1132
- app.get("/api/cache/:key", requireAuth, async (req, res) => {
1133
- const val = await redis.get(req.params.key);
1134
- if (val) res.json({ hit: true, data: JSON.parse(val) });
1135
- else res.json({ hit: false });
1136
- });
1137
-
1138
- app.get("/api/stats", requireAuth, async (req, res) => {
1139
- const info = await redis.info("stats");
1140
- res.json({ redis_info: info, bm25_docs: bm25.size });
1141
- });
1142
-
1143
- app.get("/api/obsidian/notes", requireAuth, async (req, res) => {
1144
- const { key, prefix } = req.query;
1145
- if (key) {
1146
- try {
1147
- const content = await readFromObsidian(key);
1148
- if (content === null) return res.status(404).json({ found: false, key });
1149
- return res.json({ found: true, key, content });
1150
- } catch (e) {
1151
- return res.status(400).json({ error: e.message.startsWith("invalid key") ? e.message : "read failed" });
1152
- }
1153
- }
1154
- const notes = await listObsidian(prefix);
1155
- res.json({ count: notes.length, notes });
1156
- });
1157
-
1158
- app.post("/api/obsidian/notes", requireAuth, jsonParser, async (req, res) => {
1159
- const { key, content, metadata } = req.body;
1160
- if (!key || content === undefined) return res.status(400).json({ error: "key and content required" });
1161
- try {
1162
- const filePath = await saveToObsidian(key, content, metadata);
1163
- res.json({ stored: true, key, path: filePath });
1164
- } catch (e) {
1165
- res.status(e.message.startsWith("invalid key") ? 400 : 500)
1166
- .json({ error: e.message.startsWith("invalid key") ? e.message : "write failed" });
1167
- }
1168
- });
1169
-
1170
- app.delete("/api/obsidian/notes", requireAuth, async (req, res) => {
1171
- const { key } = req.query;
1172
- if (!key) return res.status(400).json({ error: "key required" });
1173
- try {
1174
- const deleted = await deleteFromObsidian(key);
1175
- res.json({ deleted, key });
1176
- } catch (e) {
1177
- res.status(400).json({ error: e.message.startsWith("invalid key") ? e.message : "delete failed" });
1178
- }
1179
- });
1180
-
1181
- app.get("/api/obsidian/search", requireAuth, async (req, res) => {
1182
- const { q, limit } = req.query;
1183
- if (!q) return res.status(400).json({ error: "q required" });
1184
- const max = Math.max(1, Math.min(parseInt(limit) || 10, 100));
1185
- const results = await searchObsidian(q, null);
1186
- res.json({ count: results.length, results: results.slice(0, max).map(r => ({ key: r.key, path: r.path, preview: r.content.substring(0, 200) })) });
1187
- });
1188
-
1189
- // ---------------------------------------------------------------------------
1190
- // BM25 search REST — frictionless recall via curl/hook, no MCP tool overhead
1191
- // ---------------------------------------------------------------------------
1192
-
1193
- app.get("/api/search", requireAuth, async (req, res) => {
1194
- const { q, limit, tier, minScore, namespace } = req.query;
1195
- if (!q) return res.status(400).json({ error: "q required" });
1196
- const results = bm25.search(q, {
1197
- limit: Math.min(parseInt(limit) || 10, 50),
1198
- tier: tier || "all",
1199
- minScore: parseFloat(minScore) || 0,
1200
- noiseFilter: true,
1201
- namespace: namespace || null
1202
- });
1203
- res.json({
1204
- count: results.length,
1205
- index_size: bm25.size,
1206
- results: results.map(r => ({
1207
- key: r.key,
1208
- score: Math.round(r.score * 1000) / 1000,
1209
- tier: r.tier,
1210
- preview: (r.content || "").substring(0, 300)
1211
- }))
1212
- });
1213
- });
1214
-
1215
- // ---------------------------------------------------------------------------
1216
- // Session buffer — ambient turn capture, POST fire-and-forget from agent hook
1217
- // ---------------------------------------------------------------------------
1218
-
1219
- app.post("/api/buffer", requireAuth, jsonParser, async (req, res) => {
1220
- const { session_id, agent_name, role, content } = req.body;
1221
- if (!session_id || !content) return res.status(400).json({ error: "session_id and content required" });
1222
- bufferAppend(session_id, agent_name, role, content);
1223
- res.json({ buffered: true, session_id });
1224
- });
1225
-
1226
- app.post("/api/buffer/flush", requireAuth, jsonParser, async (req, res) => {
1227
- const { session_id } = req.body;
1228
- if (!session_id) return res.status(400).json({ error: "session_id required" });
1229
- await flushBuffer(session_id);
1230
- res.json({ flushed: true, session_id });
1231
- });
1232
-
1233
- // ---------------------------------------------------------------------------
1234
- // Procedural memory REST — stable agent preference knobs, no TTL
1235
- // ---------------------------------------------------------------------------
1236
-
1237
- app.post("/api/proc", requireAuth, jsonParser, async (req, res) => {
1238
- const { agent_name, key, value } = req.body;
1239
- if (!agent_name || !key || value === undefined) return res.status(400).json({ error: "agent_name, key, value required" });
1240
- try {
1241
- const rk = procKey(agent_name, key);
1242
- await redis.set(rk, typeof value === "string" ? value : JSON.stringify(value));
1243
- res.json({ stored: true, key: rk });
1244
- } catch (e) {
1245
- res.status(400).json({ error: e.message });
1246
- }
1247
- });
1248
-
1249
- app.get("/api/proc/:agent", requireAuth, async (req, res) => {
1250
- try {
1251
- if (!PROC_KEY_RE.test(req.params.agent)) return res.status(400).json({ error: "invalid agent name" });
1252
- res.json(await procGet(req.params.agent));
1253
- } catch (e) {
1254
- res.status(500).json({ error: "proc read failed" });
1255
- }
1256
- });
1257
-
1258
- app.delete("/api/proc/:agent/:key", requireAuth, async (req, res) => {
1259
- try {
1260
- const rk = procKey(req.params.agent, req.params.key);
1261
- const deleted = await redis.del(rk);
1262
- res.json({ deleted: deleted > 0, key: rk });
1263
- } catch (e) {
1264
- res.status(400).json({ error: e.message });
1265
- }
1266
- });
1267
-
1268
- // ---------------------------------------------------------------------------
1269
- // Session-context injection — compact recall for system-prompt injection.
1270
- // Returns proc knobs + last session summary + top BM25 hits, hard-capped at
1271
- // SESSION_CONTEXT_TOKENS (~400 tokens). Call via hook at session start;
1272
- // inject the JSON into system prompt. Zero MCP tool overhead.
1273
- // ---------------------------------------------------------------------------
1274
-
1275
- app.get("/api/session-context", requireAuth, async (req, res) => {
1276
- const { agent, hint } = req.query;
1277
- if (!agent) return res.status(400).json({ error: "agent required" });
1278
- if (!PROC_KEY_RE.test(agent)) return res.status(400).json({ error: "invalid agent name" });
1279
-
1280
- // 1. Procedural knobs
1281
- let proc = {};
1282
- try { proc = await procGet(agent); } catch {}
1283
-
1284
- // 2. Last session summary — scan for most recent buffer entry or session record
1285
- let last_session = null;
1286
- try {
1287
- const candidates = [];
1288
- for await (const k of scanKeys("session:*:buffer", "string")) {
1289
- const raw = await redis.get(k);
1290
- if (!raw) continue;
1291
- try {
1292
- const turns = JSON.parse(raw);
1293
- const agentTurns = turns.filter(t => t.agent === agent);
1294
- if (agentTurns.length === 0) continue;
1295
- const last = agentTurns[agentTurns.length - 1];
1296
- candidates.push({ ts: last.ts, summary: last.content.substring(0, 200) });
1297
- } catch {}
1298
- }
1299
- if (candidates.length > 0) {
1300
- candidates.sort((a, b) => b.ts.localeCompare(a.ts));
1301
- last_session = candidates[0].summary;
1302
- }
1303
- } catch {}
1304
-
1305
- // 3. BM25 relevant results, token-budgeted
1306
- let relevant = [];
1307
- if (hint) {
1308
- try {
1309
- const hits = bm25.search(hint, { limit: 15, minScore: 0.5, noiseFilter: true });
1310
- let budget = SESSION_CONTEXT_TOKENS - JSON.stringify(proc).length / 4 - 50;
1311
- for (const r of hits) {
1312
- const excerpt = (r.content || "").substring(0, 200);
1313
- budget -= Math.ceil((r.key.length + excerpt.length) / 4);
1314
- if (budget < 0) break;
1315
- relevant.push({ key: r.key, score: Math.round(r.score * 100) / 100, excerpt });
1316
- }
1317
- } catch {}
1318
- }
1319
-
1320
- // 4. Briefing field (M3)
1321
- let briefing = null;
1322
- if (BRIEFING_ENABLED) {
1323
- const tsRaw = await redis.get(briefingTsKey(agent));
1324
- if (tsRaw && (Date.now() - parseInt(tsRaw, 10)) < BRIEFING_MAX_AGE_MS) {
1325
- try {
1326
- briefing = JSON.parse(await redis.get(briefingKey(agent)) || "null");
1327
- } catch {}
1328
- } else {
1329
- // Stale or missing — trigger async regen, return null this call
1330
- runProactiveBriefings().catch(() => {});
1331
- }
1332
- // Prepend per-agent instructions if set
1333
- const agentInstructions = await redis.get(procKey(agent, "briefing_instructions")) || BRIEFING_INSTRUCTIONS;
1334
- if (briefing && agentInstructions) {
1335
- briefing = [agentInstructions, ...briefing];
1336
- }
1337
- }
1338
-
1339
- res.json({ proc, last_session, relevant, briefing });
1340
- });
1341
-
1342
- // ---------------------------------------------------------------------------
1343
- // Startup
1344
- // ---------------------------------------------------------------------------
1345
-
1346
- (async () => {
1347
- await redis.connect();
1348
- await buildIndex();
1349
- setInterval(buildIndex, 60 * 60 * 1000); // hourly BM25 rebuild
1350
- // Flush all active session buffers on heartbeat
1351
- setInterval(async () => {
1352
- for (const sessionId of sessionBuffers.keys()) {
1353
- await flushBuffer(sessionId).catch(() => {});
1354
- }
1355
- }, BUFFER_FLUSH_MS);
1356
- if (BRIEFING_ENABLED) {
1357
- // Fire once at startup after 30s warmup, then on interval
1358
- setTimeout(() => runProactiveBriefings().catch(e => console.error("[briefing] startup run failed:", e.message)), 30_000);
1359
- setInterval(() => runProactiveBriefings().catch(() => {}), BRIEFING_INTERVAL_MS);
1360
- console.log(`[heurchain] briefing worker: interval=${BRIEFING_INTERVAL_MS}ms provider=${COMPRESSOR_PROVIDER}`);
1361
- }
1362
- const port = parseInt(process.env.MCP_PORT, 10) || 3010;
1363
- app.listen(port, () => console.log(`HeurChain MCP v1.4.1 listening on port ${port} — ${bm25.size} docs indexed`));
1364
- })();