heurchain 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/package.json +1 -1
  2. package/server.js +365 -180
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "heurchain",
3
- "version": "1.0.0",
3
+ "version": "1.2.0",
4
4
  "description": "MCP (Model Context Protocol) server for HeurChain — tiered agent memory with BM25 search, session lifecycle, and infrastructure knowledge tools",
5
5
  "main": "server.js",
6
6
  "scripts": {
package/server.js CHANGED
@@ -14,13 +14,180 @@ const redis = createClient({ url: process.env.REDIS_URL || "redis://localhost:63
14
14
  redis.on("error", (err) => console.error("Redis error:", err));
15
15
 
16
16
  const OBSIDIAN_VAULT_PATH = process.env.OBSIDIAN_VAULT_PATH || "/opt/obsidian-vault";
17
- const HEURCHAIN_URL = process.env.HEURCHAIN_URL || "http://host.docker.internal:3012";
18
17
 
19
- (async () => {
20
- await redis.connect();
21
- const port = parseInt(process.env.MCP_PORT, 10) || 3010;
22
- app.listen(port, () => console.log("MCP server listening on port " + port));
23
- })();
18
+ // ---------------------------------------------------------------------------
19
+ // BM25 Index
20
+ // ---------------------------------------------------------------------------
21
+
22
+ const STOPWORDS = new Set([
23
+ "a","an","the","and","or","but","in","on","at","to","for","of","with",
24
+ "by","from","as","is","was","are","were","be","been","being","have",
25
+ "has","had","do","does","did","will","would","could","should","may",
26
+ "might","shall","can","not","no","nor","so","yet","both","either",
27
+ "than","then","when","where","which","who","that","this","these","those",
28
+ "it","its","i","me","my","we","our","you","your","he","she","they","them",
29
+ "his","her","their","what","how","all","each","any","more","also","just",
30
+ "if","up","out","about","into","through","over","after","before","between"
31
+ ]);
32
+
33
+ // Low-signal session metadata patterns — excluded from search by default
34
+ const NOISE_KEY_PATTERNS = [/session-summary/, /agent-end/, /heartbeat/];
35
+ const NOISE_CONTENT_PATTERNS = [
36
+ /^\d+ user turns,\s*\d+ total messages/i,
37
+ /^openclaw session on [a-z0-9-]+:/i,
38
+ /^session ended\./i
39
+ ];
40
+
41
+ function isSessionNoise(key, content) {
42
+ if (NOISE_KEY_PATTERNS.some(p => p.test(key))) return true;
43
+ const trimmed = (content || "").trim();
44
+ return NOISE_CONTENT_PATTERNS.some(p => p.test(trimmed));
45
+ }
46
+
47
+ class BM25Index {
48
+ constructor(k1 = 1.5, b = 0.75) {
49
+ this.k1 = k1;
50
+ this.b = b;
51
+ this.docs = new Map(); // key -> { tf, tokens, content, tier, updatedAt }
52
+ this.df = new Map(); // term -> document frequency count
53
+ this.avgLen = 0;
54
+ this.N = 0;
55
+ }
56
+
57
+ tokenize(text) {
58
+ return (text || "")
59
+ .toLowerCase()
60
+ .replace(/[^a-z0-9\s]/g, " ")
61
+ .split(/\s+/)
62
+ .filter(t => t.length > 1 && !STOPWORDS.has(t));
63
+ }
64
+
65
+ add(key, content, tier, updatedAt) {
66
+ if (this.docs.has(key)) this._removeFromIndex(key);
67
+ const tokens = this.tokenize(content);
68
+ const tf = new Map();
69
+ for (const t of tokens) tf.set(t, (tf.get(t) || 0) + 1);
70
+ this.docs.set(key, { tf, tokens, content, tier, updatedAt });
71
+ for (const term of tf.keys()) this.df.set(term, (this.df.get(term) || 0) + 1);
72
+ this._recalcStats();
73
+ }
74
+
75
+ remove(key) {
76
+ if (!this.docs.has(key)) return;
77
+ this._removeFromIndex(key);
78
+ this._recalcStats();
79
+ }
80
+
81
+ _removeFromIndex(key) {
82
+ const doc = this.docs.get(key);
83
+ if (!doc) return;
84
+ for (const term of doc.tf.keys()) {
85
+ const count = this.df.get(term) || 0;
86
+ if (count <= 1) this.df.delete(term);
87
+ else this.df.set(term, count - 1);
88
+ }
89
+ this.docs.delete(key);
90
+ }
91
+
92
+ _recalcStats() {
93
+ this.N = this.docs.size;
94
+ if (this.N === 0) { this.avgLen = 0; return; }
95
+ let total = 0;
96
+ for (const doc of this.docs.values()) total += doc.tokens.length;
97
+ this.avgLen = total / this.N;
98
+ }
99
+
100
+ _scoreDoc(queryTokens, docKey) {
101
+ const doc = this.docs.get(docKey);
102
+ if (!doc) return 0;
103
+ const dl = doc.tokens.length;
104
+ const lenNorm = this.avgLen > 0 ? dl / this.avgLen : 1;
105
+ let score = 0;
106
+ for (const term of queryTokens) {
107
+ const tf = doc.tf.get(term) || 0;
108
+ if (tf === 0) continue;
109
+ const df = this.df.get(term) || 0;
110
+ const idf = Math.log((this.N - df + 0.5) / (df + 0.5) + 1);
111
+ const tfNorm = (tf * (this.k1 + 1)) / (tf + this.k1 * (1 - this.b + this.b * lenNorm));
112
+ score += idf * tfNorm;
113
+ }
114
+ return score;
115
+ }
116
+
117
+ search(query, { limit = 10, tier = "all", minScore = 0, noiseFilter = true, namespace = null } = {}) {
118
+ const queryTokens = this.tokenize(query);
119
+ if (queryTokens.length === 0) return [];
120
+ const results = [];
121
+ for (const [key, doc] of this.docs.entries()) {
122
+ if (tier !== "all" && doc.tier !== tier) continue;
123
+ if (namespace && !key.startsWith(namespace + ":")) continue;
124
+ if (noiseFilter && isSessionNoise(key, doc.content)) continue;
125
+ const score = this._scoreDoc(queryTokens, key);
126
+ if (score > minScore) {
127
+ results.push({ key, score, tier: doc.tier, content: doc.content, updated_at: doc.updatedAt });
128
+ }
129
+ }
130
+ return results.sort((a, b) => b.score - a.score).slice(0, limit);
131
+ }
132
+
133
+ get size() { return this.N; }
134
+ }
135
+
136
+ const bm25 = new BM25Index();
137
+
138
+ // ---------------------------------------------------------------------------
139
+ // Index Build & Maintenance
140
+ // ---------------------------------------------------------------------------
141
+
142
+ async function buildIndex() {
143
+ const freshIndex = new BM25Index();
144
+
145
+ // Scan all Redis keys
146
+ let redisCount = 0;
147
+ try {
148
+ for await (const key of redis.scanIterator({ MATCH: "*", COUNT: 200 })) {
149
+ const val = await redis.get(key);
150
+ if (val) {
151
+ freshIndex.add(key, val, "longterm", null);
152
+ redisCount++;
153
+ }
154
+ }
155
+ } catch (e) {
156
+ console.error("Index build — Redis scan error:", e.message);
157
+ }
158
+
159
+ // Walk Obsidian vault
160
+ let vaultCount = 0;
161
+ async function walk(dir) {
162
+ let entries;
163
+ try { entries = await fs.readdir(dir, { withFileTypes: true }); }
164
+ catch (e) { if (e.code === "ENOENT") return; throw e; }
165
+ for (const entry of entries) {
166
+ const fullPath = path.join(dir, entry.name);
167
+ if (entry.isDirectory()) { await walk(fullPath); continue; }
168
+ if (!entry.name.endsWith(".md")) continue;
169
+ try {
170
+ const content = await fs.readFile(fullPath, "utf-8");
171
+ const stat = await fs.stat(fullPath);
172
+ const relative = path.relative(OBSIDIAN_VAULT_PATH, fullPath);
173
+ const key = relative.replace(/\//g, ":").replace(/\.md$/, "");
174
+ freshIndex.add(key, content, "working", stat.mtime.toISOString());
175
+ vaultCount++;
176
+ } catch (e) {
177
+ console.error("Index build — vault read error:", fullPath, e.message);
178
+ }
179
+ }
180
+ }
181
+ await walk(OBSIDIAN_VAULT_PATH);
182
+
183
+ // Swap in new index atomically
184
+ bm25.docs = freshIndex.docs;
185
+ bm25.df = freshIndex.df;
186
+ bm25.avgLen = freshIndex.avgLen;
187
+ bm25.N = freshIndex.N;
188
+
189
+ console.log(`BM25 index built: ${redisCount} Redis keys + ${vaultCount} vault files = ${bm25.size} docs`);
190
+ }
24
191
 
25
192
  // ---------------------------------------------------------------------------
26
193
  // Obsidian Vault Helpers
@@ -49,23 +216,14 @@ async function saveToObsidian(key, content, metadata) {
49
216
 
50
217
  async function readFromObsidian(key) {
51
218
  const filePath = keyToObsidianPath(key);
52
- try {
53
- return await fs.readFile(filePath, "utf-8");
54
- } catch (e) {
55
- if (e.code === "ENOENT") return null;
56
- throw e;
57
- }
219
+ try { return await fs.readFile(filePath, "utf-8"); }
220
+ catch (e) { if (e.code === "ENOENT") return null; throw e; }
58
221
  }
59
222
 
60
223
  async function deleteFromObsidian(key) {
61
224
  const filePath = keyToObsidianPath(key);
62
- try {
63
- await fs.unlink(filePath);
64
- return true;
65
- } catch (e) {
66
- if (e.code === "ENOENT") return false;
67
- throw e;
68
- }
225
+ try { await fs.unlink(filePath); return true; }
226
+ catch (e) { if (e.code === "ENOENT") return false; throw e; }
69
227
  }
70
228
 
71
229
  async function searchObsidian(query) {
@@ -74,23 +232,18 @@ async function searchObsidian(query) {
74
232
  const entries = await fs.readdir(dir, { withFileTypes: true });
75
233
  for (const entry of entries) {
76
234
  const fullPath = path.join(dir, entry.name);
77
- if (entry.isDirectory()) {
78
- await walk(fullPath);
79
- } else if (entry.name.endsWith(".md")) {
80
- const content = await fs.readFile(fullPath, "utf-8");
81
- if (content.toLowerCase().includes(query.toLowerCase())) {
82
- const relative = path.relative(OBSIDIAN_VAULT_PATH, fullPath);
83
- const key = relative.replace(/\//g, ":").replace(/\.md$/, "");
84
- results.push({ key, content, path: relative });
85
- }
235
+ if (entry.isDirectory()) { await walk(fullPath); continue; }
236
+ if (!entry.name.endsWith(".md")) continue;
237
+ const content = await fs.readFile(fullPath, "utf-8");
238
+ if (content.toLowerCase().includes(query.toLowerCase())) {
239
+ const relative = path.relative(OBSIDIAN_VAULT_PATH, fullPath);
240
+ const key = relative.replace(/\//g, ":").replace(/\.md$/, "");
241
+ results.push({ key, content, path: relative });
86
242
  }
87
243
  }
88
244
  }
89
- try {
90
- await walk(OBSIDIAN_VAULT_PATH);
91
- } catch (e) {
92
- if (e.code !== "ENOENT") throw e;
93
- }
245
+ try { await walk(OBSIDIAN_VAULT_PATH); }
246
+ catch (e) { if (e.code !== "ENOENT") throw e; }
94
247
  return results;
95
248
  }
96
249
 
@@ -100,22 +253,15 @@ async function listObsidian(prefix) {
100
253
  const entries = await fs.readdir(dir, { withFileTypes: true });
101
254
  for (const entry of entries) {
102
255
  const fullPath = path.join(dir, entry.name);
103
- if (entry.isDirectory()) {
104
- await walk(fullPath);
105
- } else if (entry.name.endsWith(".md")) {
106
- const relative = path.relative(OBSIDIAN_VAULT_PATH, fullPath);
107
- const key = relative.replace(/\//g, ":").replace(/\.md$/, "");
108
- if (!prefix || key.startsWith(prefix)) {
109
- results.push({ key, path: relative });
110
- }
111
- }
256
+ if (entry.isDirectory()) { await walk(fullPath); continue; }
257
+ if (!entry.name.endsWith(".md")) continue;
258
+ const relative = path.relative(OBSIDIAN_VAULT_PATH, fullPath);
259
+ const key = relative.replace(/\//g, ":").replace(/\.md$/, "");
260
+ if (!prefix || key.startsWith(prefix)) results.push({ key, path: relative });
112
261
  }
113
262
  }
114
- try {
115
- await walk(OBSIDIAN_VAULT_PATH);
116
- } catch (e) {
117
- if (e.code !== "ENOENT") throw e;
118
- }
263
+ try { await walk(OBSIDIAN_VAULT_PATH); }
264
+ catch (e) { if (e.code !== "ENOENT") throw e; }
119
265
  return results;
120
266
  }
121
267
 
@@ -126,34 +272,42 @@ async function listObsidian(prefix) {
126
272
  function createMcpServerInstance() {
127
273
  const server = new McpServer({
128
274
  name: "heurchain-mcp",
129
- version: "1.1.0"
275
+ version: "1.2.0"
130
276
  });
131
277
 
132
278
  // --- Cache Tools ---
133
279
 
134
- server.tool("cache_set", "Store a key-value pair in Redis cache with optional TTL", {
280
+ server.tool("cache_set", "Store a key-value pair in Redis (longterm tier). Optionally scope to a namespace.", {
135
281
  key: z.string().describe("Cache key"),
136
282
  value: z.string().describe("Value to cache"),
137
- ttl: z.number().optional().describe("Time-to-live in seconds")
138
- }, async ({ key, value, ttl }) => {
139
- if (ttl) await redis.setEx(key, ttl, value);
140
- else await redis.set(key, value);
141
- return { content: [{ type: "text", text: JSON.stringify({ cached: true, key }) }] };
283
+ ttl: z.number().optional().describe("Time-to-live in seconds"),
284
+ namespace: z.string().optional().describe("Optional namespace prefix, e.g. 'agent-hermes'. Key becomes namespace:key.")
285
+ }, async ({ key, value, ttl, namespace }) => {
286
+ const fullKey = namespace ? `${namespace}:${key}` : key;
287
+ if (ttl) await redis.setEx(fullKey, ttl, value);
288
+ else await redis.set(fullKey, value);
289
+ bm25.add(fullKey, value, "longterm", new Date().toISOString());
290
+ return { content: [{ type: "text", text: JSON.stringify({ cached: true, key: fullKey }) }] };
142
291
  });
143
292
 
144
293
  server.tool("cache_get", "Retrieve a cached value from Redis by key", {
145
- key: z.string().describe("Cache key to retrieve")
146
- }, async ({ key }) => {
147
- const val = await redis.get(key);
294
+ key: z.string().describe("Cache key to retrieve"),
295
+ namespace: z.string().optional().describe("Namespace prefix used when storing")
296
+ }, async ({ key, namespace }) => {
297
+ const fullKey = namespace ? `${namespace}:${key}` : key;
298
+ const val = await redis.get(fullKey);
148
299
  if (val) return { content: [{ type: "text", text: val }] };
149
300
  return { content: [{ type: "text", text: JSON.stringify({ hit: false }) }] };
150
301
  });
151
302
 
152
303
  server.tool("cache_delete", "Delete a cached key from Redis", {
153
- key: z.string().describe("Cache key to delete")
154
- }, async ({ key }) => {
155
- const deleted = await redis.del(key);
156
- return { content: [{ type: "text", text: JSON.stringify({ deleted: deleted > 0, key }) }] };
304
+ key: z.string().describe("Cache key to delete"),
305
+ namespace: z.string().optional().describe("Namespace prefix used when storing")
306
+ }, async ({ key, namespace }) => {
307
+ const fullKey = namespace ? `${namespace}:${key}` : key;
308
+ const deleted = await redis.del(fullKey);
309
+ bm25.remove(fullKey);
310
+ return { content: [{ type: "text", text: JSON.stringify({ deleted: deleted > 0, key: fullKey }) }] };
157
311
  });
158
312
 
159
313
  server.tool("redis_stats", "Get Redis server statistics and memory info", {}, async () => {
@@ -164,10 +318,7 @@ function createMcpServerInstance() {
164
318
  server.tool("health_check", "Check the health of the MCP server, Redis, and Obsidian vault", {}, async () => {
165
319
  const redisOk = redis.isReady;
166
320
  let obsidianOk = false;
167
- try {
168
- await fs.access(OBSIDIAN_VAULT_PATH);
169
- obsidianOk = true;
170
- } catch {}
321
+ try { await fs.access(OBSIDIAN_VAULT_PATH); obsidianOk = true; } catch {}
171
322
  return {
172
323
  content: [{
173
324
  type: "text",
@@ -176,67 +327,79 @@ function createMcpServerInstance() {
176
327
  redis: redisOk,
177
328
  obsidian: obsidianOk,
178
329
  vault_path: OBSIDIAN_VAULT_PATH,
330
+ bm25_docs: bm25.size,
179
331
  uptime: process.uptime()
180
332
  })
181
333
  }]
182
334
  };
183
335
  });
184
336
 
185
- // --- Obsidian Vault Tools (Long-Term MD Storage) ---
337
+ // --- Obsidian Vault Tools (Working Tier / Human-Readable Glacier) ---
186
338
 
187
- server.tool("obsidian_write_note", "Write or overwrite a markdown note in the Obsidian vault for long-term storage", {
188
- key: z.string().describe("Note key (colons become folder separators, e.g., doc:proxmox:setup)"),
339
+ server.tool("obsidian_write_note", "Write a markdown note to the Obsidian vault (working tier — human-readable glacier). Key colons become folder separators.", {
340
+ key: z.string().describe("Note key (e.g. doc:proxmox:setup)"),
189
341
  content: z.string().describe("Markdown body"),
190
- metadata: z.record(z.string()).optional().describe("Optional frontmatter metadata (key-value pairs)")
191
- }, async ({ key, content, metadata }) => {
342
+ metadata: z.record(z.string()).optional().describe("Optional frontmatter metadata"),
343
+ namespace: z.string().optional().describe("Optional namespace prefix")
344
+ }, async ({ key, content, metadata, namespace }) => {
345
+ const fullKey = namespace ? `${namespace}:${key}` : key;
192
346
  try {
193
- const filePath = await saveToObsidian(key, content, metadata);
194
- return { content: [{ type: "text", text: JSON.stringify({ stored: true, key, path: filePath, tier: "reference" }) }] };
347
+ const filePath = await saveToObsidian(fullKey, content, metadata);
348
+ bm25.add(fullKey, content, "working", new Date().toISOString());
349
+ return { content: [{ type: "text", text: JSON.stringify({ stored: true, key: fullKey, path: filePath, tier: "working" }) }] };
195
350
  } catch (e) {
196
- return { content: [{ type: "text", text: JSON.stringify({ stored: false, key, error: e.message }) }] };
351
+ return { content: [{ type: "text", text: JSON.stringify({ stored: false, key: fullKey, error: e.message }) }] };
197
352
  }
198
353
  });
199
354
 
200
355
  server.tool("obsidian_read_note", "Read a markdown note from the Obsidian vault by key", {
201
- key: z.string().describe("Note key")
202
- }, async ({ key }) => {
356
+ key: z.string().describe("Note key"),
357
+ namespace: z.string().optional().describe("Namespace prefix used when storing")
358
+ }, async ({ key, namespace }) => {
359
+ const fullKey = namespace ? `${namespace}:${key}` : key;
203
360
  try {
204
- const content = await readFromObsidian(key);
205
- if (content === null) {
206
- return { content: [{ type: "text", text: JSON.stringify({ found: false, key }) }] };
207
- }
208
- return { content: [{ type: "text", text: JSON.stringify({ found: true, key, content }) }] };
361
+ const content = await readFromObsidian(fullKey);
362
+ if (content === null) return { content: [{ type: "text", text: JSON.stringify({ found: false, key: fullKey }) }] };
363
+ return { content: [{ type: "text", text: JSON.stringify({ found: true, key: fullKey, content }) }] };
209
364
  } catch (e) {
210
- return { content: [{ type: "text", text: JSON.stringify({ found: false, key, error: e.message }) }] };
365
+ return { content: [{ type: "text", text: JSON.stringify({ found: false, key: fullKey, error: e.message }) }] };
211
366
  }
212
367
  });
213
368
 
214
369
  server.tool("obsidian_delete_note", "Delete a markdown note from the Obsidian vault by key", {
215
- key: z.string().describe("Note key to delete")
216
- }, async ({ key }) => {
370
+ key: z.string().describe("Note key to delete"),
371
+ namespace: z.string().optional().describe("Namespace prefix used when storing")
372
+ }, async ({ key, namespace }) => {
373
+ const fullKey = namespace ? `${namespace}:${key}` : key;
217
374
  try {
218
- const deleted = await deleteFromObsidian(key);
219
- return { content: [{ type: "text", text: JSON.stringify({ deleted, key }) }] };
375
+ const deleted = await deleteFromObsidian(fullKey);
376
+ bm25.remove(fullKey);
377
+ return { content: [{ type: "text", text: JSON.stringify({ deleted, key: fullKey }) }] };
220
378
  } catch (e) {
221
- return { content: [{ type: "text", text: JSON.stringify({ deleted: false, key, error: e.message }) }] };
379
+ return { content: [{ type: "text", text: JSON.stringify({ deleted: false, key: fullKey, error: e.message }) }] };
222
380
  }
223
381
  });
224
382
 
225
- server.tool("obsidian_search_notes", "Search all markdown notes in the Obsidian vault by keyword", {
383
+ server.tool("obsidian_search_notes", "Search markdown notes in the Obsidian vault by keyword (substring match, vault only). Prefer heurchain_search for ranked results across all tiers.", {
226
384
  query: z.string().describe("Search term"),
227
- limit: z.number().optional().describe("Max results to return").default(10)
228
- }, async ({ query, limit }) => {
385
+ limit: z.number().optional().describe("Max results").default(10),
386
+ namespace: z.string().optional().describe("Restrict to a namespace prefix")
387
+ }, async ({ query, limit, namespace }) => {
229
388
  try {
230
- const results = await searchObsidian(query);
389
+ let results = await searchObsidian(query);
390
+ if (namespace) results = results.filter(r => r.key.startsWith(namespace + ":"));
231
391
  const sliced = results.slice(0, limit);
232
- return { content: [{ type: "text", text: JSON.stringify({ count: results.length, results: sliced.map(r => ({ key: r.key, preview: r.content.substring(0, 200) })) }) }] };
392
+ return { content: [{ type: "text", text: JSON.stringify({
393
+ count: results.length,
394
+ results: sliced.map(r => ({ key: r.key, preview: r.content.substring(0, 200) }))
395
+ }) }] };
233
396
  } catch (e) {
234
397
  return { content: [{ type: "text", text: JSON.stringify({ error: e.message }) }] };
235
398
  }
236
399
  });
237
400
 
238
401
  server.tool("obsidian_list_notes", "List all markdown notes in the Obsidian vault, optionally filtered by prefix", {
239
- prefix: z.string().optional().describe("Key prefix filter, e.g., doc:proxmox:")
402
+ prefix: z.string().optional().describe("Key prefix filter, e.g. doc:proxmox:")
240
403
  }, async ({ prefix }) => {
241
404
  try {
242
405
  const results = await listObsidian(prefix);
@@ -246,26 +409,74 @@ function createMcpServerInstance() {
246
409
  }
247
410
  });
248
411
 
249
- // --- HeurChain BM25 Search ---
412
+ // --- HeurChain BM25 Search (native, in-process) ---
250
413
 
251
- server.tool("heurchain_search", "Search the full knowledge base using HeurChain's BM25 ranked index (preferred over obsidian_search_notes — covers all 460+ Redis keys with relevance scoring)", {
414
+ server.tool("heurchain_search",
415
+ "Search the full knowledge base using HeurChain's native BM25 ranked index. Covers all Redis keys (longterm tier) and Obsidian vault files (working tier) with relevance scoring. Preferred over obsidian_search_notes.", {
252
416
  query: z.string().describe("Search query"),
253
417
  limit: z.number().optional().describe("Max results (default 10, max 50)").default(10),
254
- tier: z.string().optional().describe("'all' | 'longterm' | 'working' (default: all)").default("all")
255
- }, async ({ query, limit, tier }) => {
418
+ tier: z.string().optional().describe("'all' | 'longterm' | 'working' (default: all)").default("all"),
419
+ minScore: z.number().optional().describe("Minimum BM25 score threshold (default 0 = all results; try 0.5 to cut noise)").default(0),
420
+ noiseFilter: z.boolean().optional().describe("Skip low-signal session metadata entries (default true)").default(true),
421
+ namespace: z.string().optional().describe("Restrict search to a namespace, e.g. 'agent-hermes'")
422
+ }, async ({ query, limit, tier, minScore, noiseFilter, namespace }) => {
256
423
  try {
257
- const url = `${HEURCHAIN_URL}/search?q=${encodeURIComponent(query)}&limit=${limit}&tier=${tier}`;
258
- const response = await fetch(url);
259
- if (!response.ok) {
260
- return { content: [{ type: "text", text: JSON.stringify({ error: `HeurChain returned HTTP ${response.status}` }) }] };
261
- }
262
- const results = await response.json();
263
- return { content: [{ type: "text", text: JSON.stringify({ count: results.length, results: results.map(r => ({ key: r.key, score: r.score, tier: r.tier, preview: (r.content || "").substring(0, 300), updated_at: r.updated_at })) }) }] };
424
+ const results = bm25.search(query, {
425
+ limit: Math.min(limit, 50),
426
+ tier,
427
+ minScore,
428
+ noiseFilter,
429
+ namespace: namespace || null
430
+ });
431
+ return { content: [{ type: "text", text: JSON.stringify({
432
+ count: results.length,
433
+ index_size: bm25.size,
434
+ results: results.map(r => ({
435
+ key: r.key,
436
+ score: Math.round(r.score * 1000) / 1000,
437
+ tier: r.tier,
438
+ preview: (r.content || "").substring(0, 300),
439
+ updated_at: r.updated_at
440
+ }))
441
+ }) }] };
264
442
  } catch (e) {
265
443
  return { content: [{ type: "text", text: JSON.stringify({ error: e.message }) }] };
266
444
  }
267
445
  });
268
446
 
447
+ // --- heurchain_export (Glacier Sync) ---
448
+
449
+ server.tool("heurchain_export",
450
+ "Export Redis longterm-tier entries to Obsidian vault markdown files. Makes the Redis knowledge base human-readable and browsable. Idempotent — safe to run repeatedly.", {
451
+ prefix: z.string().optional().describe("Only export keys matching this prefix (e.g. 'doc:', 'memory:'). Defaults to all keys."),
452
+ overwrite: z.boolean().optional().describe("Overwrite existing vault files (default false — skip if file exists)").default(false)
453
+ }, async ({ prefix, overwrite }) => {
454
+ let exported = 0, skipped = 0, errors = 0;
455
+ try {
456
+ for await (const key of redis.scanIterator({ MATCH: prefix ? `${prefix}*` : "*", COUNT: 100 })) {
457
+ const val = await redis.get(key);
458
+ if (!val) continue;
459
+ const filePath = keyToObsidianPath(key);
460
+ if (!overwrite) {
461
+ try { await fs.access(filePath); skipped++; continue; } catch {}
462
+ }
463
+ try {
464
+ await fs.mkdir(path.dirname(filePath), { recursive: true });
465
+ const header = `---\nkey: ${key}\nexported_at: ${new Date().toISOString()}\nsource: redis-longterm\n---\n\n`;
466
+ await fs.writeFile(filePath, header + val, "utf-8");
467
+ bm25.add(key, val, "longterm", new Date().toISOString());
468
+ exported++;
469
+ } catch (e) {
470
+ console.error(`heurchain_export: failed to write ${key}:`, e.message);
471
+ errors++;
472
+ }
473
+ }
474
+ return { content: [{ type: "text", text: JSON.stringify({ exported, skipped, errors, vault_path: OBSIDIAN_VAULT_PATH }) }] };
475
+ } catch (e) {
476
+ return { content: [{ type: "text", text: JSON.stringify({ error: e.message, exported, skipped, errors }) }] };
477
+ }
478
+ });
479
+
269
480
  // --- Prometheus Monitoring Tools ---
270
481
 
271
482
  server.tool("prometheus_get_targets", "Returns the status of all targets Prometheus is currently scraping", {}, async () => {
@@ -274,10 +485,7 @@ function createMcpServerInstance() {
274
485
  if (!response.ok) return { content: [{ type: "text", text: JSON.stringify({ error: `HTTP ${response.status}` }) }] };
275
486
  const data = await response.json();
276
487
  const targets = data.data.activeTargets.map(t => ({
277
- job: t.labels.job,
278
- instance: t.labels.instance,
279
- health: t.health,
280
- last_scrape: t.lastScrape
488
+ job: t.labels.job, instance: t.labels.instance, health: t.health, last_scrape: t.lastScrape
281
489
  }));
282
490
  return { content: [{ type: "text", text: JSON.stringify({ targets }) }] };
283
491
  } catch (e) {
@@ -315,12 +523,9 @@ function createMcpServerInstance() {
315
523
 
316
524
  server.tool("grafana_get_health", "Check the health status of the Grafana server", {}, async () => {
317
525
  try {
318
- const response = await fetch("http://grafana:3000/api/health", {
319
- headers: { "Authorization": grafanaAuth }
320
- });
526
+ const response = await fetch("http://grafana:3000/api/health", { headers: { "Authorization": grafanaAuth } });
321
527
  if (!response.ok) return { content: [{ type: "text", text: JSON.stringify({ error: `HTTP ${response.status}` }) }] };
322
- const data = await response.json();
323
- return { content: [{ type: "text", text: JSON.stringify(data) }] };
528
+ return { content: [{ type: "text", text: JSON.stringify(await response.json()) }] };
324
529
  } catch (e) {
325
530
  return { content: [{ type: "text", text: JSON.stringify({ error: e.message }) }] };
326
531
  }
@@ -328,9 +533,7 @@ function createMcpServerInstance() {
328
533
 
329
534
  server.tool("grafana_list_dashboards", "List all available dashboards in Grafana", {}, async () => {
330
535
  try {
331
- const response = await fetch("http://grafana:3000/api/search?type=dash-db", {
332
- headers: { "Authorization": grafanaAuth }
333
- });
536
+ const response = await fetch("http://grafana:3000/api/search?type=dash-db", { headers: { "Authorization": grafanaAuth } });
334
537
  if (!response.ok) return { content: [{ type: "text", text: JSON.stringify({ error: `HTTP ${response.status}` }) }] };
335
538
  const dashboards = await response.json();
336
539
  return { content: [{ type: "text", text: JSON.stringify({ dashboards: dashboards.map(d => ({ title: d.title, url: d.url })) }) }] };
@@ -343,9 +546,7 @@ function createMcpServerInstance() {
343
546
 
344
547
  async function searchDocs(keyPattern, contentFilter) {
345
548
  const keys = [];
346
- for await (const key of redis.scanIterator({ MATCH: keyPattern, COUNT: 100 })) {
347
- keys.push(key);
348
- }
549
+ for await (const key of redis.scanIterator({ MATCH: keyPattern, COUNT: 100 })) keys.push(key);
349
550
  if (keys.length === 0) return [];
350
551
  const results = [];
351
552
  for (const key of keys) {
@@ -359,17 +560,13 @@ function createMcpServerInstance() {
359
560
 
360
561
  server.tool("proxmox_get_cluster_status", "Retrieve the Proxmox cluster status document from the knowledge base", {}, async () => {
361
562
  const results = await searchDocs("doc:proxmox:cluster-status*", null);
362
- if (results.length === 0) {
363
- return { content: [{ type: "text", text: JSON.stringify({ error: "No cluster status document found. Store one with key 'doc:proxmox:cluster-status' using cache_set." }) }] };
364
- }
563
+ if (results.length === 0) return { content: [{ type: "text", text: JSON.stringify({ error: "No cluster status document found. Store one with key 'doc:proxmox:cluster-status' using cache_set." }) }] };
365
564
  return { content: [{ type: "text", text: results[0].content }] };
366
565
  });
367
566
 
368
567
  server.tool("proxmox_list_nodes", "List all Proxmox nodes documented in the knowledge base", {}, async () => {
369
568
  const results = await searchDocs("doc:proxmox:node:*", null);
370
- if (results.length === 0) {
371
- return { content: [{ type: "text", text: JSON.stringify({ error: "No node documents found. Store them with keys like 'doc:proxmox:node:pver430' using cache_set." }) }] };
372
- }
569
+ if (results.length === 0) return { content: [{ type: "text", text: JSON.stringify({ error: "No node documents found. Store them with keys like 'doc:proxmox:node:pver430' using cache_set." }) }] };
373
570
  return { content: [{ type: "text", text: JSON.stringify({ nodes: results.map(r => ({ key: r.key, title: r.key.split(":").pop() })) }) }] };
374
571
  });
375
572
 
@@ -379,9 +576,7 @@ function createMcpServerInstance() {
379
576
  let results = await searchDocs("doc:proxmox:vm:*", vm_id_or_name);
380
577
  if (results.length === 0) results = await searchDocs("doc:proxmox:ct:*", vm_id_or_name);
381
578
  if (results.length === 0) results = await searchDocs("doc:proxmox:*", vm_id_or_name);
382
- if (results.length === 0) {
383
- return { content: [{ type: "text", text: JSON.stringify({ error: `No VM or CT found matching "${vm_id_or_name}".` }) }] };
384
- }
579
+ if (results.length === 0) return { content: [{ type: "text", text: JSON.stringify({ error: `No VM or CT found matching "${vm_id_or_name}".` }) }] };
385
580
  return { content: [{ type: "text", text: results[0].content }] };
386
581
  });
387
582
 
@@ -389,17 +584,13 @@ function createMcpServerInstance() {
389
584
 
390
585
  server.tool("ceph_get_health_status", "Retrieve the latest Ceph cluster health status from the knowledge base", {}, async () => {
391
586
  const results = await searchDocs("doc:ceph:cluster-status*", null);
392
- if (results.length === 0) {
393
- return { content: [{ type: "text", text: JSON.stringify({ error: "No Ceph status document found. Store one with key 'doc:ceph:cluster-status' using cache_set." }) }] };
394
- }
587
+ if (results.length === 0) return { content: [{ type: "text", text: JSON.stringify({ error: "No Ceph status document found. Store one with key 'doc:ceph:cluster-status' using cache_set." }) }] };
395
588
  return { content: [{ type: "text", text: results[0].content }] };
396
589
  });
397
590
 
398
591
  server.tool("ceph_list_osd_notes", "List all Ceph OSD documentation notes from the knowledge base", {}, async () => {
399
592
  const results = await searchDocs("doc:ceph:osd:*", null);
400
- if (results.length === 0) {
401
- return { content: [{ type: "text", text: JSON.stringify({ error: "No OSD documents found. Store them with keys like 'doc:ceph:osd:0' using cache_set." }) }] };
402
- }
593
+ if (results.length === 0) return { content: [{ type: "text", text: JSON.stringify({ error: "No OSD documents found. Store them with keys like 'doc:ceph:osd:0' using cache_set." }) }] };
403
594
  return { content: [{ type: "text", text: JSON.stringify({ osd_notes: results.map(r => r.key) }) }] };
404
595
  });
405
596
 
@@ -408,22 +599,16 @@ function createMcpServerInstance() {
408
599
  server.tool("network_search_docs", "Search across network and infrastructure documentation in the knowledge base", {
409
600
  search_query: z.string().describe("Search query to find relevant documentation")
410
601
  }, async ({ search_query }) => {
411
- const results = await searchDocs("doc:network:*", search_query);
412
- const infraResults = await searchDocs("doc:infrastructure:*", search_query);
413
- const all = [...results, ...infraResults];
414
- if (all.length === 0) {
415
- return { content: [{ type: "text", text: JSON.stringify({ status: "No relevant documents found." }) }] };
416
- }
417
- return { content: [{ type: "text", text: JSON.stringify({ results: all.slice(0, 5).map(r => ({ key: r.key, preview: r.content.substring(0, 200) })) }) }] };
602
+ const results = [...await searchDocs("doc:network:*", search_query), ...await searchDocs("doc:infrastructure:*", search_query)];
603
+ if (results.length === 0) return { content: [{ type: "text", text: JSON.stringify({ status: "No relevant documents found." }) }] };
604
+ return { content: [{ type: "text", text: JSON.stringify({ results: results.slice(0, 5).map(r => ({ key: r.key, preview: r.content.substring(0, 200) })) }) }] };
418
605
  });
419
606
 
420
607
  server.tool("network_get_runbook", "Retrieve a specific network runbook by topic from the knowledge base", {
421
608
  topic: z.string().describe("Runbook topic to search for")
422
609
  }, async ({ topic }) => {
423
610
  const results = await searchDocs("doc:runbook:*", topic);
424
- if (results.length === 0) {
425
- return { content: [{ type: "text", text: JSON.stringify({ error: `No runbook found for topic: "${topic}". Store runbooks with keys like 'doc:runbook:topic-name' using cache_set.` }) }] };
426
- }
611
+ if (results.length === 0) return { content: [{ type: "text", text: JSON.stringify({ error: `No runbook found for topic: "${topic}". Store runbooks with keys like 'doc:runbook:topic-name' using cache_set.` }) }] };
427
612
  return { content: [{ type: "text", text: results[0].content }] };
428
613
  });
429
614
 
@@ -441,7 +626,9 @@ function createMcpServerInstance() {
441
626
  if (!existing) existing = `## User History for ${user_id} - ${today}\n\n`;
442
627
  const timestamp = new Date().toLocaleTimeString();
443
628
  const entry = `**[${timestamp}]**\n${content}\n\n---\n\n`;
444
- await redis.setEx(key, USER_HISTORY_TTL, existing + entry);
629
+ const updated = existing + entry;
630
+ await redis.setEx(key, USER_HISTORY_TTL, updated);
631
+ bm25.add(key, updated, "longterm", new Date().toISOString());
445
632
  return { content: [{ type: "text", text: JSON.stringify({ success: true, key, entry_added: entry }) }] };
446
633
  });
447
634
 
@@ -451,9 +638,7 @@ function createMcpServerInstance() {
451
638
  }, async ({ user_id, date }) => {
452
639
  const key = `doc:users:${user_id}:history:${date}`;
453
640
  const val = await redis.get(key);
454
- if (!val) {
455
- return { content: [{ type: "text", text: JSON.stringify({ status: "Not Found", message: `No history found for user '${user_id}' on date '${date}'.` }) }] };
456
- }
641
+ if (!val) return { content: [{ type: "text", text: JSON.stringify({ status: "Not Found", message: `No history found for user '${user_id}' on date '${date}'.` }) }] };
457
642
  return { content: [{ type: "text", text: JSON.stringify({ user_id, date, history: val }) }] };
458
643
  });
459
644
 
@@ -462,59 +647,52 @@ function createMcpServerInstance() {
462
647
  query: z.string().describe("Search query")
463
648
  }, async ({ user_id, query }) => {
464
649
  const results = await searchDocs(`doc:users:${user_id}:history:*`, query);
465
- if (results.length === 0) {
466
- return { content: [{ type: "text", text: JSON.stringify({ status: "No matches found." }) }] };
467
- }
650
+ if (results.length === 0) return { content: [{ type: "text", text: JSON.stringify({ status: "No matches found." }) }] };
468
651
  return { content: [{ type: "text", text: JSON.stringify({
469
- results: results.map(r => ({
470
- date: r.key.split(":").pop(),
471
- match_preview: r.content.substring(0, 200) + "..."
472
- }))
652
+ results: results.map(r => ({ date: r.key.split(":").pop(), match_preview: r.content.substring(0, 200) + "..." }))
473
653
  }) }] };
474
654
  });
475
655
 
476
656
  return server;
477
657
  }
478
658
 
479
- // SSE transport management
659
+ // ---------------------------------------------------------------------------
660
+ // SSE Transport
661
+ // ---------------------------------------------------------------------------
662
+
480
663
  const transports = {};
481
664
 
482
665
  app.get("/sse", async (req, res) => {
483
666
  const transport = new SSEServerTransport("/messages", res);
484
667
  const mcpServer = createMcpServerInstance();
485
668
  transports[transport.sessionId] = { transport, server: mcpServer };
486
- res.on("close", () => {
487
- mcpServer.close();
488
- delete transports[transport.sessionId];
489
- });
669
+ res.on("close", () => { mcpServer.close(); delete transports[transport.sessionId]; });
490
670
  await mcpServer.connect(transport);
491
671
  });
492
672
 
493
673
  app.post("/messages", async (req, res) => {
494
674
  const sessionId = req.query.sessionId;
495
675
  const entry = transports[sessionId];
496
- if (entry) {
497
- await entry.transport.handlePostMessage(req, res);
498
- } else {
499
- res.status(400).json({ error: "No transport found for sessionId" });
500
- }
676
+ if (entry) await entry.transport.handlePostMessage(req, res);
677
+ else res.status(400).json({ error: "No transport found for sessionId" });
501
678
  });
502
679
 
503
- // --- REST API (backwards compatibility) ---
680
+ // ---------------------------------------------------------------------------
681
+ // REST API (backwards compatibility)
682
+ // ---------------------------------------------------------------------------
683
+
504
684
  const jsonParser = express.json();
505
685
 
506
686
  app.get("/health", async (req, res) => {
507
687
  const redisOk = redis.isReady;
508
688
  let obsidianOk = false;
509
- try {
510
- await fs.access(OBSIDIAN_VAULT_PATH);
511
- obsidianOk = true;
512
- } catch {}
689
+ try { await fs.access(OBSIDIAN_VAULT_PATH); obsidianOk = true; } catch {}
513
690
  res.json({
514
691
  status: (redisOk && obsidianOk) ? "healthy" : "degraded",
515
692
  redis: redisOk,
516
693
  obsidian: obsidianOk,
517
694
  vault_path: OBSIDIAN_VAULT_PATH,
695
+ bm25_docs: bm25.size,
518
696
  uptime: process.uptime()
519
697
  });
520
698
  });
@@ -524,7 +702,7 @@ app.get("/api/tools", (req, res) => {
524
702
  "cache_set", "cache_get", "cache_delete", "redis_stats", "health_check",
525
703
  "obsidian_write_note", "obsidian_read_note", "obsidian_delete_note",
526
704
  "obsidian_search_notes", "obsidian_list_notes",
527
- "heurchain_search",
705
+ "heurchain_search", "heurchain_export",
528
706
  "prometheus_get_targets", "prometheus_get_alerts", "prometheus_query",
529
707
  "grafana_get_health", "grafana_list_dashboards",
530
708
  "proxmox_get_cluster_status", "proxmox_list_nodes", "proxmox_find_vm",
@@ -536,21 +714,20 @@ app.get("/api/tools", (req, res) => {
536
714
 
537
715
  app.post("/api/cache", jsonParser, async (req, res) => {
538
716
  const { key, value, ttl } = req.body;
539
- const stored = typeof value === "string" ? value : JSON.stringify(value);
540
- if (ttl) await redis.setEx(key, ttl, stored);
541
- else await redis.set(key, stored);
717
+ if (ttl) await redis.setEx(key, ttl, JSON.stringify(value));
718
+ else await redis.set(key, JSON.stringify(value));
542
719
  res.json({ cached: true });
543
720
  });
544
721
 
545
722
  app.get("/api/cache/:key", async (req, res) => {
546
723
  const val = await redis.get(req.params.key);
547
- if (val) res.json({ hit: true, data: val });
724
+ if (val) res.json({ hit: true, data: JSON.parse(val) });
548
725
  else res.json({ hit: false });
549
726
  });
550
727
 
551
728
  app.get("/api/stats", async (req, res) => {
552
729
  const info = await redis.info("stats");
553
- res.json({ redis_info: info });
730
+ res.json({ redis_info: info, bm25_docs: bm25.size });
554
731
  });
555
732
 
556
733
  // --- Obsidian REST Endpoints ---
@@ -591,11 +768,19 @@ app.get("/api/obsidian/search", async (req, res) => {
591
768
  const max = parseInt(limit) || 10;
592
769
  res.json({
593
770
  count: results.length,
594
- results: results.slice(0, max).map(r => ({
595
- key: r.key,
596
- path: r.path,
597
- preview: r.content.substring(0, 200)
598
- }))
771
+ results: results.slice(0, max).map(r => ({ key: r.key, path: r.path, preview: r.content.substring(0, 200) }))
599
772
  });
600
773
  });
601
774
 
775
+ // ---------------------------------------------------------------------------
776
+ // Startup
777
+ // ---------------------------------------------------------------------------
778
+
779
+ (async () => {
780
+ await redis.connect();
781
+ await buildIndex();
782
+ // Rebuild index hourly to catch external writes
783
+ setInterval(buildIndex, 60 * 60 * 1000);
784
+ const port = parseInt(process.env.MCP_PORT, 10) || 3010;
785
+ app.listen(port, () => console.log(`HeurChain MCP v1.2.0 listening on port ${port} — ${bm25.size} docs indexed`));
786
+ })();