heurchain 1.2.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/CHANGELOG.md +67 -0
  2. package/package.json +4 -3
  3. package/server.js +262 -11
package/CHANGELOG.md ADDED
@@ -0,0 +1,67 @@
1
+ # HeurChain Changelog
2
+
3
+ ## v1.3.0 — 2026-05-07
4
+
5
+ ### New: Frictionless REST layer
6
+
7
+ - `GET /api/session-context?agent=X&hint=Y` — token-budgeted payload (proc knobs + last session + top BM25 hits) designed for system-prompt injection via hook at session start. Zero MCP tool calls required.
8
+ - `GET /api/search?q=...` — BM25 search over the full index via plain HTTP. No SSE session, no tool definition overhead.
9
+ - `POST /api/buffer` — fire-and-forget turn capture. Flushes to Redis on 5-min heartbeat or session end. No agent action required.
10
+ - `POST/GET/DELETE /api/proc` — procedural memory tier. Stable agent preferences (coding style, language, patterns) stored with no TTL or decay.
11
+
12
+ ### New: Slim MCP profile
13
+
14
+ - Connect with `?profile=slim` on `/sse` to load 14 core tools instead of 29 (~750 token savings on tool definitions per context window).
15
+ - Infrastructure tools (Prometheus, Grafana, Proxmox, Ceph) only load on `?profile=full`.
16
+
17
+ ### New: MCP tools
18
+
19
+ - `proc_set` / `proc_get` — read and write procedural memory from within an MCP session.
20
+
21
+ ### Fix: BM25 index was Redis-blind
22
+
23
+ - Silent bug in node-redis `scanIterator` caused it to yield key pages (arrays) instead of individual keys. Every `redis.get(array)` call failed silently, so BM25 was built from Obsidian vault files only — Redis keys were never indexed. Fixed via `scanKeys()` normalizer. Index now correctly covers both tiers (465 docs vs 447 before).
24
+
25
+ ---
26
+
27
+ ## v1.2.1 — 2026-05-06
28
+
29
+ ### Security
30
+
31
+ - Path traversal fix: `validateKey()` now rejects `..` segments, null bytes, and slash characters before resolving vault paths.
32
+ - Bearer token auth (`MCP_AUTH_TOKEN`) enforced on all endpoints. `/health` remains open for healthcheck probes.
33
+ - Grafana tool fails closed when credentials are not configured rather than exposing unauthenticated access.
34
+
35
+ ### Fixes
36
+
37
+ - SSE session cap (50) with cleanup on both `req.on("close")` and `res.on("close")` to prevent transport leaks.
38
+ - BM25 rebuild is atomic: `isBuilding` flag + `pendingOps` queue replays writes that arrived during a rebuild onto the fresh index after swap. No silent memory loss during hourly rebuild.
39
+ - `bulkAdd()` for O(N log N) index builds instead of O(N²) per-insert `_recalcStats()` calls.
40
+ - Per-key try/catch in `buildIndex` Redis scan — a single bad key type no longer aborts the entire scan.
41
+ - Tier validation: unknown tier strings fall back to `"all"` instead of returning empty results.
42
+
43
+ ### Features
44
+
45
+ - `heurchain_export` — glacier sync: exports Redis longterm-tier entries to Obsidian vault as markdown. Idempotent.
46
+ - `minScore` parameter on `heurchain_search` to filter low-signal results.
47
+ - Namespace isolation: `namespace` parameter scopes search and write operations to an agent-specific key prefix.
48
+ - Session noise filter: suppresses low-signal session metadata keys from BM25 results.
49
+
50
+ ---
51
+
52
+ ## v1.2.0 — 2026-05-05
53
+
54
+ ### Features
55
+
56
+ - Native BM25 search (Okapi BM25, k1=1.5, b=0.75) implemented in-process. Replaced dead broker call to port 3012.
57
+ - Redis integration: `redis-cache` service added to Docker stack; `REDIS_URL` fixed to use Docker service name.
58
+ - `heurchain_search` now indexes all Redis string keys + Obsidian vault `.md` files on startup with hourly rebuild.
59
+
60
+ ---
61
+
62
+ ## v1.1.0 — initial release
63
+
64
+ - MCP SSE server with 27 tools across cache, vault, search, monitoring, infrastructure, and user context categories.
65
+ - Tiered storage: Redis (longterm) + Obsidian vault (working).
66
+ - nginx gateway for SSE proxying.
67
+ - Ansible role for managed host deployment.
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "heurchain",
3
- "version": "1.2.1",
4
- "description": "MCP (Model Context Protocol) server for HeurChain — tiered agent memory with BM25 search, session lifecycle, and infrastructure knowledge tools",
3
+ "version": "1.3.0",
4
+ "description": "MCP (Model Context Protocol) server for HeurChain — tiered agent memory with BM25 search, session lifecycle, procedural memory, ambient capture, and frictionless session-context injection",
5
5
  "main": "server.js",
6
6
  "scripts": {
7
7
  "start": "node server.js"
@@ -9,7 +9,8 @@
9
9
  "author": "Peter Medina <peter@carlssoncreative.com>",
10
10
  "files": [
11
11
  "server.js",
12
- "AGENT_CONFIG.json"
12
+ "AGENT_CONFIG.json",
13
+ "CHANGELOG.md"
13
14
  ],
14
15
  "keywords": [
15
16
  "mcp",
package/server.js CHANGED
@@ -16,6 +16,22 @@ redis.on("error", (err) => console.error("Redis error:", err));
16
16
  const OBSIDIAN_VAULT_PATH = path.resolve(process.env.OBSIDIAN_VAULT_PATH || "/opt/obsidian-vault");
17
17
  const VAULT_ROOT = OBSIDIAN_VAULT_PATH + path.sep;
18
18
 
19
+ const BUFFER_CAPACITY = parseInt(process.env.BUFFER_CAPACITY, 10) || 50;
20
+ const BUFFER_FLUSH_MS = parseInt(process.env.BUFFER_FLUSH_MS, 10) || 5 * 60 * 1000; // 5 min
21
+ const SESSION_CONTEXT_TOKENS = parseInt(process.env.SESSION_CONTEXT_TOKENS, 10) || 400;
22
+ const PROC_KEY_RE = /^[a-zA-Z0-9_\-\.]+$/; // safe proc key segment
23
+
24
+ // node-redis scanIterator yields pages (arrays) in some versions and individual
25
+ // keys in others. scanKeys normalizes to individual key strings in all cases.
26
+ async function* scanKeys(pattern, type) {
27
+ const opts = { MATCH: pattern, COUNT: 200 };
28
+ if (type) opts.TYPE = type;
29
+ for await (const item of redis.scanIterator(opts)) {
30
+ if (Array.isArray(item)) { for (const k of item) yield k; }
31
+ else yield item;
32
+ }
33
+ }
34
+
19
35
  // ---------------------------------------------------------------------------
20
36
  // Auth middleware — set MCP_AUTH_TOKEN env var to enforce bearer token auth.
21
37
  // If unset, access is unrestricted (warn loudly).
@@ -199,6 +215,57 @@ class BM25Index {
199
215
 
200
216
  const bm25 = new BM25Index();
201
217
 
218
+ // ---------------------------------------------------------------------------
219
+ // Session buffer — ambient turn capture without explicit agent store calls.
220
+ // Agents POST to /api/buffer fire-and-forget; buffer auto-flushes to Redis.
221
+ // ---------------------------------------------------------------------------
222
+
223
+ const sessionBuffers = new Map(); // session_id -> [{ role, content, agent, ts }]
224
+
225
+ async function flushBuffer(sessionId) {
226
+ const buf = sessionBuffers.get(sessionId);
227
+ if (!buf || buf.length === 0) return;
228
+ const key = `session:${sessionId}:buffer`;
229
+ try {
230
+ const existing = await redis.get(key);
231
+ const prev = existing ? JSON.parse(existing) : [];
232
+ await redis.setEx(key, 7 * 24 * 60 * 60, JSON.stringify([...prev, ...buf]));
233
+ sessionBuffers.set(sessionId, []);
234
+ } catch (e) {
235
+ console.error(`Buffer flush error session ${sessionId}:`, e.message);
236
+ }
237
+ }
238
+
239
+ function bufferAppend(sessionId, agentName, role, content) {
240
+ if (!sessionBuffers.has(sessionId)) sessionBuffers.set(sessionId, []);
241
+ const buf = sessionBuffers.get(sessionId);
242
+ buf.push({ role: role || "user", content, agent: agentName || "unknown", ts: new Date().toISOString() });
243
+ if (buf.length >= BUFFER_CAPACITY) flushBuffer(sessionId).catch(() => {});
244
+ }
245
+
246
+ // ---------------------------------------------------------------------------
247
+ // Procedural memory — stable agent preferences/style knobs, no TTL/decay.
248
+ // Redis namespace: proc:agent:{name}:{key}
249
+ // ---------------------------------------------------------------------------
250
+
251
+ function procKey(agentName, key) {
252
+ if (!PROC_KEY_RE.test(agentName)) throw new Error("invalid agent_name");
253
+ if (!PROC_KEY_RE.test(key)) throw new Error("invalid proc key");
254
+ return `proc:agent:${agentName}:${key}`;
255
+ }
256
+
257
+ async function procGet(agentName) {
258
+ const prefix = `proc:agent:${agentName}:`;
259
+ const result = {};
260
+ for await (const k of scanKeys(`${prefix}*`, "string")) {
261
+ const val = await redis.get(k);
262
+ if (!val) continue;
263
+ const shortKey = k.slice(prefix.length);
264
+ try { result[shortKey] = JSON.parse(val); } catch { result[shortKey] = val; }
265
+ }
266
+ return result;
267
+ }
268
+
202
269
  // Pending-ops queue: writes/removes that occur during a rebuild are applied
203
270
  // after the swap so they aren't silently discarded — fix #4
204
271
  let isBuilding = false;
@@ -225,9 +292,9 @@ async function buildIndex() {
225
292
 
226
293
  const entries = [];
227
294
 
228
- // Scan Redis — use TYPE: "string" to skip hashes/lists/sets
295
+ // Scan Redis — scanKeys normalizes page/individual yields from scanIterator
229
296
  try {
230
- for await (const key of redis.scanIterator({ MATCH: "*", COUNT: 200, TYPE: "string" })) {
297
+ for await (const key of scanKeys("*", "string")) {
231
298
  try {
232
299
  const val = await redis.get(key);
233
300
  if (val) entries.push({ key, content: val, tier: "longterm", updatedAt: null });
@@ -358,8 +425,8 @@ async function listObsidian(prefix) {
358
425
  // MCP server factory
359
426
  // ---------------------------------------------------------------------------
360
427
 
361
- function createMcpServerInstance() {
362
- const server = new McpServer({ name: "heurchain-mcp", version: "1.2.1" });
428
+ function createMcpServerInstance(profile = "full") {
429
+ const server = new McpServer({ name: "heurchain-mcp", version: "1.3.0" });
363
430
 
364
431
  // --- Cache Tools ---
365
432
 
@@ -541,7 +608,7 @@ function createMcpServerInstance() {
541
608
  }, async ({ prefix, overwrite }) => {
542
609
  let exported = 0, skipped = 0, errors = 0;
543
610
  try {
544
- for await (const key of redis.scanIterator({ MATCH: prefix ? `${prefix}*` : "*", COUNT: 100, TYPE: "string" })) {
611
+ for await (const key of scanKeys(prefix ? `${prefix}*` : "*", "string")) {
545
612
  // skip keys that would fail path validation
546
613
  try { validateKey(key); } catch { skipped++; continue; }
547
614
  const val = await redis.get(key);
@@ -572,6 +639,38 @@ function createMcpServerInstance() {
572
639
  }
573
640
  });
574
641
 
642
+ // --- Procedural Memory Tools ---
643
+
644
+ server.tool("proc_set",
645
+ "Store a stable agent preference or style knob in procedural memory. These persist indefinitely across sessions — no TTL, no decay. Use for learned behaviors, coding style, user preferences.", {
646
+ agent_name: z.string().describe("Agent identifier (e.g. 'claude-code', 'hermes')"),
647
+ key: z.string().describe("Preference name (alphanumeric, hyphens, underscores — e.g. 'coding_style')"),
648
+ value: z.string().describe("Preference value")
649
+ }, async ({ agent_name, key, value }) => {
650
+ try {
651
+ const rk = procKey(agent_name, key);
652
+ await redis.set(rk, value);
653
+ return { content: [{ type: "text", text: JSON.stringify({ stored: true, key: rk }) }] };
654
+ } catch (e) {
655
+ return { content: [{ type: "text", text: JSON.stringify({ stored: false, error: e.message }) }] };
656
+ }
657
+ });
658
+
659
+ server.tool("proc_get",
660
+ "Read all procedural memory knobs for an agent as compact JSON. Call once at session start and inject into system prompt — zero tool calls needed during the session.", {
661
+ agent_name: z.string().describe("Agent identifier")
662
+ }, async ({ agent_name }) => {
663
+ try {
664
+ const result = await procGet(agent_name);
665
+ return { content: [{ type: "text", text: JSON.stringify(result) }] };
666
+ } catch (e) {
667
+ return { content: [{ type: "text", text: JSON.stringify({ error: e.message }) }] };
668
+ }
669
+ });
670
+
671
+ // --- Infrastructure Tools (full profile only) ---
672
+ if (profile === "full") {
673
+
575
674
  // --- Prometheus Monitoring Tools ---
576
675
 
577
676
  server.tool("prometheus_get_targets", "Returns the status of all targets Prometheus is currently scraping", {}, async () => {
@@ -650,7 +749,7 @@ function createMcpServerInstance() {
650
749
 
651
750
  async function searchDocs(keyPattern, contentFilter) {
652
751
  const keys = [];
653
- for await (const key of redis.scanIterator({ MATCH: keyPattern, COUNT: 100, TYPE: "string" })) keys.push(key);
752
+ for await (const key of scanKeys(keyPattern, "string")) keys.push(key);
654
753
  if (keys.length === 0) return [];
655
754
  const results = [];
656
755
  for (const key of keys) {
@@ -755,6 +854,8 @@ function createMcpServerInstance() {
755
854
  }) }] };
756
855
  });
757
856
 
857
+ } // end profile === "full"
858
+
758
859
  return server;
759
860
  }
760
861
 
@@ -769,8 +870,9 @@ app.get("/sse", requireAuth, async (req, res) => {
769
870
  if (Object.keys(transports).length >= MAX_SESSIONS) {
770
871
  return res.status(503).json({ error: "Max concurrent sessions reached" });
771
872
  }
873
+ const profile = req.query.profile === "slim" ? "slim" : "full";
772
874
  const transport = new SSEServerTransport("/messages", res);
773
- const mcpServer = createMcpServerInstance();
875
+ const mcpServer = createMcpServerInstance(profile);
774
876
  const sessionId = transport.sessionId;
775
877
  transports[sessionId] = { transport, server: mcpServer };
776
878
 
@@ -819,18 +921,27 @@ app.get("/health", async (req, res) => {
819
921
  });
820
922
 
821
923
  app.get("/api/tools", requireAuth, (req, res) => {
822
- res.json({ tools: [
924
+ const profile = req.query.profile === "slim" ? "slim" : "full";
925
+ const core = [
823
926
  "cache_set", "cache_get", "cache_delete", "redis_stats", "health_check",
824
927
  "obsidian_write_note", "obsidian_read_note", "obsidian_delete_note",
825
928
  "obsidian_search_notes", "obsidian_list_notes",
826
929
  "heurchain_search", "heurchain_export",
930
+ "proc_set", "proc_get"
931
+ ];
932
+ const infra = [
827
933
  "prometheus_get_targets", "prometheus_get_alerts", "prometheus_query",
828
934
  "grafana_get_health", "grafana_list_dashboards",
829
935
  "proxmox_get_cluster_status", "proxmox_list_nodes", "proxmox_find_vm",
830
936
  "ceph_get_health_status", "ceph_list_osd_notes",
831
937
  "network_search_docs", "network_get_runbook",
832
938
  "user_context_add_entry", "user_context_get_history", "user_context_search_history"
833
- ]});
939
+ ];
940
+ res.json({
941
+ profile,
942
+ tools: profile === "slim" ? core : [...core, ...infra],
943
+ rest: ["/api/search", "/api/buffer", "/api/proc", "/api/session-context"]
944
+ });
834
945
  });
835
946
 
836
947
  app.post("/api/cache", requireAuth, jsonParser, async (req, res) => {
@@ -897,6 +1008,140 @@ app.get("/api/obsidian/search", requireAuth, async (req, res) => {
897
1008
  res.json({ count: results.length, results: results.slice(0, max).map(r => ({ key: r.key, path: r.path, preview: r.content.substring(0, 200) })) });
898
1009
  });
899
1010
 
1011
+ // ---------------------------------------------------------------------------
1012
+ // BM25 search REST — frictionless recall via curl/hook, no MCP tool overhead
1013
+ // ---------------------------------------------------------------------------
1014
+
1015
+ app.get("/api/search", requireAuth, async (req, res) => {
1016
+ const { q, limit, tier, minScore, namespace } = req.query;
1017
+ if (!q) return res.status(400).json({ error: "q required" });
1018
+ const results = bm25.search(q, {
1019
+ limit: Math.min(parseInt(limit) || 10, 50),
1020
+ tier: tier || "all",
1021
+ minScore: parseFloat(minScore) || 0,
1022
+ noiseFilter: true,
1023
+ namespace: namespace || null
1024
+ });
1025
+ res.json({
1026
+ count: results.length,
1027
+ index_size: bm25.size,
1028
+ results: results.map(r => ({
1029
+ key: r.key,
1030
+ score: Math.round(r.score * 1000) / 1000,
1031
+ tier: r.tier,
1032
+ preview: (r.content || "").substring(0, 300)
1033
+ }))
1034
+ });
1035
+ });
1036
+
1037
+ // ---------------------------------------------------------------------------
1038
+ // Session buffer — ambient turn capture, POST fire-and-forget from agent hook
1039
+ // ---------------------------------------------------------------------------
1040
+
1041
+ app.post("/api/buffer", requireAuth, jsonParser, async (req, res) => {
1042
+ const { session_id, agent_name, role, content } = req.body;
1043
+ if (!session_id || !content) return res.status(400).json({ error: "session_id and content required" });
1044
+ bufferAppend(session_id, agent_name, role, content);
1045
+ res.json({ buffered: true, session_id });
1046
+ });
1047
+
1048
+ app.post("/api/buffer/flush", requireAuth, jsonParser, async (req, res) => {
1049
+ const { session_id } = req.body;
1050
+ if (!session_id) return res.status(400).json({ error: "session_id required" });
1051
+ await flushBuffer(session_id);
1052
+ res.json({ flushed: true, session_id });
1053
+ });
1054
+
1055
+ // ---------------------------------------------------------------------------
1056
+ // Procedural memory REST — stable agent preference knobs, no TTL
1057
+ // ---------------------------------------------------------------------------
1058
+
1059
+ app.post("/api/proc", requireAuth, jsonParser, async (req, res) => {
1060
+ const { agent_name, key, value } = req.body;
1061
+ if (!agent_name || !key || value === undefined) return res.status(400).json({ error: "agent_name, key, value required" });
1062
+ try {
1063
+ const rk = procKey(agent_name, key);
1064
+ await redis.set(rk, typeof value === "string" ? value : JSON.stringify(value));
1065
+ res.json({ stored: true, key: rk });
1066
+ } catch (e) {
1067
+ res.status(400).json({ error: e.message });
1068
+ }
1069
+ });
1070
+
1071
+ app.get("/api/proc/:agent", requireAuth, async (req, res) => {
1072
+ try {
1073
+ if (!PROC_KEY_RE.test(req.params.agent)) return res.status(400).json({ error: "invalid agent name" });
1074
+ res.json(await procGet(req.params.agent));
1075
+ } catch (e) {
1076
+ res.status(500).json({ error: "proc read failed" });
1077
+ }
1078
+ });
1079
+
1080
+ app.delete("/api/proc/:agent/:key", requireAuth, async (req, res) => {
1081
+ try {
1082
+ const rk = procKey(req.params.agent, req.params.key);
1083
+ const deleted = await redis.del(rk);
1084
+ res.json({ deleted: deleted > 0, key: rk });
1085
+ } catch (e) {
1086
+ res.status(400).json({ error: e.message });
1087
+ }
1088
+ });
1089
+
1090
+ // ---------------------------------------------------------------------------
1091
+ // Session-context injection — compact recall for system-prompt injection.
1092
+ // Returns proc knobs + last session summary + top BM25 hits, hard-capped at
1093
+ // SESSION_CONTEXT_TOKENS (~400 tokens). Call via hook at session start;
1094
+ // inject the JSON into system prompt. Zero MCP tool overhead.
1095
+ // ---------------------------------------------------------------------------
1096
+
1097
+ app.get("/api/session-context", requireAuth, async (req, res) => {
1098
+ const { agent, hint } = req.query;
1099
+ if (!agent) return res.status(400).json({ error: "agent required" });
1100
+ if (!PROC_KEY_RE.test(agent)) return res.status(400).json({ error: "invalid agent name" });
1101
+
1102
+ // 1. Procedural knobs
1103
+ let proc = {};
1104
+ try { proc = await procGet(agent); } catch {}
1105
+
1106
+ // 2. Last session summary — scan for most recent buffer entry or session record
1107
+ let last_session = null;
1108
+ try {
1109
+ const candidates = [];
1110
+ for await (const k of scanKeys("session:*:buffer", "string")) {
1111
+ const raw = await redis.get(k);
1112
+ if (!raw) continue;
1113
+ try {
1114
+ const turns = JSON.parse(raw);
1115
+ const agentTurns = turns.filter(t => t.agent === agent);
1116
+ if (agentTurns.length === 0) continue;
1117
+ const last = agentTurns[agentTurns.length - 1];
1118
+ candidates.push({ ts: last.ts, summary: last.content.substring(0, 200) });
1119
+ } catch {}
1120
+ }
1121
+ if (candidates.length > 0) {
1122
+ candidates.sort((a, b) => b.ts.localeCompare(a.ts));
1123
+ last_session = candidates[0].summary;
1124
+ }
1125
+ } catch {}
1126
+
1127
+ // 3. BM25 relevant results, token-budgeted
1128
+ let relevant = [];
1129
+ if (hint) {
1130
+ try {
1131
+ const hits = bm25.search(hint, { limit: 15, minScore: 0.5, noiseFilter: true });
1132
+ let budget = SESSION_CONTEXT_TOKENS - JSON.stringify(proc).length / 4 - 50;
1133
+ for (const r of hits) {
1134
+ const excerpt = (r.content || "").substring(0, 200);
1135
+ budget -= Math.ceil((r.key.length + excerpt.length) / 4);
1136
+ if (budget < 0) break;
1137
+ relevant.push({ key: r.key, score: Math.round(r.score * 100) / 100, excerpt });
1138
+ }
1139
+ } catch {}
1140
+ }
1141
+
1142
+ res.json({ proc, last_session, relevant });
1143
+ });
1144
+
900
1145
  // ---------------------------------------------------------------------------
901
1146
  // Startup
902
1147
  // ---------------------------------------------------------------------------
@@ -904,7 +1149,13 @@ app.get("/api/obsidian/search", requireAuth, async (req, res) => {
904
1149
  (async () => {
905
1150
  await redis.connect();
906
1151
  await buildIndex();
907
- setInterval(buildIndex, 60 * 60 * 1000); // hourly rebuild
1152
+ setInterval(buildIndex, 60 * 60 * 1000); // hourly BM25 rebuild
1153
+ // Flush all active session buffers on heartbeat
1154
+ setInterval(async () => {
1155
+ for (const sessionId of sessionBuffers.keys()) {
1156
+ await flushBuffer(sessionId).catch(() => {});
1157
+ }
1158
+ }, BUFFER_FLUSH_MS);
908
1159
  const port = parseInt(process.env.MCP_PORT, 10) || 3010;
909
- app.listen(port, () => console.log(`HeurChain MCP v1.2.1 listening on port ${port} — ${bm25.size} docs indexed`));
1160
+ app.listen(port, () => console.log(`HeurChain MCP v1.3.0 listening on port ${port} — ${bm25.size} docs indexed`));
910
1161
  })();