@ulrichc1/sparn 1.2.2 → 1.4.0

package/dist/mcp/index.js

@@ -1,11 +1,71 @@
 #!/usr/bin/env node
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+// node_modules/tsup/assets/esm_shims.js
+import path from "path";
+import { fileURLToPath } from "url";
+var init_esm_shims = __esm({
+  "node_modules/tsup/assets/esm_shims.js"() {
+    "use strict";
+  }
+});
+
+// src/utils/tokenizer.ts
+var tokenizer_exports = {};
+__export(tokenizer_exports, {
+  countTokensPrecise: () => countTokensPrecise,
+  estimateTokens: () => estimateTokens,
+  setPreciseTokenCounting: () => setPreciseTokenCounting
+});
+import { encode } from "gpt-tokenizer";
+function setPreciseTokenCounting(enabled) {
+  usePrecise = enabled;
+}
+function countTokensPrecise(text) {
+  if (!text || text.length === 0) {
+    return 0;
+  }
+  return encode(text).length;
+}
+function estimateTokens(text) {
+  if (!text || text.length === 0) {
+    return 0;
+  }
+  if (usePrecise) {
+    return encode(text).length;
+  }
+  const words = text.split(/\s+/).filter((w) => w.length > 0);
+  const wordCount = words.length;
+  const charCount = text.length;
+  const charEstimate = Math.ceil(charCount / 4);
+  const wordEstimate = Math.ceil(wordCount * 0.75);
+  return Math.max(wordEstimate, charEstimate);
+}
+var usePrecise;
+var init_tokenizer = __esm({
+  "src/utils/tokenizer.ts"() {
+    "use strict";
+    init_esm_shims();
+    usePrecise = false;
+  }
+});
 
 // src/mcp/index.ts
+init_esm_shims();
 import { mkdirSync } from "fs";
 import { dirname, resolve } from "path";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 
 // src/core/kv-memory.ts
+init_esm_shims();
 import { copyFileSync, existsSync } from "fs";
 import Database from "better-sqlite3";
 function createBackup(dbPath) {
@@ -27,6 +87,7 @@ async function createKVMemory(dbPath) {
     const integrityCheck = db.pragma("quick_check", { simple: true });
     if (integrityCheck !== "ok") {
       console.error("\u26A0 Database corruption detected!");
+      db.close();
       if (existsSync(dbPath)) {
         const backupPath = createBackup(dbPath);
         if (backupPath) {
@@ -34,7 +95,6 @@ async function createKVMemory(dbPath) {
         }
       }
       console.log("Attempting database recovery...");
-      db.close();
       db = new Database(dbPath);
     }
   } catch (error) {
@@ -46,6 +106,7 @@ async function createKVMemory(dbPath) {
     db = new Database(dbPath);
   }
   db.pragma("journal_mode = WAL");
+  db.pragma("foreign_keys = ON");
   db.exec(`
     CREATE TABLE IF NOT EXISTS entries_index (
       id TEXT PRIMARY KEY NOT NULL,
@@ -85,6 +146,36 @@ async function createKVMemory(dbPath) {
     CREATE INDEX IF NOT EXISTS idx_entries_timestamp ON entries_index(timestamp DESC);
     CREATE INDEX IF NOT EXISTS idx_stats_timestamp ON optimization_stats(timestamp DESC);
   `);
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5(id, content, tokenize='porter');
+  `);
+  db.exec(`
+    CREATE TRIGGER IF NOT EXISTS entries_fts_insert
+    AFTER INSERT ON entries_value
+    BEGIN
+      INSERT OR REPLACE INTO entries_fts(id, content) VALUES (NEW.id, NEW.content);
+    END;
+  `);
+  db.exec(`
+    CREATE TRIGGER IF NOT EXISTS entries_fts_delete
+    AFTER DELETE ON entries_value
+    BEGIN
+      DELETE FROM entries_fts WHERE id = OLD.id;
+    END;
+  `);
+  db.exec(`
+    CREATE TRIGGER IF NOT EXISTS entries_fts_update
+    AFTER UPDATE ON entries_value
+    BEGIN
+      DELETE FROM entries_fts WHERE id = OLD.id;
+      INSERT INTO entries_fts(id, content) VALUES (NEW.id, NEW.content);
+    END;
+  `);
+  db.exec(`
+    INSERT OR IGNORE INTO entries_fts(id, content)
+    SELECT id, content FROM entries_value
+    WHERE id NOT IN (SELECT id FROM entries_fts);
+  `);
   const putIndexStmt = db.prepare(`
     INSERT OR REPLACE INTO entries_index
     (id, hash, timestamp, score, ttl, state, accessCount, isBTSP)
@@ -173,14 +264,20 @@ async function createKVMemory(dbPath) {
         sql += " AND i.isBTSP = ?";
         params.push(filters.isBTSP ? 1 : 0);
       }
+      if (filters.tags && filters.tags.length > 0) {
+        for (const tag of filters.tags) {
+          sql += " AND v.tags LIKE ?";
+          params.push(`%"${tag}"%`);
+        }
+      }
       sql += " ORDER BY i.score DESC";
       if (filters.limit) {
         sql += " LIMIT ?";
         params.push(filters.limit);
-      }
-      if (filters.offset) {
-        sql += " OFFSET ?";
-        params.push(filters.offset);
+        if (filters.offset) {
+          sql += " OFFSET ?";
+          params.push(filters.offset);
+        }
       }
       const stmt = db.prepare(sql);
       const rows = stmt.all(...params);
@@ -215,7 +312,22 @@ async function createKVMemory(dbPath) {
     },
     async compact() {
       const before = db.prepare("SELECT COUNT(*) as count FROM entries_index").get();
-      db.exec("DELETE FROM entries_index WHERE ttl <= 0");
+      const now = Date.now();
+      db.prepare("DELETE FROM entries_index WHERE isBTSP = 0 AND (timestamp + ttl * 1000) < ?").run(
+        now
+      );
+      db.exec("DELETE FROM entries_index WHERE isBTSP = 0 AND ttl <= 0");
+      const candidates = db.prepare("SELECT id, timestamp, ttl FROM entries_index WHERE isBTSP = 0").all();
+      for (const row of candidates) {
+        const ageSeconds = Math.max(0, (now - row.timestamp) / 1e3);
+        const ttlSeconds = row.ttl;
+        if (ttlSeconds <= 0) continue;
+        const decay = 1 - Math.exp(-ageSeconds / ttlSeconds);
+        if (decay >= 0.95) {
+          db.prepare("DELETE FROM entries_index WHERE id = ?").run(row.id);
+        }
+      }
+      db.exec("DELETE FROM entries_value WHERE id NOT IN (SELECT id FROM entries_index)");
       db.exec("VACUUM");
       const after = db.prepare("SELECT COUNT(*) as count FROM entries_index").get();
       return before.count - after.count;
@@ -235,6 +347,9 @@ async function createKVMemory(dbPath) {
         stats.entries_pruned,
         stats.duration_ms
       );
+      db.prepare(
+        "DELETE FROM optimization_stats WHERE id NOT IN (SELECT id FROM optimization_stats ORDER BY timestamp DESC LIMIT 1000)"
+      ).run();
     },
     async getOptimizationStats() {
       const stmt = db.prepare(`
@@ -247,28 +362,70 @@ async function createKVMemory(dbPath) {
     },
     async clearOptimizationStats() {
       db.exec("DELETE FROM optimization_stats");
+    },
+    async searchFTS(query, limit = 10) {
+      if (!query || query.trim().length === 0) return [];
+      const sanitized = query.replace(/[{}()[\]"':*^~]/g, " ").trim();
+      if (sanitized.length === 0) return [];
+      const stmt = db.prepare(`
+        SELECT
+          f.id, f.content, rank,
+          i.hash, i.timestamp, i.score, i.ttl, i.state, i.accessCount, i.isBTSP,
+          v.tags, v.metadata
+        FROM entries_fts f
+        JOIN entries_index i ON f.id = i.id
+        JOIN entries_value v ON f.id = v.id
+        WHERE entries_fts MATCH ?
+        ORDER BY rank
+        LIMIT ?
+      `);
+      try {
+        const rows = stmt.all(sanitized, limit);
+        return rows.map((r) => ({
+          entry: {
+            id: r.id,
+            content: r.content,
+            hash: r.hash,
+            timestamp: r.timestamp,
+            score: r.score,
+            ttl: r.ttl,
+            state: r.state,
+            accessCount: r.accessCount,
+            tags: r.tags ? JSON.parse(r.tags) : [],
+            metadata: r.metadata ? JSON.parse(r.metadata) : {},
+            isBTSP: r.isBTSP === 1
+          },
+          rank: r.rank
+        }));
+      } catch {
+        return [];
+      }
     }
   };
 }
 
 // src/mcp/server.ts
+init_esm_shims();
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { z } from "zod";
 
 // src/adapters/generic.ts
+init_esm_shims();
 import { randomUUID as randomUUID2 } from "crypto";
 
 // src/core/btsp-embedder.ts
+init_esm_shims();
 import { randomUUID } from "crypto";
 
 // src/utils/hash.ts
+init_esm_shims();
 import { createHash } from "crypto";
 function hashContent(content) {
   return createHash("sha256").update(content, "utf8").digest("hex");
 }
 
 // src/core/btsp-embedder.ts
-function createBTSPEmbedder() {
+function createBTSPEmbedder(config) {
   const BTSP_PATTERNS = [
     // Error patterns
     /\b(error|exception|failure|fatal|critical|panic)\b/i,
@@ -287,6 +444,14 @@ function createBTSPEmbedder() {
     /^=======/m,
     /^>>>>>>> /m
   ];
+  if (config?.customPatterns) {
+    for (const pattern of config.customPatterns) {
+      try {
+        BTSP_PATTERNS.push(new RegExp(pattern));
+      } catch {
+      }
+    }
+  }
   function detectBTSP(content) {
     return BTSP_PATTERNS.some((pattern) => pattern.test(content));
   }
@@ -315,13 +480,14 @@ function createBTSPEmbedder() {
 }
 
 // src/core/confidence-states.ts
+init_esm_shims();
 function createConfidenceStates(config) {
   const { activeThreshold, readyThreshold } = config;
   function calculateState(entry) {
     if (entry.isBTSP) {
       return "active";
     }
-    if (entry.score > activeThreshold) {
+    if (entry.score >= activeThreshold) {
       return "active";
     }
     if (entry.score >= readyThreshold) {
@@ -357,8 +523,11 @@ function createConfidenceStates(config) {
 }
 
 // src/core/engram-scorer.ts
+init_esm_shims();
 function createEngramScorer(config) {
   const { defaultTTL } = config;
+  const recencyWindowMs = (config.recencyBoostMinutes ?? 30) * 60 * 1e3;
+  const recencyMultiplier = config.recencyBoostMultiplier ?? 1.3;
   function calculateDecay(ageInSeconds, ttlInSeconds) {
     if (ttlInSeconds === 0) return 1;
     if (ageInSeconds <= 0) return 0;
@@ -378,6 +547,13 @@ function createEngramScorer(config) {
     if (entry.isBTSP) {
       score = Math.max(score, 0.9);
     }
+    if (!entry.isBTSP && recencyWindowMs > 0) {
+      const ageMs = currentTime - entry.timestamp;
+      if (ageMs >= 0 && ageMs < recencyWindowMs) {
+        const boostFactor = 1 + (recencyMultiplier - 1) * (1 - ageMs / recencyWindowMs);
+        score = score * boostFactor;
+      }
+    }
     return Math.max(0, Math.min(1, score));
   }
   function refreshTTL(entry) {
@@ -395,49 +571,53 @@ function createEngramScorer(config) {
   };
 }
 
-// src/utils/tokenizer.ts
-function estimateTokens(text) {
-  if (!text || text.length === 0) {
-    return 0;
+// src/core/sparse-pruner.ts
+init_esm_shims();
+
+// src/utils/tfidf.ts
+init_esm_shims();
+function tokenize(text) {
+  return text.toLowerCase().split(/\s+/).filter((word) => word.length > 0);
+}
+function calculateTF(term, tokens) {
+  const count = tokens.filter((t) => t === term).length;
+  return Math.sqrt(count);
+}
+function createTFIDFIndex(entries) {
+  const documentFrequency = /* @__PURE__ */ new Map();
+  for (const entry of entries) {
+    const tokens = tokenize(entry.content);
+    const uniqueTerms = new Set(tokens);
+    for (const term of uniqueTerms) {
+      documentFrequency.set(term, (documentFrequency.get(term) || 0) + 1);
+    }
   }
-  const words = text.split(/\s+/).filter((w) => w.length > 0);
-  const wordCount = words.length;
-  const charCount = text.length;
-  const charEstimate = Math.ceil(charCount / 4);
-  const wordEstimate = Math.ceil(wordCount * 0.75);
-  return Math.max(wordEstimate, charEstimate);
+  return {
+    documentFrequency,
+    totalDocuments: entries.length
+  };
+}
+function scoreTFIDF(entry, index) {
+  const tokens = tokenize(entry.content);
+  if (tokens.length === 0) return 0;
+  const uniqueTerms = new Set(tokens);
+  let totalScore = 0;
+  for (const term of uniqueTerms) {
+    const tf = calculateTF(term, tokens);
+    const docsWithTerm = index.documentFrequency.get(term) || 0;
+    if (docsWithTerm === 0) continue;
+    const idf = Math.log(index.totalDocuments / docsWithTerm);
+    totalScore += tf * idf;
+  }
+  return totalScore / tokens.length;
 }
 
 // src/core/sparse-pruner.ts
+init_tokenizer();
 function createSparsePruner(config) {
   const { threshold } = config;
-  function tokenize(text) {
-    return text.toLowerCase().split(/\s+/).filter((word) => word.length > 0);
-  }
-  function calculateTF(term, tokens) {
-    const count = tokens.filter((t) => t === term).length;
-    return Math.sqrt(count);
-  }
-  function calculateIDF(term, allEntries) {
-    const totalDocs = allEntries.length;
-    const docsWithTerm = allEntries.filter((entry) => {
-      const tokens = tokenize(entry.content);
-      return tokens.includes(term);
-    }).length;
-    if (docsWithTerm === 0) return 0;
-    return Math.log(totalDocs / docsWithTerm);
-  }
   function scoreEntry(entry, allEntries) {
-    const tokens = tokenize(entry.content);
-    if (tokens.length === 0) return 0;
-    const uniqueTerms = [...new Set(tokens)];
-    let totalScore = 0;
-    for (const term of uniqueTerms) {
-      const tf = calculateTF(term, tokens);
-      const idf = calculateIDF(term, allEntries);
-      totalScore += tf * idf;
-    }
-    return totalScore / tokens.length;
+    return scoreTFIDF(entry, createTFIDFIndex(allEntries));
   }
   function prune(entries) {
     if (entries.length === 0) {
@@ -449,9 +629,10 @@ function createSparsePruner(config) {
       };
     }
     const originalTokens = entries.reduce((sum, e) => sum + estimateTokens(e.content), 0);
+    const tfidfIndex = createTFIDFIndex(entries);
     const scored = entries.map((entry) => ({
       entry,
-      score: scoreEntry(entry, entries)
+      score: scoreTFIDF(entry, tfidfIndex)
     }));
     scored.sort((a, b) => b.score - a.score);
     const keepCount = Math.max(1, Math.ceil(entries.length * (threshold / 100)));
@@ -472,29 +653,33 @@ function createSparsePruner(config) {
 }
 
 // src/adapters/generic.ts
+init_tokenizer();
 function createGenericAdapter(memory, config) {
   const pruner = createSparsePruner(config.pruning);
   const scorer = createEngramScorer(config.decay);
   const states = createConfidenceStates(config.states);
-  const btsp = createBTSPEmbedder();
+  const btsp = createBTSPEmbedder({ customPatterns: config.btspPatterns });
   async function optimize(context, options = {}) {
     const startTime = Date.now();
     const lines = context.split("\n").filter((line) => line.trim().length > 0);
-    const entries = lines.map((content) => ({
-      id: randomUUID2(),
-      content,
-      hash: hashContent(content),
-      timestamp: Date.now(),
-      score: btsp.detectBTSP(content) ? 1 : 0.5,
-      // BTSP gets high initial score
-      ttl: config.decay.defaultTTL * 3600,
-      // Convert hours to seconds
-      state: "ready",
-      accessCount: 0,
-      tags: [],
-      metadata: {},
-      isBTSP: btsp.detectBTSP(content)
-    }));
+    const now = Date.now();
+    const entries = lines.map((content, index) => {
+      const isBTSP = btsp.detectBTSP(content);
+      return {
+        id: randomUUID2(),
+        content,
+        hash: hashContent(content),
+        timestamp: now + index,
+        // Unique timestamps preserve ordering
+        score: isBTSP ? 1 : 0.5,
+        ttl: config.decay.defaultTTL * 3600,
+        state: "ready",
+        accessCount: 0,
+        tags: [],
+        metadata: {},
+        isBTSP
+      };
+    });
     const tokensBefore = entries.reduce((sum, e) => sum + estimateTokens(e.content), 0);
     const scoredEntries = entries.map((entry) => ({
       ...entry,
@@ -547,6 +732,7 @@ function createGenericAdapter(memory, config) {
 }
 
 // src/core/sleep-compressor.ts
+init_esm_shims();
 function createSleepCompressor() {
   const scorer = createEngramScorer({ defaultTTL: 24, decayThreshold: 0.95 });
   function consolidate(entries) {
@@ -637,13 +823,15 @@ function createSleepCompressor() {
   function cosineSimilarity(text1, text2) {
     const words1 = tokenize(text1);
     const words2 = tokenize(text2);
-    const vocab = /* @__PURE__ */ new Set([...words1, ...words2]);
     const vec1 = {};
     const vec2 = {};
-    for (const word of vocab) {
-      vec1[word] = words1.filter((w) => w === word).length;
-      vec2[word] = words2.filter((w) => w === word).length;
+    for (const word of words1) {
+      vec1[word] = (vec1[word] ?? 0) + 1;
+    }
+    for (const word of words2) {
+      vec2[word] = (vec2[word] ?? 0) + 1;
     }
+    const vocab = /* @__PURE__ */ new Set([...words1, ...words2]);
     let dotProduct = 0;
     let mag1 = 0;
     let mag2 = 0;
@@ -659,9 +847,6 @@ function createSleepCompressor() {
     if (mag1 === 0 || mag2 === 0) return 0;
     return dotProduct / (mag1 * mag2);
   }
-  function tokenize(text) {
-    return text.toLowerCase().split(/\s+/).filter((word) => word.length > 0);
-  }
   return {
     consolidate,
     findDuplicates,
@@ -670,6 +855,7 @@ function createSleepCompressor() {
 }
 
 // src/types/config.ts
+init_esm_shims();
 var DEFAULT_CONFIG = {
   pruning: {
     threshold: 5,
@@ -708,11 +894,12 @@ function createSparnMcpServer(options) {
   const { memory, config = DEFAULT_CONFIG } = options;
   const server = new McpServer({
     name: "sparn",
-    version: "1.1.1"
+    version: "1.4.0"
   });
   registerOptimizeTool(server, memory, config);
   registerStatsTool(server, memory);
   registerConsolidateTool(server, memory);
+  registerSearchTool(server, memory);
   return server;
 }
 function registerOptimizeTool(server, memory, config) {
@@ -720,7 +907,7 @@ function registerOptimizeTool(server, memory, config) {
     "sparn_optimize",
     {
       title: "Sparn Optimize",
-      description: "Optimize context using neuroscience-inspired pruning. Applies BTSP detection, engram scoring, confidence states, and sparse pruning to reduce token usage while preserving important information.",
+      description: "Optimize context using multi-stage pruning. Applies critical event detection, relevance scoring, entry classification, and sparse pruning to reduce token usage while preserving important information.",
       inputSchema: {
         context: z.string().describe("The context text to optimize"),
         dryRun: z.boolean().optional().default(false).describe("If true, do not persist changes to the memory store"),
@@ -850,12 +1037,58 @@ function registerStatsTool(server, memory) {
     }
   );
 }
+function registerSearchTool(server, memory) {
+  server.registerTool(
+    "sparn_search",
+    {
+      title: "Sparn Search",
+      description: "Search memory entries using full-text search. Returns matching entries with relevance ranking, score, and state information.",
+      inputSchema: {
+        query: z.string().describe("Search query text"),
+        limit: z.number().int().min(1).max(100).optional().default(10).describe("Maximum number of results (1-100, default 10)")
+      }
+    },
+    async ({ query, limit }) => {
+      try {
+        const results = await memory.searchFTS(query, limit);
+        const response = results.map((r) => ({
+          id: r.entry.id,
+          content: r.entry.content.length > 500 ? `${r.entry.content.slice(0, 500)}...` : r.entry.content,
+          score: r.entry.score,
+          state: r.entry.state,
+          rank: r.rank,
+          tags: r.entry.tags,
+          isBTSP: r.entry.isBTSP
+        }));
+        return {
+          content: [
+            {
+              type: "text",
+              text: JSON.stringify({ results: response, total: response.length }, null, 2)
+            }
+          ]
+        };
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return {
+          content: [
+            {
+              type: "text",
+              text: JSON.stringify({ error: message })
+            }
+          ],
+          isError: true
+        };
+      }
+    }
+  );
+}
 function registerConsolidateTool(server, memory) {
   server.registerTool(
     "sparn_consolidate",
     {
       title: "Sparn Consolidate",
-      description: "Run memory consolidation (sleep replay). Removes decayed entries and merges duplicates to reclaim space. Inspired by the neuroscience principle of sleep-based memory consolidation."
+      description: "Run memory consolidation. Removes decayed entries and merges duplicates to reclaim space."
     },
     async () => {
       try {
@@ -911,6 +1144,10 @@ function registerConsolidateTool(server, memory) {
 
 // src/mcp/index.ts
 async function main() {
+  if (process.env["SPARN_PRECISE_TOKENS"] === "true") {
+    const { setPreciseTokenCounting: setPreciseTokenCounting2 } = await Promise.resolve().then(() => (init_tokenizer(), tokenizer_exports));
+    setPreciseTokenCounting2(true);
+  }
   const dbPath = resolve(process.env["SPARN_DB_PATH"] ?? ".sparn/memory.db");
   mkdirSync(dirname(dbPath), { recursive: true });
   const memory = await createKVMemory(dbPath);