@alaarab/cortex 1.13.5 → 1.13.6

package/mcp/dist/shared-search-fallback.js

@@ -14,6 +14,26 @@ let dfCacheGeneration = 0;
 export function invalidateDfCache() {
     dfCacheGeneration++;
     dfCache.clear();
+    tokenCache.clear();
+}
+// Module-level cache for tokenized document content.
+// Keyed by a short content hash so the same document content is only tokenized once per server lifetime.
+// Cleared on full rebuild (same lifecycle as dfCache). Max 2000 entries to bound memory.
+const MAX_TOKEN_CACHE = 2000;
+const tokenCache = new Map();
+function cachedTokenize(text) {
+    // Use first 16 chars + length as a cheap key (avoids hashing cost for most cases)
+    const key = `${text.length}:${text.slice(0, 32)}`;
+    const hit = tokenCache.get(key);
+    if (hit)
+        return hit;
+    const tokens = tokenize(text);
+    if (tokenCache.size >= MAX_TOKEN_CACHE) {
+        // Evict oldest entry
+        tokenCache.delete(tokenCache.keys().next().value ?? "");
+    }
+    tokenCache.set(key, tokens);
+    return tokens;
 }
 /**
  * Tokenize text into non-stop-word tokens for TF-IDF computation, with stemming.
@@ -34,10 +54,10 @@ function tfidfCosine(docs, query) {
     const queryTokens = tokenize(query);
     if (queryTokens.length === 0)
         return docs.map(() => 0);
-    // Collect all unique terms from query + all docs
+    // Collect all unique terms from query + all docs (use cached tokenization for repeated content)
     const allTokens = new Set(queryTokens);
     const docTokenLists = docs.map(d => {
-        const tokens = tokenize(d);
+        const tokens = cachedTokenize(d);
         for (const t of tokens)
             allTokens.add(t);
         return tokens;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@alaarab/cortex",
-  "version": "1.13.5",
+  "version": "1.13.6",
   "description": "Long-term memory for AI agents — stored as markdown in a git repo you own.",
   "type": "module",
   "bin": {