@danielsimonjr/memoryjs 1.9.0 → 1.9.1

package/dist/cli/index.js

@@ -15984,7 +15984,7 @@ var init_ContextWindowManager = __esm({
     init_esm_shims();
     init_agent_memory();
     init_ContextProfileManager();
-    ContextWindowManager = class {
+    ContextWindowManager = class _ContextWindowManager {
       storage;
       salienceEngine;
       config;
@@ -16751,9 +16751,249 @@ var init_ContextWindowManager = __esm({
             console.error("[ContextWindowManager.wakeUp] L1 entity loading failed:", err);
           }
         }
+        if (options.compress && l1) {
+          try {
+            const level = typeof options.compress === "string" ? options.compress : "medium";
+            const compressResult = this.compressForContext(l1, { level });
+            if (compressResult.stats.savedTokens > 0) {
+              l1 = compressResult.compressed;
+            }
+          } catch (err) {
+            console.error("[ContextWindowManager.wakeUp] Compression failed, using uncompressed:", err);
+          }
+        }
         const totalTokens = this.estimateStringTokens(l0) + this.estimateStringTokens(l1);
         return { l0, l1, totalTokens, entityCount };
       }
+      // ==================== Context Compression ====================
+      /** Unicode abbreviation map for aggressive-level compression (code keywords). */
+      static COMMON_PATTERNS = {
+        "function ": "\u0192 ",
+        "return ": "\u0280 ",
+        "const ": "\u1D04 ",
+        "export ": "\u1D07 ",
+        "import ": "\u026A ",
+        "interface ": "\u026A\u0274\u1D1B ",
+        "class ": "\u1D04\u029Fs ",
+        "async ": "\u1D00 ",
+        "await ": "\u1D00\u1D21 ",
+        "undefined": "\u1D1C\u0274\u1D05",
+        "null": "\u0274\u1D1C\u029F",
+        "true": "\u1D1B",
+        "false": "\uA730",
+        "```typescript": "```ts",
+        "```javascript": "```js",
+        "## ": "\u2E2B ",
+        "### ": "\u2E2C ",
+        "#### ": "\u2E2D ",
+        '"description"': '"desc"',
+        '"dependencies"': '"deps"',
+        '"devDependencies"': '"devDeps"',
+        '"repository"': '"repo"',
+        '"homepage"': '"home"',
+        '"keywords"': '"keys"',
+        '"license"': '"lic"',
+        '"version"': '"ver"',
+        '"required"': '"req"',
+        '"optional"': '"opt"',
+        '"default"': '"def"',
+        '"example"': '"ex"',
+        '"properties"': '"props"',
+        '"additionalProperties"': '"addProps"',
+        "node_modules/": "nm/",
+        "src/": "s/",
+        "dist/": "d/",
+        "test/": "t/",
+        "tests/": "t/",
+        ".typescript": ".ts",
+        ".javascript": ".js"
+      };
+      /**
+       * Compress text for token-efficient context loading.
+       * Finds repeated substrings, replaces with paragraph-sign codes, generates a legend.
+       * Inspired by the CTON compress-for-context tool.
+       */
+      compressForContext(text, options) {
+        const level = options?.level ?? "medium";
+        let compressed = text;
+        const legend = {};
+        if (level !== "light") {
+          compressed = compressed.replace(/[ \t]+/g, " ");
+          compressed = compressed.replace(/\n{3,}/g, "\n\n");
+        }
+        if (level === "aggressive") {
+          const patternResult = this.applyCommonPatterns(compressed);
+          compressed = patternResult.text;
+          Object.assign(legend, patternResult.legend);
+        }
+        const minLength2 = level === "light" ? 8 : level === "medium" ? 6 : 5;
+        const minOccurrences = level === "light" ? 4 : 3;
+        const maxSubstrings = level === "light" ? 20 : level === "medium" ? 30 : 36;
+        const substrings = this.findRepeatedSubstrings(compressed, minLength2, minOccurrences, maxSubstrings);
+        if (substrings.length > 0) {
+          const totalSavings = substrings.reduce((sum, s) => sum + s.savings, 0);
+          if (totalSavings > 5) {
+            const result = this.applySubstringCompression(compressed, substrings);
+            Object.assign(legend, result.legend);
+            compressed = result.compressed;
+          }
+        }
+        if (Object.keys(legend).length > 0) {
+          const legendStr = "=== Legend ===\n" + Object.entries(legend).map(([a, f]) => `${a} = ${f}`).join("\n") + "\n=============\n\n";
+          compressed = legendStr + compressed;
+        }
+        const originalTokens = this.estimateStringTokens(text);
+        const hasLegend = Object.keys(legend).length > 0;
+        const compressedTokens = hasLegend ? this.estimateStringTokens(compressed) : originalTokens;
+        const savedTokens = Math.max(0, originalTokens - compressedTokens);
+        return {
+          compressed: hasLegend ? compressed : text,
+          legend,
+          stats: {
+            originalTokens,
+            compressedTokens,
+            savedTokens,
+            savedPercent: originalTokens > 0 ? Math.round(savedTokens / originalTokens * 100) : 0
+          }
+        };
+      }
+      /**
+       * Compress entities for context loading. Formats entities as compact text,
+       * then applies compression. Sorted by importance descending.
+       */
+      compressEntitiesForContext(entities, options) {
+        const sorted = [...entities].filter((e) => e.entityType !== "profile" && e.entityType !== "diary").sort((a, b) => (b.importance ?? 0) - (a.importance ?? 0));
+        const lines = [];
+        let tokenCount = 0;
+        let entityCount = 0;
+        const maxTokens = options?.maxTokens ?? Infinity;
+        for (const e of sorted) {
+          const obs = e.observations?.slice(0, 3).join("; ") ?? "";
+          const line = `[${e.entityType}] ${e.name}: ${obs}`;
+          const lineTokens = this.estimateStringTokens(line);
+          if (tokenCount + lineTokens > maxTokens) break;
+          lines.push(line);
+          tokenCount += lineTokens;
+          entityCount++;
+        }
+        const raw = lines.join("\n");
+        const result = this.compressForContext(raw, { level: options?.level });
+        return { ...result, entityCount };
+      }
+      // ==================== Compression Helpers ====================
+      /**
+       * Find repeated substrings and calculate compression potential.
+       * Returns substrings sorted by net savings (highest first).
+       * @internal
+       */
+      findRepeatedSubstrings(text, minLength2, minOccurrences, maxSubstrings = 36) {
+        if (text.length < minLength2 * 2) return [];
+        const substringCounts = /* @__PURE__ */ new Map();
+        const MAX_MAP_SIZE = 1e4;
+        const tokens = text.split(/(\s+|[{}()\[\]<>:;,."'`|=])/);
+        outer: for (let n = 1; n <= 6; n++) {
+          for (let i = 0; i <= tokens.length - n; i++) {
+            const ngram = tokens.slice(i, i + n).join("");
+            if (ngram.length < minLength2 || ngram.length > 50) continue;
+            if (/^\s*$/.test(ngram)) continue;
+            if ((ngram.match(/\s/g) || []).length > ngram.length * 0.5) continue;
+            const opens = (ngram.match(/[{(\[<]/g) || []).length;
+            const closes = (ngram.match(/[})\]>]/g) || []).length;
+            if (opens !== closes) continue;
+            substringCounts.set(ngram, (substringCounts.get(ngram) || 0) + 1);
+            if (substringCounts.size >= MAX_MAP_SIZE) break outer;
+          }
+        }
+        const pathRe = /[a-zA-Z0-9_\-./]+\/[a-zA-Z0-9_\-./]+/g;
+        let pathMatch;
+        while ((pathMatch = pathRe.exec(text)) !== null) {
+          const p = pathMatch[0];
+          if (p.length >= minLength2 && substringCounts.size < MAX_MAP_SIZE) {
+            substringCounts.set(p, (substringCounts.get(p) || 0) + 1);
+          }
+        }
+        const candidates = [];
+        for (const [substring, _mapCount] of substringCounts.entries()) {
+          const actualCount = text.split(substring).length - 1;
+          if (actualCount >= minOccurrences) {
+            const abbrevLength = 2;
+            const legendCost = abbrevLength + substring.length + 4;
+            const savingsPerOccurrence = substring.length - abbrevLength;
+            const netSavings = savingsPerOccurrence * actualCount - legendCost;
+            if (netSavings > 5) {
+              candidates.push({ substring, count: actualCount, savings: netSavings });
+            }
+          }
+        }
+        candidates.sort((a, b) => b.savings - a.savings);
+        const selected = [];
+        const usedSubstrings = [];
+        for (const candidate of candidates) {
+          let isTooSimilar = false;
+          const candidateTrimmed = candidate.substring.trim();
+          if (candidateTrimmed.length < 3) continue;
+          for (const used of usedSubstrings) {
+            const usedTrimmed = used.trim();
+            if (used.includes(candidate.substring) || candidate.substring.includes(used)) {
+              isTooSimilar = true;
+              break;
+            }
+            if (candidateTrimmed === usedTrimmed || candidateTrimmed.includes(usedTrimmed) || usedTrimmed.includes(candidateTrimmed)) {
+              isTooSimilar = true;
+              break;
+            }
+            const shorter = candidateTrimmed.length < usedTrimmed.length ? candidateTrimmed : usedTrimmed;
+            const longer = candidateTrimmed.length >= usedTrimmed.length ? candidateTrimmed : usedTrimmed;
+            if (longer.includes(shorter.slice(0, Math.floor(shorter.length * 0.7)))) {
+              isTooSimilar = true;
+              break;
+            }
+          }
+          if (!isTooSimilar) {
+            selected.push(candidate);
+            usedSubstrings.push(candidate.substring);
+            if (selected.length >= maxSubstrings) break;
+          }
+        }
+        return selected;
+      }
+      /**
+       * Apply substring replacements to text.
+       * Applies replacements in savings-descending order (as returned by findRepeatedSubstrings).
+       * @internal
+       */
+      applySubstringCompression(text, substrings) {
+        const legend = {};
+        let compressed = text;
+        substrings.forEach((item, index) => {
+          const abbrev = `\xA7${index.toString(36)}`;
+          legend[abbrev] = item.substring;
+          compressed = compressed.split(item.substring).join(abbrev);
+        });
+        return { compressed, legend };
+      }
+      /**
+       * Apply COMMON_PATTERNS unicode abbreviations (aggressive level only).
+       * @internal
+       */
+      applyCommonPatterns(text) {
+        let result = text;
+        const legend = {};
+        for (const [pattern2, replacement] of Object.entries(_ContextWindowManager.COMMON_PATTERNS)) {
+          try {
+            if (!result.includes(pattern2)) continue;
+            const count = result.split(pattern2).length - 1;
+            const savings = (pattern2.length - replacement.length) * count;
+            if (savings > pattern2.length + replacement.length + 5) {
+              legend[replacement] = pattern2;
+              result = result.split(pattern2).join(replacement);
+            }
+          } catch {
+            continue;
+          }
+        }
+        return { text: result, legend };
+      }
     };
   }
 });