@danielsimonjr/memoryjs 1.9.0 → 1.9.1

package/dist/index.cjs

@@ -23214,7 +23214,7 @@ var ContextProfileManager = class {
 };
 
 // src/agent/ContextWindowManager.ts
-var ContextWindowManager = class {
+var ContextWindowManager = class _ContextWindowManager {
   storage;
   salienceEngine;
   config;
@@ -23981,9 +23981,249 @@ var ContextWindowManager = class {
         console.error("[ContextWindowManager.wakeUp] L1 entity loading failed:", err);
       }
     }
+    if (options.compress && l1) {
+      try {
+        const level = typeof options.compress === "string" ? options.compress : "medium";
+        const compressResult = this.compressForContext(l1, { level });
+        if (compressResult.stats.savedTokens > 0) {
+          l1 = compressResult.compressed;
+        }
+      } catch (err) {
+        console.error("[ContextWindowManager.wakeUp] Compression failed, using uncompressed:", err);
+      }
+    }
     const totalTokens = this.estimateStringTokens(l0) + this.estimateStringTokens(l1);
     return { l0, l1, totalTokens, entityCount };
   }
+  // ==================== Context Compression ====================
+  /** Unicode abbreviation map for aggressive-level compression (code keywords). */
+  static COMMON_PATTERNS = {
+    "function ": "\u0192 ",
+    "return ": "\u0280 ",
+    "const ": "\u1D04 ",
+    "export ": "\u1D07 ",
+    "import ": "\u026A ",
+    "interface ": "\u026A\u0274\u1D1B ",
+    "class ": "\u1D04\u029Fs ",
+    "async ": "\u1D00 ",
+    "await ": "\u1D00\u1D21 ",
+    "undefined": "\u1D1C\u0274\u1D05",
+    "null": "\u0274\u1D1C\u029F",
+    "true": "\u1D1B",
+    "false": "\uA730",
+    "```typescript": "```ts",
+    "```javascript": "```js",
+    "## ": "\u2E2B ",
+    "### ": "\u2E2C ",
+    "#### ": "\u2E2D ",
+    '"description"': '"desc"',
+    '"dependencies"': '"deps"',
+    '"devDependencies"': '"devDeps"',
+    '"repository"': '"repo"',
+    '"homepage"': '"home"',
+    '"keywords"': '"keys"',
+    '"license"': '"lic"',
+    '"version"': '"ver"',
+    '"required"': '"req"',
+    '"optional"': '"opt"',
+    '"default"': '"def"',
+    '"example"': '"ex"',
+    '"properties"': '"props"',
+    '"additionalProperties"': '"addProps"',
+    "node_modules/": "nm/",
+    "src/": "s/",
+    "dist/": "d/",
+    "test/": "t/",
+    "tests/": "t/",
+    ".typescript": ".ts",
+    ".javascript": ".js"
+  };
+  /**
+   * Compress text for token-efficient context loading.
+   * Finds repeated substrings, replaces with paragraph-sign codes, generates a legend.
+   * Inspired by the CTON compress-for-context tool.
+   */
+  compressForContext(text, options) {
+    const level = options?.level ?? "medium";
+    let compressed = text;
+    const legend = {};
+    if (level !== "light") {
+      compressed = compressed.replace(/[ \t]+/g, " ");
+      compressed = compressed.replace(/\n{3,}/g, "\n\n");
+    }
+    if (level === "aggressive") {
+      const patternResult = this.applyCommonPatterns(compressed);
+      compressed = patternResult.text;
+      Object.assign(legend, patternResult.legend);
+    }
+    const minLength2 = level === "light" ? 8 : level === "medium" ? 6 : 5;
+    const minOccurrences = level === "light" ? 4 : 3;
+    const maxSubstrings = level === "light" ? 20 : level === "medium" ? 30 : 36;
+    const substrings = this.findRepeatedSubstrings(compressed, minLength2, minOccurrences, maxSubstrings);
+    if (substrings.length > 0) {
+      const totalSavings = substrings.reduce((sum, s) => sum + s.savings, 0);
+      if (totalSavings > 5) {
+        const result = this.applySubstringCompression(compressed, substrings);
+        Object.assign(legend, result.legend);
+        compressed = result.compressed;
+      }
+    }
+    if (Object.keys(legend).length > 0) {
+      const legendStr = "=== Legend ===\n" + Object.entries(legend).map(([a, f]) => `${a} = ${f}`).join("\n") + "\n=============\n\n";
+      compressed = legendStr + compressed;
+    }
+    const originalTokens = this.estimateStringTokens(text);
+    const hasLegend = Object.keys(legend).length > 0;
+    const compressedTokens = hasLegend ? this.estimateStringTokens(compressed) : originalTokens;
+    const savedTokens = Math.max(0, originalTokens - compressedTokens);
+    return {
+      compressed: hasLegend ? compressed : text,
+      legend,
+      stats: {
+        originalTokens,
+        compressedTokens,
+        savedTokens,
+        savedPercent: originalTokens > 0 ? Math.round(savedTokens / originalTokens * 100) : 0
+      }
+    };
+  }
+  /**
+   * Compress entities for context loading. Formats entities as compact text,
+   * then applies compression. Sorted by importance descending.
+   */
+  compressEntitiesForContext(entities, options) {
+    const sorted = [...entities].filter((e) => e.entityType !== "profile" && e.entityType !== "diary").sort((a, b) => (b.importance ?? 0) - (a.importance ?? 0));
+    const lines = [];
+    let tokenCount = 0;
+    let entityCount = 0;
+    const maxTokens = options?.maxTokens ?? Infinity;
+    for (const e of sorted) {
+      const obs = e.observations?.slice(0, 3).join("; ") ?? "";
+      const line = `[${e.entityType}] ${e.name}: ${obs}`;
+      const lineTokens = this.estimateStringTokens(line);
+      if (tokenCount + lineTokens > maxTokens) break;
+      lines.push(line);
+      tokenCount += lineTokens;
+      entityCount++;
+    }
+    const raw = lines.join("\n");
+    const result = this.compressForContext(raw, { level: options?.level });
+    return { ...result, entityCount };
+  }
+  // ==================== Compression Helpers ====================
+  /**
+   * Find repeated substrings and calculate compression potential.
+   * Returns substrings sorted by net savings (highest first).
+   * @internal
+   */
+  findRepeatedSubstrings(text, minLength2, minOccurrences, maxSubstrings = 36) {
+    if (text.length < minLength2 * 2) return [];
+    const substringCounts = /* @__PURE__ */ new Map();
+    const MAX_MAP_SIZE = 1e4;
+    const tokens = text.split(/(\s+|[{}()\[\]<>:;,."'`|=])/);
+    outer: for (let n = 1; n <= 6; n++) {
+      for (let i = 0; i <= tokens.length - n; i++) {
+        const ngram = tokens.slice(i, i + n).join("");
+        if (ngram.length < minLength2 || ngram.length > 50) continue;
+        if (/^\s*$/.test(ngram)) continue;
+        if ((ngram.match(/\s/g) || []).length > ngram.length * 0.5) continue;
+        const opens = (ngram.match(/[{(\[<]/g) || []).length;
+        const closes = (ngram.match(/[})\]>]/g) || []).length;
+        if (opens !== closes) continue;
+        substringCounts.set(ngram, (substringCounts.get(ngram) || 0) + 1);
+        if (substringCounts.size >= MAX_MAP_SIZE) break outer;
+      }
+    }
+    const pathRe = /[a-zA-Z0-9_\-./]+\/[a-zA-Z0-9_\-./]+/g;
+    let pathMatch;
+    while ((pathMatch = pathRe.exec(text)) !== null) {
+      const p = pathMatch[0];
+      if (p.length >= minLength2 && substringCounts.size < MAX_MAP_SIZE) {
+        substringCounts.set(p, (substringCounts.get(p) || 0) + 1);
+      }
+    }
+    const candidates = [];
+    for (const [substring, _mapCount] of substringCounts.entries()) {
+      const actualCount = text.split(substring).length - 1;
+      if (actualCount >= minOccurrences) {
+        const abbrevLength = 2;
+        const legendCost = abbrevLength + substring.length + 4;
+        const savingsPerOccurrence = substring.length - abbrevLength;
+        const netSavings = savingsPerOccurrence * actualCount - legendCost;
+        if (netSavings > 5) {
+          candidates.push({ substring, count: actualCount, savings: netSavings });
+        }
+      }
+    }
+    candidates.sort((a, b) => b.savings - a.savings);
+    const selected = [];
+    const usedSubstrings = [];
+    for (const candidate of candidates) {
+      let isTooSimilar = false;
+      const candidateTrimmed = candidate.substring.trim();
+      if (candidateTrimmed.length < 3) continue;
+      for (const used of usedSubstrings) {
+        const usedTrimmed = used.trim();
+        if (used.includes(candidate.substring) || candidate.substring.includes(used)) {
+          isTooSimilar = true;
+          break;
+        }
+        if (candidateTrimmed === usedTrimmed || candidateTrimmed.includes(usedTrimmed) || usedTrimmed.includes(candidateTrimmed)) {
+          isTooSimilar = true;
+          break;
+        }
+        const shorter = candidateTrimmed.length < usedTrimmed.length ? candidateTrimmed : usedTrimmed;
+        const longer = candidateTrimmed.length >= usedTrimmed.length ? candidateTrimmed : usedTrimmed;
+        if (longer.includes(shorter.slice(0, Math.floor(shorter.length * 0.7)))) {
+          isTooSimilar = true;
+          break;
+        }
+      }
+      if (!isTooSimilar) {
+        selected.push(candidate);
+        usedSubstrings.push(candidate.substring);
+        if (selected.length >= maxSubstrings) break;
+      }
+    }
+    return selected;
+  }
+  /**
+   * Apply substring replacements to text.
+   * Applies replacements in savings-descending order (as returned by findRepeatedSubstrings).
+   * @internal
+   */
+  applySubstringCompression(text, substrings) {
+    const legend = {};
+    let compressed = text;
+    substrings.forEach((item, index) => {
+      const abbrev = `\xA7${index.toString(36)}`;
+      legend[abbrev] = item.substring;
+      compressed = compressed.split(item.substring).join(abbrev);
+    });
+    return { compressed, legend };
+  }
+  /**
+   * Apply COMMON_PATTERNS unicode abbreviations (aggressive level only).
+   * @internal
+   */
+  applyCommonPatterns(text) {
+    let result = text;
+    const legend = {};
+    for (const [pattern2, replacement] of Object.entries(_ContextWindowManager.COMMON_PATTERNS)) {
+      try {
+        if (!result.includes(pattern2)) continue;
+        const count = result.split(pattern2).length - 1;
+        const savings = (pattern2.length - replacement.length) * count;
+        if (savings > pattern2.length + replacement.length + 5) {
+          legend[replacement] = pattern2;
+          result = result.split(pattern2).join(replacement);
+        }
+      } catch {
+        continue;
+      }
+    }
+    return { text: result, legend };
+  }
 };
 
 // src/agent/MemoryFormatter.ts