lynkr 9.10.0 → 9.11.0

package/src/memory/distiller.js

@@ -0,0 +1,350 @@
+/**
+ * Conversation Distiller — L0-L3 Pipeline (TencentDB-Agent-Memory inspired)
+ *
+ * Long conversations resend every turn on every request. Once a
+ * conversation reaches the turn threshold, this module replaces the older
+ * turns with one compact distilled block and keeps only the most recent
+ * turns verbatim:
+ *
+ *   L0 — raw turns (the messages themselves, dropped after distillation)
+ *   L1 — facts/decisions extracted from the dropped turns (heuristic)
+ *   L2 — scenario summary: what was asked, done, and decided
+ *   L3 — persona: durable user preferences pulled from the memory store
+ *
+ * Large repeated blocks inside the dropped turns are dereferenced through
+ * the wiki registry before summarization. All processing is local and
+ * synchronous — no LLM calls.
+ */
+
+const store = require("./store");
+const extractor = require("./extractor");
+const wiki = require("./wiki");
+const config = require("../config");
+const logger = require("../logger");
+
+const MAX_SCENARIO_POINTS = 12;
+const MAX_PERSONA_ITEMS = 5;
+const MAX_POINT_CHARS = 140;
+const DEFAULT_REFRESH_EVERY_TURNS = 5;
+const MAX_FROZEN_SESSIONS = 500;
+
+function distillConfig() {
+  return config.memory?.distillation ?? {};
+}
+
+// ---------------------------------------------------------------------------
+// Frozen distilled blocks (Phase 5, cache-aware routing).
+//
+// Re-distilling on every request rewrites the front of the conversation each
+// turn, which invalidates the provider's prompt cache wholesale — on exactly
+// the long sessions where caching matters most. Once emitted, a session's
+// distilled block is frozen: the same block bytes and the same split point
+// are served verbatim until K more user turns have accumulated
+// (config.memory.distillation.refreshEveryTurns, default 5). Each refresh is
+// then ONE deliberate, scheduled cache write instead of one per request.
+// ---------------------------------------------------------------------------
+
+/** @type {Map<string, {frozenAtTurns:number, splitIdx:number, boundaryFp:string, distilledContent:string, stats:Object}>} */
+const _frozen = new Map();
+
+function _refreshEveryTurns() {
+  const k = distillConfig().refreshEveryTurns;
+  return Number.isFinite(k) && k > 0 ? k : DEFAULT_REFRESH_EVERY_TURNS;
+}
+
+/**
+ * Cheap byte-stability fingerprint for the frozen prefix: split position
+ * plus samples of the first and boundary messages. Detects client-side
+ * history rewrites (compaction, edits) that make the frozen block stale.
+ */
+function _boundaryFingerprint(messages, splitIdx) {
+  const sample = (m) => {
+    if (!m) return "?";
+    const c = typeof m.content === "string" ? m.content : JSON.stringify(m.content ?? "");
+    return `${m.role}:${c.slice(0, 80)}`;
+  };
+  return `${splitIdx}|${sample(messages[0])}|${sample(messages[splitIdx - 1])}`;
+}
+
+function _rememberFrozen(sessionId, entry) {
+  if (!sessionId) return;
+  _frozen.delete(sessionId);
+  _frozen.set(sessionId, entry);
+  if (_frozen.size > MAX_FROZEN_SESSIONS) {
+    const oldest = _frozen.keys().next().value;
+    if (oldest !== undefined) _frozen.delete(oldest);
+  }
+}
+
+/** Test helper — drop all frozen blocks. */
+function _clearFrozen() {
+  _frozen.clear();
+}
+
+/**
+ * A "real" user turn carries user-authored text — tool_result-only
+ * user-role messages are plumbing, not turns.
+ */
+function isRealUserTurn(msg) {
+  if (msg?.role !== "user") return false;
+  if (typeof msg.content === "string") return msg.content.trim().length > 0;
+  if (Array.isArray(msg.content)) {
+    return msg.content.some(b => b?.type === "text" && b.text?.trim());
+  }
+  return false;
+}
+
+function realUserTurnIndices(messages) {
+  const indices = [];
+  for (let i = 0; i < messages.length; i++) {
+    if (isRealUserTurn(messages[i])) indices.push(i);
+  }
+  return indices;
+}
+
+const CHARS_PER_TOKEN = 4;
+
+/**
+ * Rough token estimate for the message history (text + tool content).
+ */
+function estimateHistoryTokens(messages) {
+  let chars = 0;
+  for (const msg of messages) {
+    if (typeof msg.content === "string") {
+      chars += msg.content.length;
+    } else if (Array.isArray(msg.content)) {
+      for (const block of msg.content) {
+        if (block?.text) chars += block.text.length;
+        else if (typeof block?.content === "string") chars += block.content.length;
+        else if (Array.isArray(block?.content)) {
+          for (const item of block.content) {
+            chars += typeof item === "string" ? item.length : (item?.text?.length ?? 0);
+          }
+        }
+        if (block?.input) chars += JSON.stringify(block.input).length;
+      }
+    }
+  }
+  return Math.ceil(chars / CHARS_PER_TOKEN);
+}
+
+/**
+ * Whether the conversation should be distilled. Two triggers:
+ *  - turn count: conversation reached turnThreshold user turns, OR
+ *  - size rescue: history alone exceeds tokenThreshold estimated tokens
+ *    (protects small-context models that overflow long before the turn
+ *    threshold — observed with 4k-context Ollama models dying at turn 8).
+ * Either way there must be at least one turn older than the keep window,
+ * or there is nothing to distill.
+ */
+function needsDistillation(messages) {
+  const cfg = distillConfig();
+  if (cfg.enabled === false) return false;
+  if (!messages?.length) return false;
+
+  const turns = realUserTurnIndices(messages).length;
+  const keepRecent = cfg.keepRecentTurns ?? 3;
+  if (turns <= keepRecent) return false;
+
+  if (turns >= (cfg.turnThreshold ?? 10)) return true;
+
+  const tokenThreshold = cfg.tokenThreshold ?? 3000;
+  return estimateHistoryTokens(messages) >= tokenThreshold;
+}
+
+function extractText(msg) {
+  if (typeof msg?.content === "string") return msg.content;
+  if (Array.isArray(msg?.content)) {
+    return msg.content
+      .filter(b => b?.type === "text" && b.text)
+      .map(b => b.text)
+      .join(" ");
+  }
+  return "";
+}
+
+function truncate(text, max = MAX_POINT_CHARS) {
+  const clean = text.replace(/\s+/g, " ").trim();
+  return clean.length > max ? `${clean.slice(0, max)}…` : clean;
+}
+
+/**
+ * L2 — scenario summary of the dropped turns: user asks, decisions and
+ * facts surfaced by the assistant, and tool usage counts.
+ */
+function buildScenario(oldMessages) {
+  const asks = [];
+  const toolCounts = new Map();
+  let assistantText = "";
+
+  for (const msg of oldMessages) {
+    if (isRealUserTurn(msg)) {
+      asks.push(truncate(extractText(msg), 100));
+    } else if (msg.role === "assistant") {
+      assistantText += `${extractText(msg)}\n`;
+      if (Array.isArray(msg.content)) {
+        for (const block of msg.content) {
+          if (block?.type === "tool_use" && block.name) {
+            toolCounts.set(block.name, (toolCounts.get(block.name) ?? 0) + 1);
+          }
+        }
+      }
+    }
+  }
+
+  // L1 — reuse the extraction patterns on the dropped assistant output
+  const decisions = extractor.extractByType(assistantText, "decision").slice(0, 4);
+  const facts = extractor.extractByType(assistantText, "fact").slice(0, 4);
+
+  const parts = [];
+  if (asks.length) {
+    const shown = asks.slice(-MAX_SCENARIO_POINTS);
+    const omitted = asks.length - shown.length;
+    parts.push(`Requests${omitted > 0 ? ` (${omitted} earlier omitted)` : ""}: ${shown.join(" → ")}`);
+  }
+  if (decisions.length) parts.push(`Decisions: ${decisions.map(d => truncate(d)).join("; ")}`);
+  if (facts.length) parts.push(`Facts: ${facts.map(f => truncate(f)).join("; ")}`);
+  if (toolCounts.size) {
+    const tools = Array.from(toolCounts.entries())
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 8)
+      .map(([name, count]) => (count > 1 ? `${name}×${count}` : name))
+      .join(", ");
+    parts.push(`Tools used: ${tools}`);
+  }
+
+  return parts.join("\n");
+}
+
+/**
+ * L3 — persona line from durable preference memories (global + session).
+ */
+function buildPersona(sessionId) {
+  try {
+    const prefs = store
+      .getMemoriesByType("preference", 50)
+      .filter(m => m.sessionId === null || m.sessionId === sessionId)
+      .sort((a, b) => (b.importance ?? 0) - (a.importance ?? 0))
+      .slice(0, MAX_PERSONA_ITEMS);
+
+    if (!prefs.length) return "";
+    return prefs.map(m => truncate(m.content, 80)).join("; ");
+  } catch (err) {
+    logger.warn({ err, sessionId }, "[distiller] Persona build failed");
+    return "";
+  }
+}
+
+/**
+ * Distill a long conversation: dereference repeated large blocks via wiki,
+ * summarize everything before the last keepRecentTurns user turns into one
+ * block, and keep the recent turns verbatim.
+ *
+ * The split lands on a real user turn boundary, so assistant tool_use /
+ * user tool_result pairs are never separated.
+ *
+ * @param {Array} messages - Full conversation
+ * @param {Object} options
+ * @param {string} options.sessionId
+ * @returns {{messages: Array, applied: boolean, stats: Object}}
+ */
+function distillMessages(messages, options = {}) {
+  const { sessionId = null } = options;
+
+  if (!needsDistillation(messages)) {
+    return { messages, applied: false, stats: {} };
+  }
+
+  const keepRecent = distillConfig().keepRecentTurns ?? 3;
+  const turnIndices = realUserTurnIndices(messages);
+
+  // Phase 5 freeze: serve the session's frozen block verbatim while it is
+  // still fresh (fewer than K user turns since it was built) and the
+  // history prefix it covers is byte-stable. The block and split point are
+  // identical across requests, so the provider prompt cache keeps hitting.
+  if (sessionId) {
+    const cached = _frozen.get(sessionId);
+    if (
+      cached
+      && cached.splitIdx < messages.length
+      && turnIndices.length - cached.frozenAtTurns < _refreshEveryTurns()
+      && _boundaryFingerprint(messages, cached.splitIdx) === cached.boundaryFp
+    ) {
+      return {
+        messages: [
+          { role: "user", content: cached.distilledContent },
+          ...messages.slice(cached.splitIdx),
+        ],
+        applied: true,
+        stats: { ...cached.stats, fromFrozenCache: true },
+      };
+    }
+  }
+
+  const splitIdx = turnIndices[Math.max(0, turnIndices.length - keepRecent)];
+
+  if (!splitIdx) {
+    return { messages, applied: false, stats: {} };
+  }
+
+  const oldMessages = messages.slice(0, splitIdx);
+  const recentMessages = messages.slice(splitIdx);
+
+  // Wiki pass over dropped turns: registers large blocks for cross-request
+  // dedup and shrinks repeats before the scenario is built
+  const { messages: dereferenced, stats: wikiStats } = wiki.dereferenceMessages(oldMessages);
+
+  const scenario = buildScenario(dereferenced);
+  const persona = buildPersona(sessionId);
+
+  const sections = [`[Distilled context — earlier ${turnIndices.length - keepRecent} turns compressed]`];
+  if (persona) sections.push(`User profile: ${persona}`);
+  if (scenario) sections.push(scenario);
+
+  const distilledBlock = {
+    role: "user",
+    content: sections.join("\n"),
+  };
+
+  const originalChars = JSON.stringify(oldMessages).length;
+  const distilledChars = distilledBlock.content.length;
+
+  const stats = {
+    droppedMessages: oldMessages.length,
+    keptMessages: recentMessages.length,
+    originalChars,
+    distilledChars,
+    savingsPct: originalChars > 0
+      ? (((originalChars - distilledChars) / originalChars) * 100).toFixed(1)
+      : "0.0",
+    wiki: wikiStats,
+  };
+
+  logger.debug({ sessionId, ...stats }, "[distiller] Conversation distilled");
+
+  // Freeze the block for the next K user turns (Phase 5). Keyed by the
+  // split fingerprint so a client-side history rewrite invalidates it.
+  _rememberFrozen(sessionId, {
+    frozenAtTurns: turnIndices.length,
+    splitIdx,
+    boundaryFp: _boundaryFingerprint(messages, splitIdx),
+    distilledContent: distilledBlock.content,
+    stats,
+  });
+
+  return {
+    messages: [distilledBlock, ...recentMessages],
+    applied: true,
+    stats,
+  };
+}
+
+module.exports = {
+  needsDistillation,
+  distillMessages,
+  buildScenario,
+  buildPersona,
+  isRealUserTurn,
+  estimateHistoryTokens,
+  _clearFrozen,
+};

package/src/memory/extractor.js

@@ -358,19 +358,11 @@ function calculateInitialImportance(type, surpriseScore) {
   return Math.min(1.0, base + (surpriseScore * 0.3));
 }
 
-/**
- * Parse entities from content
- */
-function parseEntities(content) {
-  return extractEntities(content);
-}
-
 module.exports = {
   extractMemories,
   extractContent,
   extractByType,
   extractEntities,
   extractRelationships,
-  parseEntities,
   classifyCategory,
 };

package/src/memory/format.js

@@ -20,11 +20,26 @@ function formatMemoriesForContext(memories, format = 'compact') {
   return formatCompact(memories);
 }
 
+/**
+ * Deterministic render order (cache-aware routing, Phase 5): retrieval
+ * returns equal-relevance memories in unstable order, and the rendered
+ * bullets sit in the system prompt — the very front of the provider's
+ * prompt-cache prefix. A reordered bullet list is a byte change that
+ * silently invalidates the entire cached prefix (observed live: identical
+ * memory sets flipping order between consecutive requests). Sorting by
+ * content pins the bytes; presentation order carries no meaning here.
+ */
+function stableOrder(memories) {
+  return [...memories].sort((a, b) =>
+    String(a.content) < String(b.content) ? -1 : String(a.content) > String(b.content) ? 1 : 0
+  );
+}
+
 /**
  * Compact memory format - 75% fewer tokens
  */
 function formatCompact(memories) {
-  const items = memories
+  const items = stableOrder(memories)
     .map(mem => `- ${mem.content}`)
     .join('\n');
 
@@ -35,7 +50,7 @@ function formatCompact(memories) {
  * Verbose XML format (original)
  */
 function formatVerbose(memories) {
-  const items = memories.map((mem, idx) => {
+  const items = stableOrder(memories).map((mem, idx) => {
     const age = formatAge(mem.createdAt);
     const type = mem.type ? `[${mem.type}] ` : '';
     return `${idx + 1}. ${type}${mem.content} (${age})`;

package/src/memory/index.js

@@ -15,6 +15,9 @@ const retriever = require("./retriever");
 const extractor = require("./extractor");
 const surprise = require("./surprise");
 const tools = require("./tools");
+const distiller = require("./distiller");
+const wiki = require("./wiki");
+const skillsCache = require("./skills-cache");
 
 module.exports = {
   // Store operations
@@ -52,4 +55,11 @@ module.exports = {
   // Tools
   tools,
   MEMORY_TOOLS: tools.MEMORY_TOOLS,
+
+  // TencentDB-inspired L0-L3 token optimization
+  distiller,
+  needsDistillation: distiller.needsDistillation,
+  distillMessages: distiller.distillMessages,
+  wiki,
+  skillsCache,
 };

package/src/memory/retriever.js

@@ -3,6 +3,10 @@ const search = require("./search");
 const logger = require("../logger");
 const format = require("./format");
 
+// Infrastructure memory types (wiki registry, compression skills) are used
+// by the distillation pipeline, never injected as conversational context
+const INTERNAL_MEMORY_TYPES = new Set(["wiki", "skill"]);
+
 /**
  * Retrieve relevant memories using multi-signal ranking
  *
@@ -41,8 +45,9 @@ function retrieveRelevantMemories(query, options = {}) {
       sessionId: includeGlobal ? null : sessionId,
     });
 
-    // 4. Merge and deduplicate
-    const candidates = mergeUnique([ftsResults, recentMemories, importantMemories]);
+    // 4. Merge and deduplicate, dropping infrastructure entries
+    const candidates = mergeUnique([ftsResults, recentMemories, importantMemories])
+      .filter(m => !INTERNAL_MEMORY_TYPES.has(m.type));
 
     // 5. Score and rank
     const scored = candidates.map(memory => ({

package/src/memory/skills-cache.js

@@ -0,0 +1,180 @@
+/**
+ * Skills Cache — Compression Strategy Memory (TencentDB-Agent-Memory inspired)
+ *
+ * Remembers which compression method worked (and how well) for each
+ * structural shape of tool output. Validated skills (savings >= minSavings)
+ * persist in the memories table (type='skill') and let future compressions:
+ *  - skip section-dedup for shapes where it never helps (latency win)
+ *  - compress proven-compressible shapes more aggressively (token win)
+ */
+
+const store = require("./store");
+const distill = require("../context/distill");
+const config = require("../config");
+const logger = require("../logger");
+
+const MAX_CACHED_SKILLS = 1000;
+const SIGNATURE_LINES = 30;
+
+// signature -> { method, avgSavings, hits, persistedId }
+let skillMap = null;
+
+function skillsConfig() {
+  return config.memory?.skills ?? {};
+}
+
+/**
+ * Structural shape signature for a tool output: leading token of each of
+ * the first N normalized lines. Two grep outputs, two test runs, or two
+ * JSON blobs with the same shape produce the same signature even when the
+ * values differ.
+ */
+function shapeSignature(text) {
+  if (!text) return "empty";
+  const lines = distill.normalizeText(text).split("\n").slice(0, SIGNATURE_LINES);
+  const shape = lines
+    .map(l => {
+      const trimmed = l.trim();
+      if (!trimmed) return "";
+      // Leading structural token: punctuation kept, words folded
+      const lead = trimmed.match(/^[{}[\]"'\-+*#>|]|^[A-Za-z_$]+|^\d+/);
+      return lead ? (/^\d+$/.test(lead[0]) ? "N" : lead[0]) : trimmed[0];
+    })
+    .join(",");
+  // djb2 hash keeps keys short
+  let hash = 5381;
+  for (let i = 0; i < shape.length; i++) {
+    hash = ((hash << 5) + hash + shape.charCodeAt(i)) | 0;
+  }
+  return `s${lines.length}_${(hash >>> 0).toString(36)}`;
+}
+
+function loadCache() {
+  if (skillMap) return skillMap;
+
+  skillMap = new Map();
+  try {
+    const rows = store.getMemoriesByType("skill", MAX_CACHED_SKILLS);
+    for (const row of rows) {
+      const meta = typeof row.metadata === "string" ? JSON.parse(row.metadata) : row.metadata;
+      if (!meta?.signature) continue;
+      skillMap.set(meta.signature, {
+        method: meta.method,
+        avgSavings: meta.avgSavings ?? 0,
+        hits: meta.hits ?? 1,
+        persistedId: row.id,
+      });
+    }
+  } catch (err) {
+    logger.warn({ err }, "[skills] Failed to load skills, starting empty");
+  }
+  return skillMap;
+}
+
+/**
+ * Look up a known compression skill for a shape signature.
+ * @returns {{method: string, avgSavings: number, hits: number}|null}
+ */
+function lookup(signature) {
+  if (skillsConfig().enabled === false) return null;
+  return loadCache().get(signature) ?? null;
+}
+
+/**
+ * Record a compression outcome. Kept in memory always (so known-futile
+ * shapes can skip dedup work); persisted only when the running average
+ * savings clears minSavings — the "validated skill" bar.
+ *
+ * @param {string} signature - shapeSignature() of the original text
+ * @param {string} method - compression method that ran ('delta'|'distill'|'passthrough')
+ * @param {number} savings - fraction saved, 0..1
+ */
+function record(signature, method, savings) {
+  if (skillsConfig().enabled === false || !signature) return;
+
+  const cache = loadCache();
+  const existing = cache.get(signature);
+  const entry = existing
+    ? {
+        ...existing,
+        method,
+        hits: existing.hits + 1,
+        avgSavings: (existing.avgSavings * existing.hits + savings) / (existing.hits + 1),
+      }
+    : { method, avgSavings: savings, hits: 1, persistedId: null };
+
+  cache.set(signature, entry);
+  if (cache.size > MAX_CACHED_SKILLS) {
+    cache.delete(cache.keys().next().value);
+  }
+
+  const minSavings = skillsConfig().minSavings ?? 0.6;
+  if (entry.avgSavings < minSavings) return;
+
+  // Persist validated skill off the hot path
+  setImmediate(() => {
+    try {
+      const metadata = {
+        signature,
+        method: entry.method,
+        avgSavings: entry.avgSavings,
+        hits: entry.hits,
+      };
+      if (entry.persistedId) {
+        const current = store.getMemory(entry.persistedId);
+        if (current) {
+          store.updateMemory(entry.persistedId, { metadata });
+          return;
+        }
+      }
+      const memory = store.createMemory({
+        sessionId: null,
+        content: `Compression skill: ${entry.method} saves ${(entry.avgSavings * 100).toFixed(0)}% on shape ${signature}`,
+        type: "skill",
+        category: "optimization",
+        importance: 0.3,
+        metadata,
+      });
+      entry.persistedId = memory.id;
+    } catch (err) {
+      logger.warn({ err, signature }, "[skills] Failed to persist skill");
+    }
+  });
+}
+
+/**
+ * Compression hints for a given text, derived from validated skills.
+ *
+ * @param {string} text - Tool result text about to be compressed
+ * @returns {{signature: string, skipDedup: boolean, maxLengthFactor: number}}
+ */
+function getCompressionHints(text) {
+  const signature = shapeSignature(text);
+  const skill = lookup(signature);
+  const minSavings = skillsConfig().minSavings ?? 0.6;
+
+  if (!skill || skill.hits < 2) {
+    return { signature, skipDedup: false, maxLengthFactor: 1 };
+  }
+
+  return {
+    signature,
+    // Dedup/delta repeatedly achieved almost nothing — don't burn CPU on it
+    skipDedup: skill.method === "passthrough" || skill.avgSavings < 0.05,
+    // Proven highly-compressible shape — compress harder
+    maxLengthFactor: skill.avgSavings >= minSavings ? 0.8 : 1,
+  };
+}
+
+/** Reset the in-memory cache (test support). */
+function resetCache() {
+  skillMap = null;
+}
+
+module.exports = {
+  shapeSignature,
+  lookup,
+  record,
+  getCompressionHints,
+  resetCache,
+};