lynkr 2.0.0 → 3.0.0

package/src/memory/extractor.js

@@ -0,0 +1,350 @@
+const store = require("./store");
+const surprise = require("./surprise");
+const config = require("../config");
+const logger = require("../logger");
+
+// Extraction patterns for different memory types
+const EXTRACTION_PATTERNS = {
+  preference: [
+    /(?:I|user|they|you)\s+(?:prefer|like|want|need|always|never|usually|typically)\s+([^.!?\n]{10,100})/gi,
+    /(?:my|your|their|the user's)\s+(?:preference|choice|favorite)\s+(?:is|for|would be)\s+([^.!?\n]{10,100})/gi,
+    /(?:should|must|need to)\s+(?:use|implement|do|follow)\s+([^.!?\n]{10,100})/gi,
+  ],
+
+  decision: [
+    /(?:decided|choosing|going with|selected|picked|opted for)\s+([^.!?\n]{10,100})/gi,
+    /(?:the|our|my)\s+(?:approach|strategy|plan|solution)\s+(?:is|will be|should be)\s+([^.!?\n]{10,100})/gi,
+    /(?:let's|we'll|I'll)\s+(?:use|implement|go with|choose)\s+([^.!?\n]{10,100})/gi,
+    /(?:agreed|confirmed)\s+(?:to|that|on)\s+([^.!?\n]{10,100})/gi,
+  ],
+
+  fact: [
+    /(?:this|the)\s+(?:project|codebase|application|system)\s+(?:uses|is|has|implements|requires)\s+([^.!?\n]{10,150})/gi,
+    /(?:important|note|remember|keep in mind):\s*([^.!?\n]{10,150})/gi,
+    /(?:the|this)\s+(?:file|function|class|module|component)\s+(?:is|handles|manages|does)\s+([^.!?\n]{10,150})/gi,
+    /(?:currently|now|at the moment)\s+(?:using|implementing|running)\s+([^.!?\n]{10,100})/gi,
+  ],
+
+  entity: [
+    /(?:file|function|class|module|component|package|library)\s+['"`]?([A-Za-z0-9_./\-]+)['"`]?/gi,
+    /in\s+['"`]([A-Za-z0-9_./\-]+\.(?:js|ts|py|java|go|rs|cpp|c|h))['"`]/gi,
+    /`([A-Za-z0-9_]+(?:\.[A-Za-z0-9_]+)*)`/g, // Code references
+  ],
+
+  relationship: [
+    /([A-Za-z0-9_]+)\s+(?:depends on|imports|uses|extends|implements|inherits from)\s+([A-Za-z0-9_]+)/gi,
+    /([A-Za-z0-9_./\-]+)\s+(?:calls|invokes|references)\s+([A-Za-z0-9_./\-]+)/gi,
+  ],
+};
+
+/**
+ * Extract memories from assistant response and conversation context
+ */
+async function extractMemories(assistantResponse, conversationMessages, context = {}) {
+  if (!config.memory?.extraction?.enabled) {
+    return [];
+  }
+
+  const { sessionId = null } = context;
+  const memories = [];
+
+  try {
+    // Extract assistant message content
+    const assistantContent = extractContent(assistantResponse);
+    if (!assistantContent) return [];
+
+    // Get last user message for context
+    const lastUserMessage = conversationMessages
+      ?.filter(m => m.role === 'user')
+      ?.pop();
+    const userContent = lastUserMessage ? extractContent(lastUserMessage) : '';
+
+    // Extract different types of memories
+    const preferences = extractByType(assistantContent, 'preference');
+    const decisions = extractByType(assistantContent, 'decision');
+    const facts = extractByType(assistantContent, 'fact');
+    const entities = extractEntities(assistantContent);
+    const relationships = extractRelationships(assistantContent);
+
+    // Create memory objects with surprise scores
+    for (const content of preferences) {
+      const memory = await createMemoryWithSurprise({
+        content,
+        type: 'preference',
+        category: 'user',
+        sessionId,
+        userContent,
+      });
+      if (memory) memories.push(memory);
+    }
+
+    for (const content of decisions) {
+      const memory = await createMemoryWithSurprise({
+        content,
+        type: 'decision',
+        category: 'project',
+        sessionId,
+        userContent,
+      });
+      if (memory) memories.push(memory);
+    }
+
+    for (const content of facts) {
+      const memory = await createMemoryWithSurprise({
+        content,
+        type: 'fact',
+        category: classifyCategory(content),
+        sessionId,
+        userContent,
+      });
+      if (memory) memories.push(memory);
+    }
+
+    for (const entityName of entities) {
+      // Track entity
+      store.trackEntity('code', entityName, { source: 'extraction' });
+
+      const memory = await createMemoryWithSurprise({
+        content: `Entity: ${entityName}`,
+        type: 'entity',
+        category: 'code',
+        sessionId,
+        userContent,
+        metadata: { entityName },
+      });
+      if (memory) memories.push(memory);
+    }
+
+    for (const { from, to, relationship } of relationships) {
+      const memory = await createMemoryWithSurprise({
+        content: `${from} ${relationship} ${to}`,
+        type: 'relationship',
+        category: 'code',
+        sessionId,
+        userContent,
+        metadata: { from, to, relationship },
+      });
+      if (memory) memories.push(memory);
+    }
+
+    logger.debug({
+      sessionId,
+      memoriesExtracted: memories.length,
+      types: {
+        preference: preferences.length,
+        decision: decisions.length,
+        fact: facts.length,
+        entity: entities.length,
+        relationship: relationships.length,
+      },
+    }, 'Memory extraction completed');
+
+    return memories;
+  } catch (err) {
+    logger.error({ err, sessionId }, 'Memory extraction failed');
+    return [];
+  }
+}
+
+/**
+ * Extract content from message (handle different formats)
+ */
+function extractContent(message) {
+  if (!message) return '';
+
+  // Handle different message formats
+  if (typeof message === 'string') return message;
+
+  if (message.content) {
+    if (typeof message.content === 'string') return message.content;
+
+    // Handle array of content blocks
+    if (Array.isArray(message.content)) {
+      return message.content
+        .filter(block => block?.type === 'text' || typeof block === 'string')
+        .map(block => typeof block === 'string' ? block : block.text)
+        .join('\n');
+    }
+  }
+
+  // Handle choices array (from model responses)
+  if (message.choices && Array.isArray(message.choices)) {
+    const choice = message.choices[0];
+    if (choice?.message?.content) {
+      return extractContent(choice.message);
+    }
+  }
+
+  return '';
+}
+
+/**
+ * Extract memories by type using patterns
+ */
+function extractByType(text, type) {
+  const patterns = EXTRACTION_PATTERNS[type] || [];
+  const matches = new Set();
+
+  for (const pattern of patterns) {
+    let match;
+    while ((match = pattern.exec(text)) !== null) {
+      const captured = match[1]?.trim();
+      if (captured && captured.length >= 10 && captured.length <= 200) {
+        matches.add(captured);
+      }
+    }
+  }
+
+  return Array.from(matches);
+}
+
+/**
+ * Extract entity references
+ */
+function extractEntities(text) {
+  const entities = new Set();
+  const patterns = EXTRACTION_PATTERNS.entity;
+
+  for (const pattern of patterns) {
+    let match;
+    while ((match = pattern.exec(text)) !== null) {
+      const entityName = match[1]?.trim();
+      if (entityName && entityName.length >= 3 && entityName.length <= 100) {
+        entities.add(entityName);
+      }
+    }
+  }
+
+  return Array.from(entities);
+}
+
+/**
+ * Extract relationships between entities
+ */
+function extractRelationships(text) {
+  const relationships = [];
+  const patterns = EXTRACTION_PATTERNS.relationship;
+
+  for (const pattern of patterns) {
+    let match;
+    while ((match = pattern.exec(text)) !== null) {
+      const from = match[1]?.trim();
+      const to = match[2]?.trim();
+      if (from && to) {
+        relationships.push({
+          from,
+          to,
+          relationship: 'depends_on', // Generalized relationship type
+        });
+      }
+    }
+  }
+
+  return relationships;
+}
+
+/**
+ * Classify content category
+ */
+function classifyCategory(content) {
+  const lower = content.toLowerCase();
+
+  if (lower.includes('code') || lower.includes('function') || lower.includes('file') ||
+      lower.includes('class') || lower.includes('module')) {
+    return 'code';
+  }
+
+  if (lower.includes('project') || lower.includes('application') || lower.includes('system')) {
+    return 'project';
+  }
+
+  if (lower.includes('user') || lower.includes('prefer') || lower.includes('like')) {
+    return 'user';
+  }
+
+  return 'general';
+}
+
+/**
+ * Create memory with surprise score calculation
+ */
+async function createMemoryWithSurprise(options) {
+  const { content, type, category, sessionId, userContent, metadata = {} } = options;
+
+  // Get existing memories for surprise calculation
+  const existingMemories = store.getRecentMemories({ limit: 100, sessionId });
+
+  // Calculate surprise score
+  const surpriseScore = surprise.calculateSurprise({
+    content,
+    type,
+    category,
+  }, existingMemories, { userContent });
+
+  // Only store if surprise score exceeds threshold
+  const threshold = config.memory?.surpriseThreshold ?? 0.3;
+  if (surpriseScore < threshold) {
+    logger.debug({ content, surpriseScore, threshold }, 'Memory filtered by surprise threshold');
+    return null;
+  }
+
+  // Calculate initial importance based on surprise and type
+  const importance = calculateInitialImportance(type, surpriseScore);
+
+  // Store memory
+  try {
+    const memory = store.createMemory({
+      sessionId,
+      content,
+      type,
+      category,
+      importance,
+      surpriseScore,
+      metadata: {
+        ...metadata,
+        extractedAt: Date.now(),
+      },
+    });
+
+    return memory;
+  } catch (err) {
+    logger.warn({ err, content }, 'Failed to store memory');
+    return null;
+  }
+}
+
+/**
+ * Calculate initial importance score
+ */
+function calculateInitialImportance(type, surpriseScore) {
+  // Base importance by type
+  const baseImportance = {
+    preference: 0.7,  // User preferences are important
+    decision: 0.8,    // Decisions are very important
+    fact: 0.6,        // Facts are moderately important
+    entity: 0.4,      // Entities are less important individually
+    relationship: 0.5,
+  };
+
+  const base = baseImportance[type] ?? 0.5;
+
+  // Boost by surprise score (0-1 scale)
+  return Math.min(1.0, base + (surpriseScore * 0.3));
+}
+
+/**
+ * Parse entities from content
+ */
+function parseEntities(content) {
+  return extractEntities(content);
+}
+
+module.exports = {
+  extractMemories,
+  extractContent,
+  extractByType,
+  extractEntities,
+  extractRelationships,
+  parseEntities,
+  classifyCategory,
+};

package/src/memory/index.js

@@ -0,0 +1,55 @@
+/**
+ * Titans-Inspired Long-Term Memory System
+ *
+ * This module provides long-term memory capabilities with:
+ * - Surprise-based memory updates (Titans core innovation)
+ * - Automatic memory extraction from conversations
+ * - FTS5 semantic search
+ * - Multi-signal memory retrieval
+ * - Memory management tools
+ */
+
+const store = require("./store");
+const search = require("./search");
+const retriever = require("./retriever");
+const extractor = require("./extractor");
+const surprise = require("./surprise");
+const tools = require("./tools");
+
+module.exports = {
+  // Store operations
+  store,
+  createMemory: store.createMemory,
+  getMemory: store.getMemory,
+  updateMemory: store.updateMemory,
+  deleteMemory: store.deleteMemory,
+  getRecentMemories: store.getRecentMemories,
+  getMemoriesByImportance: store.getMemoriesByImportance,
+  pruneOldMemories: store.pruneOldMemories,
+  countMemories: store.countMemories,
+
+  // Search operations
+  search,
+  searchMemories: search.searchMemories,
+  searchWithExpansion: search.searchWithExpansion,
+  findSimilar: search.findSimilar,
+
+  // Retrieval
+  retriever,
+  retrieveRelevantMemories: retriever.retrieveRelevantMemories,
+  formatMemoriesForContext: retriever.formatMemoriesForContext,
+  injectMemoriesIntoSystem: retriever.injectMemoriesIntoSystem,
+  getMemoryStats: retriever.getMemoryStats,
+
+  // Extraction
+  extractor,
+  extractMemories: extractor.extractMemories,
+
+  // Surprise detection
+  surprise,
+  calculateSurprise: surprise.calculateSurprise,
+
+  // Tools
+  tools,
+  MEMORY_TOOLS: tools.MEMORY_TOOLS,
+};

package/src/memory/retriever.js

@@ -0,0 +1,266 @@
+const store = require("./store");
+const search = require("./search");
+const logger = require("../logger");
+
+/**
+ * Retrieve relevant memories using multi-signal ranking
+ *
+ * Scoring algorithm:
+ * - 30% Recency: Exponential decay based on last access
+ * - 40% Importance: Stored importance value
+ * - 30% Relevance: Keyword overlap with query
+ */
+function retrieveRelevantMemories(query, options = {}) {
+  const {
+    limit = 10,
+    sessionId = null,
+    includeGlobal = true,
+    recencyWeight = 0.3,
+    importanceWeight = 0.4,
+    relevanceWeight = 0.3,
+  } = options;
+
+  try {
+    // 1. FTS5 search for keyword relevance
+    const ftsResults = search.searchMemories({
+      query,
+      limit: limit * 3, // Get more candidates
+      sessionId: includeGlobal ? null : sessionId,
+    });
+
+    // 2. Get recent memories (recency bias)
+    const recentMemories = store.getRecentMemories({
+      limit: limit * 2,
+      sessionId: includeGlobal ? null : sessionId,
+    });
+
+    // 3. Get high-importance memories
+    const importantMemories = store.getMemoriesByImportance({
+      limit: limit * 2,
+      sessionId: includeGlobal ? null : sessionId,
+    });
+
+    // 4. Merge and deduplicate
+    const candidates = mergeUnique([ftsResults, recentMemories, importantMemories]);
+
+    // 5. Score and rank
+    const scored = candidates.map(memory => ({
+      memory,
+      score: calculateRetrievalScore(memory, query, {
+        recencyWeight,
+        importanceWeight,
+        relevanceWeight,
+      }),
+    }));
+
+    // 6. Sort by score and return top K
+    const topMemories = scored
+      .sort((a, b) => b.score - a.score)
+      .slice(0, limit)
+      .map(s => s.memory);
+
+    // 7. Update access counts asynchronously
+    setImmediate(() => {
+      for (const memory of topMemories) {
+        try {
+          store.incrementAccessCount(memory.id);
+        } catch (err) {
+          logger.warn({ err, memoryId: memory.id }, 'Failed to increment access count');
+        }
+      }
+    });
+
+    return topMemories;
+  } catch (err) {
+    logger.error({ err, query }, 'Memory retrieval failed');
+    return [];
+  }
+}
+
+/**
+ * Calculate retrieval score for a memory
+ */
+function calculateRetrievalScore(memory, query, weights) {
+  // Recency score: exponential decay based on last access
+  const ageMs = Date.now() - (memory.lastAccessedAt || memory.createdAt);
+  const halfLifeMs = 7 * 24 * 60 * 60 * 1000; // 7 days
+  const recencyScore = Math.exp(-ageMs / halfLifeMs);
+
+  // Importance score: direct from stored value
+  const importanceScore = memory.importance ?? 0.5;
+
+  // Relevance score: keyword overlap with query
+  const relevanceScore = calculateKeywordOverlap(memory.content, query);
+
+  // Weighted combination
+  return (
+    weights.recencyWeight * recencyScore +
+    weights.importanceWeight * importanceScore +
+    weights.relevanceWeight * relevanceScore
+  );
+}
+
+/**
+ * Calculate keyword overlap between content and query
+ */
+function calculateKeywordOverlap(content, query) {
+  const contentKeywords = new Set(search.extractKeywords(content));
+  const queryKeywords = search.extractKeywords(query);
+
+  if (queryKeywords.length === 0 || contentKeywords.size === 0) {
+    return 0.0;
+  }
+
+  let overlapCount = 0;
+  for (const keyword of queryKeywords) {
+    if (contentKeywords.has(keyword)) {
+      overlapCount++;
+    }
+  }
+
+  return overlapCount / queryKeywords.length;
+}
+
+/**
+ * Merge arrays and remove duplicates by memory ID
+ */
+function mergeUnique(arrays) {
+  const seen = new Set();
+  const merged = [];
+
+  for (const arr of arrays) {
+    for (const item of arr) {
+      if (!seen.has(item.id)) {
+        seen.add(item.id);
+        merged.push(item);
+      }
+    }
+  }
+
+  return merged;
+}
+
+/**
+ * Extract query from message (handle different formats)
+ */
+function extractQueryFromMessage(message) {
+  if (!message) return '';
+
+  if (typeof message === 'string') return message;
+
+  if (message.content) {
+    if (typeof message.content === 'string') return message.content;
+
+    if (Array.isArray(message.content)) {
+      return message.content
+        .filter(block => block?.type === 'text' || typeof block === 'string')
+        .map(block => typeof block === 'string' ? block : block.text)
+        .filter(Boolean)
+        .join(' ');
+    }
+  }
+
+  return '';
+}
+
+/**
+ * Format memories for injection into context
+ */
+function formatMemoriesForContext(memories) {
+  if (!memories || memories.length === 0) return '';
+
+  return memories
+    .map((memory, index) => {
+      const age = formatAge(Date.now() - memory.createdAt);
+      const typeLabel = memory.type || 'memory';
+      return `${index + 1}. [${typeLabel}] ${memory.content} (${age})`;
+    })
+    .join('\n');
+}
+
+/**
+ * Format age in human-readable form
+ */
+function formatAge(ageMs) {
+  const seconds = Math.floor(ageMs / 1000);
+  const minutes = Math.floor(seconds / 60);
+  const hours = Math.floor(minutes / 60);
+  const days = Math.floor(hours / 24);
+  const weeks = Math.floor(days / 7);
+
+  if (weeks > 0) return `${weeks} week${weeks > 1 ? 's' : ''} ago`;
+  if (days > 0) return `${days} day${days > 1 ? 's' : ''} ago`;
+  if (hours > 0) return `${hours} hour${hours > 1 ? 's' : ''} ago`;
+  if (minutes > 0) return `${minutes} minute${minutes > 1 ? 's' : ''} ago`;
+  return 'just now';
+}
+
+/**
+ * Inject memories into system prompt
+ */
+function injectMemoriesIntoSystem(existingSystem, memories, format = 'system') {
+  if (!memories || memories.length === 0) return existingSystem;
+
+  const formattedMemories = formatMemoriesForContext(memories);
+
+  if (format === 'system') {
+    const memoryBlock = `
+<long_term_memory>
+The following are relevant facts and context from previous conversations:
+${formattedMemories}
+</long_term_memory>`;
+
+    return existingSystem
+      ? `${existingSystem}\n${memoryBlock}`
+      : memoryBlock;
+  }
+
+  if (format === 'assistant_preamble') {
+    return {
+      system: existingSystem,
+      memoryPreamble: formattedMemories,
+    };
+  }
+
+  return existingSystem;
+}
+
+/**
+ * Get memory statistics
+ */
+function getMemoryStats(sessionId = null) {
+  try {
+    const total = store.countMemories();
+    const byType = {};
+    const types = ['preference', 'decision', 'fact', 'entity', 'relationship'];
+
+    for (const type of types) {
+      byType[type] = store.getMemoriesByType(type, 1000).length;
+    }
+
+    const recent = store.getRecentMemories({ limit: 10, sessionId });
+    const important = store.getMemoriesByImportance({ limit: 10, sessionId });
+
+    return {
+      total,
+      byType,
+      recentCount: recent.length,
+      importantCount: important.length,
+      sessionId,
+    };
+  } catch (err) {
+    logger.error({ err }, 'Failed to get memory stats');
+    return null;
+  }
+}
+
+module.exports = {
+  retrieveRelevantMemories,
+  calculateRetrievalScore,
+  calculateKeywordOverlap,
+  extractQueryFromMessage,
+  formatMemoriesForContext,
+  injectMemoriesIntoSystem,
+  formatAge,
+  getMemoryStats,
+};