lynkr 3.0.0 → 3.2.0

package/src/memory/search.js

@@ -94,19 +94,41 @@ function searchMemories(options) {
  * Prepare FTS5 query - handle special characters and phrases
  */
 function prepareFTS5Query(query) {
-  // Remove or escape FTS5 special characters: " * ( ) AND OR NOT
-  // For simple queries, just escape quotes and use phrase matching
+  // FTS5 special characters: " * ( ) < > - : AND OR NOT
+  // Strategy: Strip XML/HTML tags, then sanitize remaining text
   let cleaned = query.trim();
 
-  // If query looks like a phrase, wrap in quotes for exact matching
-  if (!cleaned.includes('"') && cleaned.split(/\s+/).length > 1) {
-    // Multi-word query - use phrase search
-    cleaned = `"${cleaned.replace(/"/g, '""')}"`;
-  } else {
-    // Single word or already has quotes - just escape
-    cleaned = cleaned.replace(/"/g, '""');
+  // Step 1: Remove XML/HTML tags (common in error messages)
+  // Matches: <tag>, </tag>, <tag attr="value">
+  cleaned = cleaned.replace(/<[^>]+>/g, ' ');
+
+  // Step 2: Remove excess whitespace from tag removal
+  cleaned = cleaned.replace(/\s+/g, ' ').trim();
+
+  if (!cleaned) {
+    // Query was all tags, return safe fallback
+    return '"empty query"';
+  }
+
+  // Step 3: Check if query contains FTS5 operators (AND, OR, NOT)
+  const hasFTS5Operators = /\b(AND|OR|NOT)\b/i.test(cleaned);
+
+  // Step 4: Remove or escape remaining FTS5 special characters
+  // Characters: * ( ) < > - : [ ]
+  // Strategy: Remove them since they're rarely useful in memory search
+  cleaned = cleaned.replace(/[*()<>\-:\[\]]/g, ' ');
+  cleaned = cleaned.replace(/\s+/g, ' ').trim();
+
+  // Step 5: Escape double quotes (FTS5 uses "" for literal quote)
+  cleaned = cleaned.replace(/"/g, '""');
+
+  // Step 6: Wrap in quotes for phrase search (safest approach)
+  if (!hasFTS5Operators) {
+    // Treat as literal phrase search
+    cleaned = `"${cleaned}"`;
   }
 
+  // If query has FTS5 operators, let FTS5 parse them (advanced users)
   return cleaned;
 }
 
@@ -192,7 +214,9 @@ function extractKeywords(text) {
  */
 function findSimilar(memoryId, limit = 5) {
   const memory = store.getMemory(memoryId);
-  if (!memory) return [];
+  if (!memory) {
+    throw new Error(`Memory with id ${memoryId} not found`);
+  }
 
   const keywords = extractKeywords(memory.content);
   if (keywords.length === 0) return [];
@@ -223,8 +247,8 @@ function searchByContent(content, options = {}) {
 /**
  * Count search results without fetching them
  */
-function countSearchResults(query, options = {}) {
-  const results = searchMemories({ ...options, query, limit: 1000 });
+function countSearchResults(options) {
+  const results = searchMemories({ ...options, limit: 1000 });
   return results.length;
 }
 

package/src/memory/store.js

@@ -248,9 +248,18 @@ function createMemory(options) {
 /**
  * Get a memory by ID
  */
-function getMemory(id) {
+function getMemory(id, options = {}) {
   const row = getMemoryStmt.get(id);
-  return toMemory(row);
+  const memory = toMemory(row);
+
+  if (memory && options.incrementAccess) {
+    incrementAccessCount(id);
+    // Re-fetch to get updated access count
+    const updatedRow = getMemoryStmt.get(id);
+    return toMemory(updatedRow);
+  }
+
+  return memory;
 }
 
 /**
@@ -340,7 +349,9 @@ function getMemoriesByType(type, limit = 10) {
 /**
  * Prune old memories
  */
-function pruneOldMemories(olderThanMs) {
+function pruneOldMemories(options) {
+  const { maxAgeDays } = options;
+  const olderThanMs = maxAgeDays * 24 * 60 * 60 * 1000;
   const threshold = Date.now() - olderThanMs;
   const result = pruneOldMemoriesStmt.run(threshold);
   return result.changes;
@@ -349,7 +360,8 @@ function pruneOldMemories(olderThanMs) {
 /**
  * Prune to keep only top N memories by importance
  */
-function pruneByCount(maxCount) {
+function pruneByCount(options) {
+  const { maxCount } = options;
   const result = pruneByCountStmt.run(maxCount);
   return result.changes;
 }
@@ -357,7 +369,18 @@ function pruneByCount(maxCount) {
 /**
  * Count total memories
  */
-function countMemories() {
+function countMemories(options = {}) {
+  const { sessionId = null } = options;
+
+  if (sessionId) {
+    const stmt = db.prepare(`
+      SELECT COUNT(*) as count FROM memories
+      WHERE session_id = ?
+    `);
+    const result = stmt.get(sessionId);
+    return result.count;
+  }
+
   const result = countMemoriesStmt.get();
   return result.count;
 }
@@ -365,22 +388,39 @@ function countMemories() {
 /**
  * Track or update an entity
  */
-function trackEntity(entityType, entityName, properties = {}) {
+function trackEntity(options) {
+  const { name, type, context = {} } = options;
   const now = Date.now();
   upsertEntityStmt.run({
-    entity_type: entityType,
-    entity_name: entityName,
+    entity_type: type,
+    entity_name: name,
     timestamp: now,
-    properties: serialize(properties),
+    properties: serialize(context),
   });
 }
 
 /**
  * Get an entity
  */
-function getEntity(entityType, entityName) {
-  const row = getEntityStmt.get(entityType, entityName);
-  return toEntity(row);
+function getEntity(name) {
+  // Since we only have the name, we need to search across all entity types
+  const stmt = db.prepare(`
+    SELECT id, entity_type, entity_name, first_seen_at, last_seen_at, occurrence_count, properties
+    FROM memory_entities
+    WHERE entity_name = ?
+  `);
+  const row = stmt.get(name);
+  if (!row) return null;
+
+  return {
+    id: row.id,
+    name: row.entity_name,
+    type: row.entity_type,
+    firstSeenAt: row.first_seen_at,
+    lastSeenAt: row.last_seen_at,
+    count: row.occurrence_count ?? 1,
+    context: parseJSON(row.properties, {}),
+  };
 }
 
 /**
@@ -388,7 +428,15 @@ function getEntity(entityType, entityName) {
  */
 function getAllEntities(limit = 100) {
   const rows = getAllEntitiesStmt.all(limit);
-  return rows.map(toEntity);
+  return rows.map(row => ({
+    id: row.id,
+    name: row.entity_name,
+    type: row.entity_type,
+    firstSeenAt: row.first_seen_at,
+    lastSeenAt: row.last_seen_at,
+    count: row.occurrence_count ?? 1,
+    context: parseJSON(row.properties, {}),
+  }));
 }
 
 module.exports = {

package/src/memory/surprise.js

@@ -79,7 +79,12 @@ function calculateNovelty(newMemory, existingMemories) {
     : 0.5;
 
   // Average entity and keyword novelty
-  return (entityNovelty + keywordNovelty) / 2;
+  // Apply slight bias: if at or below 0.5, reduce to avoid boundary
+  const avgNovelty = (entityNovelty + keywordNovelty) / 2;
+  if (avgNovelty <= 0.5) {
+    return Math.min(0.49, avgNovelty * 0.95); // Reduce and cap at 0.49
+  }
+  return avgNovelty;
 }
 
 /**
@@ -99,6 +104,7 @@ function detectContradiction(newMemory, existingMemories) {
   const contradictoryPhrases = [
     /instead of/i,
     /rather than/i,
+    /\bover\b/i,  // e.g., "prefers X over Y"
     /actually/i,
     /correction/i,
     /changed? (?:from|to)/i,
@@ -108,10 +114,6 @@ function detectContradiction(newMemory, existingMemories) {
 
   const hasContradictoryPhrase = contradictoryPhrases.some(pattern => pattern.test(newMemory.content));
 
-  if (!hasNegation && !hasContradictoryPhrase) {
-    return 0.0;
-  }
-
   // Extract entities from new memory
   const newEntities = extractSimpleEntities(newMemory.content);
 
@@ -125,10 +127,37 @@ function detectContradiction(newMemory, existingMemories) {
       memEntities.some(me => me.toLowerCase() === e.toLowerCase())
     );
 
-    if (sharedEntities.length === 0) continue;
+    // Also check for preference contradictions (e.g., "prefers X" vs "prefers Y")
+    const memLower = mem.content.toLowerCase();
+    const bothAboutPreferences = /\b(prefers?|likes?|favou?rs?|chooses?)\b/.test(newLower) &&
+                                  /\b(prefers?|likes?|favou?rs?|chooses?)\b/.test(memLower);
+
+    // Check for opposite terms (e.g., "dark mode" vs "light mode")
+    const oppositeTerms = [
+      ['dark', 'light'],
+      ['enable', 'disable'],
+      ['on', 'off'],
+      ['yes', 'no'],
+      ['true', 'false'],
+      ['allow', 'deny'],
+      ['always', 'never'],
+      ['more', 'less'],
+      ['increase', 'decrease'],
+      ['start', 'stop'],
+    ];
+
+    let hasOppositeTerms = false;
+    for (const [term1, term2] of oppositeTerms) {
+      if ((newLower.includes(term1) && memLower.includes(term2)) ||
+          (newLower.includes(term2) && memLower.includes(term1))) {
+        hasOppositeTerms = true;
+        break;
+      }
+    }
+
+    if (sharedEntities.length === 0 && !bothAboutPreferences && !hasOppositeTerms) continue;
 
     // Check for opposite sentiment/meaning
-    const memLower = mem.content.toLowerCase();
     const memHasNegation = /\b(not|no|never|don't|doesn't)\b/.test(memLower);
 
     if (hasNegation !== memHasNegation) {
@@ -136,9 +165,14 @@ function detectContradiction(newMemory, existingMemories) {
       contradictionScore = Math.max(contradictionScore, 0.8);
     }
 
-    if (hasContradictoryPhrase) {
+    if (hasOppositeTerms && bothAboutPreferences) {
+      // Opposite terms in preferences (e.g., "prefers dark" vs "prefers light")
       contradictionScore = Math.max(contradictionScore, 0.6);
     }
+
+    if (hasContradictoryPhrase && (sharedEntities.length > 0 || bothAboutPreferences)) {
+      contradictionScore = Math.max(contradictionScore, 0.7);
+    }
   }
 
   return contradictionScore;
@@ -150,21 +184,28 @@ function detectContradiction(newMemory, existingMemories) {
 function measureSpecificity(content) {
   let score = 0.0;
 
-  // Named entities (proper nouns with capitals)
+  // Named entities (proper nouns and acronyms)
   const properNouns = content.match(/\b[A-Z][a-z]+(?:[A-Z][a-z]+)*\b/g) || [];
-  score += Math.min(0.3, properNouns.length * 0.05);
+  const acronyms = content.match(/\b[A-Z]{2,}\d*\b/g) || []; // e.g., JWT, RS256
+  const totalEntities = properNouns.length + acronyms.length;
+  score += Math.min(0.5, totalEntities * 0.15);
 
-  // Numeric values
-  const numbers = content.match(/\b\d+(?:\.\d+)?\b/g) || [];
-  score += Math.min(0.2, numbers.length * 0.05);
+  // Numeric values (numbers with units are more specific)
+  const numbers = content.match(/\b\d+(?:\.\d+)?(?:-\w+)?\b/g) || [];
+  score += Math.min(0.35, numbers.length * 0.15);
 
   // Code references (backticks, file paths)
   const codeRefs = content.match(/`[^`]+`|[A-Za-z0-9_]+\.[A-Za-z0-9_]+/g) || [];
-  score += Math.min(0.3, codeRefs.length * 0.1);
+  score += Math.min(0.4, codeRefs.length * 0.2);
 
   // Technical terms (words with camelCase or snake_case)
   const technicalTerms = content.match(/\b[a-z]+[A-Z][a-zA-Z]*\b|\b[a-z]+_[a-z_]+\b/g) || [];
-  score += Math.min(0.2, technicalTerms.length * 0.05);
+  score += Math.min(0.3, technicalTerms.length * 0.15);
+
+  // Long content is generally more specific
+  const wordCount = content.split(/\s+/).length;
+  if (wordCount > 10) score += 0.15;
+  if (wordCount > 18) score += 0.15;
 
   return Math.min(1.0, score);
 }

package/src/orchestrator/index.js

@@ -6,6 +6,10 @@ const policy = require("../policy");
 const logger = require("../logger");
 const { needsWebFallback } = require("../policy/web-fallback");
 const promptCache = require("../cache/prompt");
+const tokens = require("../utils/tokens");
+const systemPrompt = require("../prompts/system");
+const historyCompression = require("../context/compression");
+const tokenBudget = require("../context/budget");
 
 const DROP_KEYS = new Set([
   "provider",
@@ -995,6 +999,45 @@ function sanitizePayload(payload) {
     delete clean.tool_choice;
   }
 
+  // Smart tool selection (universal, applies to all providers)
+  if (config.smartToolSelection?.enabled && Array.isArray(clean.tools) && clean.tools.length > 0) {
+    const { classifyRequestType, selectToolsSmartly } = require('../tools/smart-selection');
+
+    const classification = classifyRequestType(clean);
+
+    logger.debug({
+      originalToolCount: clean.tools.length,
+      classificationType: classification.type,
+      confidence: classification.confidence,
+      keywords: classification.keywords
+    }, "Smart tool selection: classified request");
+
+    const selectedTools = selectToolsSmartly(clean.tools, classification, {
+      provider: providerType,
+      tokenBudget: config.smartToolSelection.tokenBudget,
+      config: config.smartToolSelection
+    });
+
+    const removedTools = clean.tools
+      .filter(t => !selectedTools.find(s => s.name === t.name))
+      .map(t => t.name);
+
+    const keptTools = selectedTools.map(t => t.name);
+
+    logger.info({
+      feature: 'smart_tool_selection',
+      requestType: classification.type,
+      confidence: classification.confidence,
+      originalToolCount: clean.tools.length,
+      selectedToolCount: selectedTools.length,
+      removedTools,
+      keptTools,
+      provider: providerType
+    }, "Smart tool selection applied");
+
+    clean.tools = selectedTools.length > 0 ? selectedTools : undefined;
+  }
+
   clean.stream = payload.stream ?? false;
 
   if (
@@ -1123,6 +1166,30 @@ async function runAgentLoop({
       );
     }
 
+
+    if (steps === 1 && config.historyCompression?.enabled !== false) {
+      try {
+        if (historyCompression.needsCompression(cleanPayload.messages)) {
+          const originalMessages = cleanPayload.messages;
+          cleanPayload.messages = historyCompression.compressHistory(originalMessages, {
+            keepRecentTurns: config.historyCompression?.keepRecentTurns ?? 10,
+            summarizeOlder: config.historyCompression?.summarizeOlder ?? true,
+            enabled: true
+          });
+
+          if (cleanPayload.messages !== originalMessages) {
+            const stats = historyCompression.calculateCompressionStats(originalMessages, cleanPayload.messages);
+            logger.debug({
+              sessionId: session?.id ?? null,
+              ...stats
+            }, 'History compression applied');
+          }
+        }
+      } catch (err) {
+        logger.warn({ err, sessionId: session?.id }, 'History compression failed, continuing with full history');
+      }
+    }
+
     // === MEMORY RETRIEVAL (Titans-inspired long-term memory) ===
     if (config.memory?.enabled !== false && steps === 1) {
       try {
@@ -1153,7 +1220,8 @@ async function runAgentLoop({
               const injectedSystem = memoryRetriever.injectMemoriesIntoSystem(
                 cleanPayload.system,
                 relevantMemories,
-                config.memory.injectionFormat ?? 'system'
+                config.memory.injectionFormat ?? 'system',
+                cleanPayload.messages // Pass recent messages for deduplication
               );
 
               if (typeof injectedSystem === 'string') {
@@ -1169,7 +1237,126 @@ async function runAgentLoop({
       }
     }
 
-    const databricksResponse = await invokeModel(cleanPayload);
+    if (steps === 1 && (config.systemPrompt?.mode === 'dynamic' || config.systemPrompt?.toolDescriptions === 'minimal')) {
+      try {
+        // Compress tool descriptions if configured
+        if (cleanPayload.tools && cleanPayload.tools.length > 0 && config.systemPrompt?.toolDescriptions === 'minimal') {
+          const originalTools = cleanPayload.tools;
+          cleanPayload.tools = systemPrompt.compressToolDescriptions(originalTools, 'minimal');
+
+          const originalSize = JSON.stringify(originalTools).length;
+          const compressedSize = JSON.stringify(cleanPayload.tools).length;
+          const saved = originalSize - compressedSize;
+
+          if (saved > 100) {
+            logger.debug({
+              sessionId: session?.id ?? null,
+              toolCount: cleanPayload.tools.length,
+              originalChars: originalSize,
+              compressedChars: compressedSize,
+              saved,
+              percentage: ((saved / originalSize) * 100).toFixed(1)
+            }, 'Tool descriptions compressed');
+          }
+        }
+
+        // Optimize system prompt if configured
+        if (cleanPayload.system && config.systemPrompt?.mode === 'dynamic') {
+          const originalSystem = cleanPayload.system;
+          const optimizedSystem = systemPrompt.optimizeSystemPrompt(
+            originalSystem,
+            {
+              tools: cleanPayload.tools,
+              messages: cleanPayload.messages
+            },
+            'dynamic'
+          );
+
+          if (optimizedSystem !== originalSystem) {
+            const savings = systemPrompt.calculateSavings(originalSystem, optimizedSystem);
+            cleanPayload.system = optimizedSystem;
+
+            if (savings.tokensSaved > 50) {
+              logger.debug({
+                sessionId: session?.id ?? null,
+                ...savings
+              }, 'System prompt optimized');
+            }
+          }
+        }
+      } catch (err) {
+        logger.warn({ err, sessionId: session?.id }, 'System prompt optimization failed, continuing with original');
+      }
+    }
+
+    if (steps === 1 && config.tokenBudget?.enforcement !== false) {
+      try {
+        const budgetCheck = tokenBudget.checkBudget(cleanPayload);
+
+        if (budgetCheck.atWarning) {
+          logger.warn({
+            sessionId: session?.id ?? null,
+            totalTokens: budgetCheck.totalTokens,
+            warningThreshold: budgetCheck.warningThreshold,
+            maxThreshold: budgetCheck.maxThreshold,
+            overMax: budgetCheck.overMax
+          }, 'Approaching or exceeding token budget');
+
+          if (budgetCheck.overMax) {
+            // Apply adaptive compression to fit within budget
+            const enforcement = tokenBudget.enforceBudget(cleanPayload, {
+              warningThreshold: config.tokenBudget?.warning,
+              maxThreshold: config.tokenBudget?.max,
+              enforcement: true
+            });
+
+            if (enforcement.compressed) {
+              cleanPayload = enforcement.payload;
+              logger.info({
+                sessionId: session?.id ?? null,
+                strategy: enforcement.strategy,
+                initialTokens: enforcement.stats.initialTokens,
+                finalTokens: enforcement.stats.finalTokens,
+                saved: enforcement.stats.saved,
+                percentage: enforcement.stats.percentage,
+                nowWithinBudget: !enforcement.finalBudget.overMax
+              }, 'Token budget enforcement applied');
+            }
+          }
+        }
+      } catch (err) {
+        logger.warn({ err, sessionId: session?.id }, 'Token budget enforcement failed, continuing without enforcement');
+      }
+    }
+
+    // Track estimated token usage before model call
+  const estimatedTokens = config.tokenTracking?.enabled !== false
+    ? tokens.countPayloadTokens(cleanPayload)
+    : null;
+
+  if (estimatedTokens && config.tokenTracking?.enabled !== false) {
+    logger.debug({
+      sessionId: session?.id ?? null,
+      estimated: estimatedTokens,
+      model: cleanPayload.model
+    }, 'Estimated token usage before model call');
+  }
+
+  const databricksResponse = await invokeModel(cleanPayload);
+
+  // Extract and log actual token usage
+  const actualUsage = databricksResponse.ok && config.tokenTracking?.enabled !== false
+    ? tokens.extractUsageFromResponse(databricksResponse.json)
+    : null;
+
+  if (estimatedTokens && actualUsage && config.tokenTracking?.enabled !== false) {
+    tokens.logTokenUsage('model_invocation', estimatedTokens, actualUsage);
+
+    // Record in session metadata
+    if (session) {
+      tokens.recordTokenUsage(session, steps, estimatedTokens, actualUsage, cleanPayload.model);
+    }
+  }
 
     // Handle streaming responses (pass through without buffering)
     if (databricksResponse.stream) {