lynkr 2.0.0 → 3.0.0

package/src/memory/search.js

@@ -0,0 +1,239 @@
+const db = require("../db");
+const logger = require("../logger");
+const store = require("./store");
+
+/**
+ * Search memories using FTS5 full-text search
+ */
+function searchMemories(options) {
+  const {
+    query,
+    limit = 10,
+    types = null,      // Filter by memory types
+    categories = null, // Filter by categories
+    sessionId = null,  // Filter by session
+    minImportance = null,
+  } = options;
+
+  if (!query || typeof query !== "string") {
+    logger.warn("Search query must be a non-empty string");
+    return [];
+  }
+
+  // Build FTS5 query - escape special characters
+  const ftsQuery = prepareFTS5Query(query);
+
+  // Build SQL with filters
+  let sql = `
+    SELECT m.id, m.session_id, m.content, m.type, m.category,
+           m.importance, m.surprise_score, m.access_count, m.decay_factor,
+           m.source_turn_id, m.created_at, m.updated_at, m.last_accessed_at, m.metadata,
+           fts.rank
+    FROM memories_fts fts
+    JOIN memories m ON m.id = fts.rowid
+    WHERE memories_fts MATCH ?
+  `;
+
+  const params = [ftsQuery];
+
+  // Add filters
+  if (sessionId) {
+    sql += ` AND (m.session_id = ? OR m.session_id IS NULL)`;
+    params.push(sessionId);
+  }
+
+  if (types && Array.isArray(types) && types.length > 0) {
+    const placeholders = types.map(() => "?").join(",");
+    sql += ` AND m.type IN (${placeholders})`;
+    params.push(...types);
+  }
+
+  if (categories && Array.isArray(categories) && categories.length > 0) {
+    const placeholders = categories.map(() => "?").join(",");
+    sql += ` AND m.category IN (${placeholders})`;
+    params.push(...categories);
+  }
+
+  if (minImportance !== null && typeof minImportance === "number") {
+    sql += ` AND m.importance >= ?`;
+    params.push(minImportance);
+  }
+
+  // Order by FTS5 rank and importance
+  sql += ` ORDER BY fts.rank, m.importance DESC LIMIT ?`;
+  params.push(limit);
+
+  try {
+    const stmt = db.prepare(sql);
+    const rows = stmt.all(...params);
+
+    return rows.map((row) => ({
+      id: row.id,
+      sessionId: row.session_id ?? null,
+      content: row.content,
+      type: row.type,
+      category: row.category ?? null,
+      importance: row.importance ?? 0.5,
+      surpriseScore: row.surprise_score ?? 0.0,
+      accessCount: row.access_count ?? 0,
+      decayFactor: row.decay_factor ?? 1.0,
+      sourceTurnId: row.source_turn_id ?? null,
+      createdAt: row.created_at,
+      updatedAt: row.updated_at,
+      lastAccessedAt: row.last_accessed_at ?? null,
+      metadata: row.metadata ? JSON.parse(row.metadata) : {},
+      rank: row.rank, // FTS5 relevance score
+    }));
+  } catch (err) {
+    logger.error({ err, query: ftsQuery }, "FTS5 search failed");
+    return [];
+  }
+}
+
+/**
+ * Prepare FTS5 query - handle special characters and phrases
+ */
+function prepareFTS5Query(query) {
+  // Remove or escape FTS5 special characters: " * ( ) AND OR NOT
+  // For simple queries, just escape quotes and use phrase matching
+  let cleaned = query.trim();
+
+  // If query looks like a phrase, wrap in quotes for exact matching
+  if (!cleaned.includes('"') && cleaned.split(/\s+/).length > 1) {
+    // Multi-word query - use phrase search
+    cleaned = `"${cleaned.replace(/"/g, '""')}"`;
+  } else {
+    // Single word or already has quotes - just escape
+    cleaned = cleaned.replace(/"/g, '""');
+  }
+
+  return cleaned;
+}
+
+/**
+ * Search with keyword expansion (extract key terms)
+ */
+function searchWithExpansion(options) {
+  const { query, limit = 10 } = options;
+
+  // Extract keywords from query
+  const keywords = extractKeywords(query);
+
+  // Search with original query
+  const results = searchMemories({ ...options, limit: limit * 2 });
+
+  // If not enough results, try individual keywords
+  if (results.length < limit && keywords.length > 1) {
+    const seen = new Set(results.map((r) => r.id));
+
+    for (const keyword of keywords) {
+      if (results.length >= limit) break;
+
+      const kwResults = searchMemories({
+        ...options,
+        query: keyword,
+        limit: limit - results.length,
+      });
+
+      for (const result of kwResults) {
+        if (!seen.has(result.id)) {
+          results.push(result);
+          seen.add(result.id);
+        }
+      }
+    }
+  }
+
+  return results.slice(0, limit);
+}
+
+/**
+ * Extract keywords from text (simple tokenization)
+ */
+function extractKeywords(text) {
+  if (!text) return [];
+
+  // Simple keyword extraction:
+  // - Split on whitespace
+  // - Remove stopwords
+  // - Keep words > 3 characters
+  // - Lowercase
+
+  const stopwords = new Set([
+    "the",
+    "is",
+    "at",
+    "which",
+    "on",
+    "and",
+    "or",
+    "not",
+    "this",
+    "that",
+    "with",
+    "from",
+    "for",
+    "to",
+    "in",
+    "of",
+    "a",
+    "an",
+  ]);
+
+  return text
+    .toLowerCase()
+    .split(/\s+/)
+    .map((word) => word.replace(/[^\w]/g, ""))
+    .filter((word) => word.length > 3 && !stopwords.has(word));
+}
+
+/**
+ * Find similar memories by keyword overlap
+ */
+function findSimilar(memoryId, limit = 5) {
+  const memory = store.getMemory(memoryId);
+  if (!memory) return [];
+
+  const keywords = extractKeywords(memory.content);
+  if (keywords.length === 0) return [];
+
+  // Build OR query for keywords
+  const query = keywords.join(" OR ");
+
+  const results = searchMemories({
+    query,
+    limit: limit + 1, // +1 to exclude self
+  });
+
+  // Filter out the original memory
+  return results.filter((r) => r.id !== memoryId).slice(0, limit);
+}
+
+/**
+ * Search by content similarity (simple keyword-based)
+ */
+function searchByContent(content, options = {}) {
+  const keywords = extractKeywords(content);
+  if (keywords.length === 0) return [];
+
+  const query = keywords.slice(0, 5).join(" OR "); // Top 5 keywords
+  return searchMemories({ ...options, query });
+}
+
+/**
+ * Count search results without fetching them
+ */
+function countSearchResults(query, options = {}) {
+  const results = searchMemories({ ...options, query, limit: 1000 });
+  return results.length;
+}
+
+module.exports = {
+  searchMemories,
+  searchWithExpansion,
+  extractKeywords,
+  findSimilar,
+  searchByContent,
+  countSearchResults,
+  prepareFTS5Query,
+};

package/src/memory/store.js

@@ -0,0 +1,411 @@
+const db = require("../db");
+const logger = require("../logger");
+
+// Prepared statements for memory operations
+const insertMemoryStmt = db.prepare(`
+  INSERT INTO memories (
+    session_id, content, type, category, importance, surprise_score,
+    access_count, decay_factor, source_turn_id, created_at, updated_at,
+    last_accessed_at, metadata
+  ) VALUES (
+    @session_id, @content, @type, @category, @importance, @surprise_score,
+    @access_count, @decay_factor, @source_turn_id, @created_at, @updated_at,
+    @last_accessed_at, @metadata
+  )
+`);
+
+const getMemoryStmt = db.prepare(`
+  SELECT
+    id, session_id, content, type, category, importance, surprise_score,
+    access_count, decay_factor, source_turn_id, created_at, updated_at,
+    last_accessed_at, metadata
+  FROM memories
+  WHERE id = ?
+`);
+
+const updateMemoryStmt = db.prepare(`
+  UPDATE memories
+  SET content = @content,
+      type = @type,
+      category = @category,
+      importance = @importance,
+      surprise_score = @surprise_score,
+      decay_factor = @decay_factor,
+      updated_at = @updated_at,
+      metadata = @metadata
+  WHERE id = @id
+`);
+
+const deleteMemoryStmt = db.prepare("DELETE FROM memories WHERE id = ?");
+
+const incrementAccessStmt = db.prepare(`
+  UPDATE memories
+  SET access_count = access_count + 1,
+      last_accessed_at = ?
+  WHERE id = ?
+`);
+
+const updateImportanceStmt = db.prepare(`
+  UPDATE memories
+  SET importance = ?,
+      updated_at = ?
+  WHERE id = ?
+`);
+
+const getRecentMemoriesStmt = db.prepare(`
+  SELECT
+    id, session_id, content, type, category, importance, surprise_score,
+    access_count, decay_factor, source_turn_id, created_at, updated_at,
+    last_accessed_at, metadata
+  FROM memories
+  WHERE (session_id = ? OR ? IS NULL)
+  ORDER BY created_at DESC
+  LIMIT ?
+`);
+
+const getMemoriesByImportanceStmt = db.prepare(`
+  SELECT
+    id, session_id, content, type, category, importance, surprise_score,
+    access_count, decay_factor, source_turn_id, created_at, updated_at,
+    last_accessed_at, metadata
+  FROM memories
+  WHERE (session_id = ? OR ? IS NULL)
+  ORDER BY importance DESC, created_at DESC
+  LIMIT ?
+`);
+
+const getMemoriesBySurpriseStmt = db.prepare(`
+  SELECT
+    id, session_id, content, type, category, importance, surprise_score,
+    access_count, decay_factor, source_turn_id, created_at, updated_at,
+    last_accessed_at, metadata
+  FROM memories
+  WHERE surprise_score >= ?
+  ORDER BY surprise_score DESC, created_at DESC
+  LIMIT ?
+`);
+
+const pruneOldMemoriesStmt = db.prepare(`
+  DELETE FROM memories
+  WHERE created_at < ?
+`);
+
+const pruneByCountStmt = db.prepare(`
+  DELETE FROM memories
+  WHERE id NOT IN (
+    SELECT id FROM memories
+    ORDER BY importance DESC, created_at DESC
+    LIMIT ?
+  )
+`);
+
+const countMemoriesStmt = db.prepare("SELECT COUNT(*) as count FROM memories");
+
+const getMemoriesByTypeStmt = db.prepare(`
+  SELECT
+    id, session_id, content, type, category, importance, surprise_score,
+    access_count, decay_factor, source_turn_id, created_at, updated_at,
+    last_accessed_at, metadata
+  FROM memories
+  WHERE type = ?
+  ORDER BY importance DESC, created_at DESC
+  LIMIT ?
+`);
+
+// Entity tracking
+const upsertEntityStmt = db.prepare(`
+  INSERT INTO memory_entities (entity_type, entity_name, first_seen_at, last_seen_at, occurrence_count, properties)
+  VALUES (@entity_type, @entity_name, @timestamp, @timestamp, 1, @properties)
+  ON CONFLICT(entity_type, entity_name) DO UPDATE SET
+    last_seen_at = @timestamp,
+    occurrence_count = occurrence_count + 1,
+    properties = @properties
+`);
+
+const getEntityStmt = db.prepare(`
+  SELECT id, entity_type, entity_name, first_seen_at, last_seen_at, occurrence_count, properties
+  FROM memory_entities
+  WHERE entity_type = ? AND entity_name = ?
+`);
+
+const getAllEntitiesStmt = db.prepare(`
+  SELECT id, entity_type, entity_name, first_seen_at, last_seen_at, occurrence_count, properties
+  FROM memory_entities
+  ORDER BY occurrence_count DESC
+  LIMIT ?
+`);
+
+// Helper functions
+function parseJSON(value, fallback = null) {
+  if (value === null || value === undefined) return fallback;
+  try {
+    return JSON.parse(value);
+  } catch (err) {
+    logger.warn({ err }, "Failed to parse JSON from memory store");
+    return fallback;
+  }
+}
+
+function serialize(value) {
+  if (value === undefined || value === null) return null;
+  try {
+    return JSON.stringify(value);
+  } catch (err) {
+    logger.warn({ err }, "Failed to serialize JSON for memory store");
+    return null;
+  }
+}
+
+function toMemory(row) {
+  if (!row) return null;
+  return {
+    id: row.id,
+    sessionId: row.session_id ?? null,
+    content: row.content,
+    type: row.type,
+    category: row.category ?? null,
+    importance: row.importance ?? 0.5,
+    surpriseScore: row.surprise_score ?? 0.0,
+    accessCount: row.access_count ?? 0,
+    decayFactor: row.decay_factor ?? 1.0,
+    sourceTurnId: row.source_turn_id ?? null,
+    createdAt: row.created_at,
+    updatedAt: row.updated_at,
+    lastAccessedAt: row.last_accessed_at ?? null,
+    metadata: parseJSON(row.metadata, {}),
+  };
+}
+
+function toEntity(row) {
+  if (!row) return null;
+  return {
+    id: row.id,
+    entityType: row.entity_type,
+    entityName: row.entity_name,
+    firstSeenAt: row.first_seen_at,
+    lastSeenAt: row.last_seen_at,
+    occurrenceCount: row.occurrence_count ?? 1,
+    properties: parseJSON(row.properties, {}),
+  };
+}
+
+/**
+ * Create a new memory
+ */
+function createMemory(options) {
+  const now = Date.now();
+  const {
+    sessionId = null,
+    content,
+    type,
+    category = null,
+    importance = 0.5,
+    surpriseScore = 0.0,
+    accessCount = 0,
+    decayFactor = 1.0,
+    sourceTurnId = null,
+    metadata = {},
+  } = options;
+
+  if (!content || !type) {
+    throw new Error("Memory content and type are required");
+  }
+
+  const result = insertMemoryStmt.run({
+    session_id: sessionId,
+    content,
+    type,
+    category,
+    importance,
+    surprise_score: surpriseScore,
+    access_count: accessCount,
+    decay_factor: decayFactor,
+    source_turn_id: sourceTurnId,
+    created_at: now,
+    updated_at: now,
+    last_accessed_at: null,
+    metadata: serialize(metadata),
+  });
+
+  return {
+    id: result.lastInsertRowid,
+    sessionId,
+    content,
+    type,
+    category,
+    importance,
+    surpriseScore,
+    accessCount,
+    decayFactor,
+    sourceTurnId,
+    createdAt: now,
+    updatedAt: now,
+    lastAccessedAt: null,
+    metadata,
+  };
+}
+
+/**
+ * Get a memory by ID
+ */
+function getMemory(id) {
+  const row = getMemoryStmt.get(id);
+  return toMemory(row);
+}
+
+/**
+ * Update a memory
+ */
+function updateMemory(id, updates) {
+  const memory = getMemory(id);
+  if (!memory) {
+    throw new Error(`Memory with ID ${id} not found`);
+  }
+
+  const now = Date.now();
+  updateMemoryStmt.run({
+    id,
+    content: updates.content ?? memory.content,
+    type: updates.type ?? memory.type,
+    category: updates.category ?? memory.category,
+    importance: updates.importance ?? memory.importance,
+    surprise_score: updates.surpriseScore ?? memory.surpriseScore,
+    decay_factor: updates.decayFactor ?? memory.decayFactor,
+    updated_at: now,
+    metadata: serialize(updates.metadata ?? memory.metadata),
+  });
+
+  return getMemory(id);
+}
+
+/**
+ * Delete a memory
+ */
+function deleteMemory(id) {
+  const result = deleteMemoryStmt.run(id);
+  return result.changes > 0;
+}
+
+/**
+ * Increment access count for a memory
+ */
+function incrementAccessCount(id) {
+  const now = Date.now();
+  incrementAccessStmt.run(now, id);
+}
+
+/**
+ * Update importance score
+ */
+function updateImportance(id, importance) {
+  const now = Date.now();
+  updateImportanceStmt.run(importance, now, id);
+}
+
+/**
+ * Get recent memories
+ */
+function getRecentMemories(options = {}) {
+  const { limit = 10, sessionId = null } = options;
+  const rows = getRecentMemoriesStmt.all(sessionId, sessionId, limit);
+  return rows.map(toMemory);
+}
+
+/**
+ * Get memories by importance
+ */
+function getMemoriesByImportance(options = {}) {
+  const { limit = 10, sessionId = null } = options;
+  const rows = getMemoriesByImportanceStmt.all(sessionId, sessionId, limit);
+  return rows.map(toMemory);
+}
+
+/**
+ * Get memories by surprise score
+ */
+function getMemoriesBySurprise(options = {}) {
+  const { minScore = 0.3, limit = 10 } = options;
+  const rows = getMemoriesBySurpriseStmt.all(minScore, limit);
+  return rows.map(toMemory);
+}
+
+/**
+ * Get memories by type
+ */
+function getMemoriesByType(type, limit = 10) {
+  const rows = getMemoriesByTypeStmt.all(type, limit);
+  return rows.map(toMemory);
+}
+
+/**
+ * Prune old memories
+ */
+function pruneOldMemories(olderThanMs) {
+  const threshold = Date.now() - olderThanMs;
+  const result = pruneOldMemoriesStmt.run(threshold);
+  return result.changes;
+}
+
+/**
+ * Prune to keep only top N memories by importance
+ */
+function pruneByCount(maxCount) {
+  const result = pruneByCountStmt.run(maxCount);
+  return result.changes;
+}
+
+/**
+ * Count total memories
+ */
+function countMemories() {
+  const result = countMemoriesStmt.get();
+  return result.count;
+}
+
+/**
+ * Track or update an entity
+ */
+function trackEntity(entityType, entityName, properties = {}) {
+  const now = Date.now();
+  upsertEntityStmt.run({
+    entity_type: entityType,
+    entity_name: entityName,
+    timestamp: now,
+    properties: serialize(properties),
+  });
+}
+
+/**
+ * Get an entity
+ */
+function getEntity(entityType, entityName) {
+  const row = getEntityStmt.get(entityType, entityName);
+  return toEntity(row);
+}
+
+/**
+ * Get all entities
+ */
+function getAllEntities(limit = 100) {
+  const rows = getAllEntitiesStmt.all(limit);
+  return rows.map(toEntity);
+}
+
+module.exports = {
+  createMemory,
+  getMemory,
+  updateMemory,
+  deleteMemory,
+  incrementAccessCount,
+  updateImportance,
+  getRecentMemories,
+  getMemoriesByImportance,
+  getMemoriesBySurprise,
+  getMemoriesByType,
+  pruneOldMemories,
+  pruneByCount,
+  countMemories,
+  trackEntity,
+  getEntity,
+  getAllEntities,
+};