@lotargo/memory_plugin 1.3.2 → 1.4.5

package/mcp-server/db/migrations.js

@@ -2,9 +2,9 @@ const MIGRATIONS = [
   {
     version: 1,
     name: "001_initial_rag_schema",
-    up: (db) => {
+    up: async (db) => {
       // 1. Documents Table
-      db.exec(`
+      await db.exec(`
         CREATE TABLE IF NOT EXISTS documents (
             id TEXT PRIMARY KEY,
             path TEXT UNIQUE NOT NULL,
@@ -19,7 +19,7 @@ const MIGRATIONS = [
       `);
 
       // 2. Sections Table
-      db.exec(`
+      await db.exec(`
         CREATE TABLE IF NOT EXISTS sections (
             id TEXT PRIMARY KEY,
             doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
@@ -31,7 +31,7 @@ const MIGRATIONS = [
       `);
 
       // 3. Micro-Chunks Table
-      db.exec(`
+      await db.exec(`
         CREATE TABLE IF NOT EXISTS micro_chunks (
             id TEXT PRIMARY KEY,
             section_id TEXT NOT NULL REFERENCES sections(id) ON DELETE CASCADE,
@@ -43,7 +43,7 @@ const MIGRATIONS = [
       `);
 
       // 4. Full-Text Search (BM25 Index via SQLite FTS5)
-      db.exec(`
+      await db.exec(`
         CREATE VIRTUAL TABLE IF NOT EXISTS micro_chunks_fts USING fts5(
             id UNINDEXED,
             content,
@@ -52,7 +52,7 @@ const MIGRATIONS = [
       `);
 
       // 5. GraphRAG Lite Edges Table
-      db.exec(`
+      await db.exec(`
         CREATE TABLE IF NOT EXISTS graph_edges (
             source_id TEXT NOT NULL,
             target_id TEXT NOT NULL,
@@ -65,15 +65,15 @@ const MIGRATIONS = [
   {
     version: 2,
     name: "002_agent_knowledge_graph",
-    up: (db) => {
+    up: async (db) => {
       try {
-        db.exec(`ALTER TABLE graph_edges ADD COLUMN metadata_json TEXT;`);
+        await db.exec(`ALTER TABLE graph_edges ADD COLUMN metadata_json TEXT;`);
       } catch (e) {}
       try {
-        db.exec(`ALTER TABLE graph_edges ADD COLUMN created_at INTEGER;`);
+        await db.exec(`ALTER TABLE graph_edges ADD COLUMN created_at INTEGER;`);
       } catch (e) {}
 
-      db.exec(`
+      await db.exec(`
         CREATE TABLE IF NOT EXISTS knowledge_links (
             id TEXT PRIMARY KEY,
             fact_key TEXT NOT NULL,
@@ -92,8 +92,8 @@ const MIGRATIONS = [
   {
     version: 3,
     name: "003_medium_chunks_hierarchy",
-    up: (db) => {
-      db.exec(`
+    up: async (db) => {
+      await db.exec(`
         CREATE TABLE IF NOT EXISTS medium_chunks (
             id TEXT PRIMARY KEY,
             section_id TEXT NOT NULL REFERENCES sections(id) ON DELETE CASCADE,
@@ -106,43 +106,72 @@ const MIGRATIONS = [
       `);
 
       try {
-        db.exec(`ALTER TABLE micro_chunks ADD COLUMN medium_id TEXT REFERENCES medium_chunks(id) ON DELETE CASCADE;`);
+        await db.exec(`ALTER TABLE micro_chunks ADD COLUMN medium_id TEXT REFERENCES medium_chunks(id) ON DELETE CASCADE;`);
       } catch (e) {}
     },
   },
 ];
 
-export function runMigrations(db) {
-  const versionRow = db.prepare("PRAGMA user_version;").get();
-  const currentVersion = versionRow ? versionRow.user_version : 0;
+export async function runMigrations(db) {
+  let currentVersion = 0;
+  try {
+    const row = await db.prepare("SELECT MAX(version) as v FROM schema_migrations;").get();
+    currentVersion = row ? row.v || 0 : 0;
+  } catch (e) {
+    try {
+      const versionRow = await db.prepare("PRAGMA user_version;").get();
+      currentVersion = versionRow ? (versionRow.user_version || 0) : 0;
+    } catch (e2) {}
+  }
+
+  // Ensure schema_migrations table exists for future
+  try {
+    await db.exec(`CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY);`);
+  } catch (e) {}
+
+  // Create notebooks table if not exists
+  try {
+    await db.exec(`
+      CREATE TABLE IF NOT EXISTS notebooks (
+          key TEXT PRIMARY KEY,
+          content TEXT NOT NULL,
+          updated_at INTEGER NOT NULL
+      );
+    `);
+  } catch (e) {}
 
   for (const migration of MIGRATIONS) {
     if (migration.version > currentVersion) {
-      db.exec("BEGIN IMMEDIATE;");
+      await db.exec("BEGIN;");
       try {
-        migration.up(db);
-        db.exec("COMMIT;");
-        db.exec(`PRAGMA user_version = ${migration.version};`);
+        await migration.up(db);
+        await db.prepare("INSERT INTO schema_migrations (version) VALUES (?);").run(migration.version);
+        await db.exec("COMMIT;");
+        try {
+          await db.exec(`PRAGMA user_version = ${migration.version};`);
+        } catch (e) {}
       } catch (err) {
-        db.exec("ROLLBACK;");
+        try {
+          await db.exec("ROLLBACK;");
+        } catch (e) {}
         throw new Error(`Migration ${migration.name} failed: ${err.message}`);
       }
     }
   }
 
   // Defensive table & column check for medium_chunks hierarchy
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS medium_chunks (
-        id TEXT PRIMARY KEY,
-        section_id TEXT NOT NULL REFERENCES sections(id) ON DELETE CASCADE,
-        doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
-        content TEXT NOT NULL,
-        block_type TEXT NOT NULL,
-        token_count INTEGER NOT NULL,
-        created_at INTEGER
-    );
-  `);
   try {
-    db.exec(`ALTER TABLE micro_chunks ADD COLUMN medium_id TEXT REFERENCES medium_chunks(id) ON DELETE CASCADE;`);
+    await db.exec(`
+      CREATE TABLE IF NOT EXISTS medium_chunks (
+          id TEXT PRIMARY KEY,
+          section_id TEXT NOT NULL REFERENCES sections(id) ON DELETE CASCADE,
+          doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+          content TEXT NOT NULL,
+          block_type TEXT NOT NULL,
+          token_count INTEGER NOT NULL,
+          created_at INTEGER
+      );
+    `);
+    await db.exec(`ALTER TABLE micro_chunks ADD COLUMN medium_id TEXT REFERENCES medium_chunks(id) ON DELETE CASCADE;`);
   } catch (e) {}
 }

package/mcp-server/db/sync_queue.js

@@ -0,0 +1,211 @@
+let isSyncing = false;
+
+async function processSyncTask(db, task) {
+  if (task.action === "write_memory") {
+    await db.cloudClient.execute({
+      sql: `
+        INSERT INTO notebooks (key, content, updated_at)
+        VALUES (?, ?, ?)
+        ON CONFLICT(key) DO UPDATE SET content = excluded.content, updated_at = excluded.updated_at;
+      `,
+      args: [task.key_or_id, task.payload, task.created_at],
+    });
+    return;
+  }
+
+  if (task.action === "delete_document") {
+    const docId = task.key_or_id;
+    const docRow = await db.cloudClient.execute({
+      sql: "SELECT id FROM documents WHERE id = ? OR path = ?;",
+      args: [docId, docId],
+    });
+    if (docRow.rows.length > 0) {
+      const realDocId = docRow.rows[0].id;
+      await db.cloudClient.execute({ sql: "DELETE FROM micro_chunks_fts WHERE id IN (SELECT id FROM micro_chunks WHERE doc_id = ?);", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM micro_chunks WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM medium_chunks WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM sections WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM graph_edges WHERE source_id = ? OR target_id = ?;", args: [realDocId, realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM knowledge_links WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM documents WHERE id = ?;", args: [realDocId] });
+    }
+    return;
+  }
+
+  if (task.action === "ingest_document") {
+    const data = JSON.parse(task.payload);
+    const doc = data.document;
+
+    // 1. Delete existing doc from cloud if any
+    const existingDocRow = await db.cloudClient.execute({
+      sql: "SELECT id FROM documents WHERE id = ? OR path = ?;",
+      args: [doc.id, doc.path],
+    });
+    if (existingDocRow.rows.length > 0) {
+      const realDocId = existingDocRow.rows[0].id;
+      await db.cloudClient.execute({ sql: "DELETE FROM micro_chunks_fts WHERE id IN (SELECT id FROM micro_chunks WHERE doc_id = ?);", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM micro_chunks WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM medium_chunks WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM sections WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM graph_edges WHERE source_id = ? OR target_id = ?;", args: [realDocId, realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM knowledge_links WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM documents WHERE id = ?;", args: [realDocId] });
+    }
+
+    // 2. Insert document
+    await db.cloudClient.execute({
+      sql: `
+        INSERT INTO documents (id, path, blob_hash, title, checksum, toc_json, metadata_json, created_at, updated_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);
+      `,
+      args: [
+        doc.id,
+        doc.path,
+        doc.blob_hash,
+        doc.title,
+        doc.checksum,
+        doc.toc_json ? (typeof doc.toc_json === "string" ? doc.toc_json : JSON.stringify(doc.toc_json)) : null,
+        doc.metadata_json ? (typeof doc.metadata_json === "string" ? doc.metadata_json : JSON.stringify(doc.metadata_json)) : null,
+        doc.created_at,
+        doc.updated_at,
+      ],
+    });
+
+    // 3. Insert sections
+    if (Array.isArray(data.sections)) {
+      for (const s of data.sections) {
+        await db.cloudClient.execute({
+          sql: "INSERT INTO sections (id, doc_id, heading, breadcrumbs, content, token_count) VALUES (?, ?, ?, ?, ?, ?);",
+          args: [s.id, doc.id, s.heading, s.breadcrumbs, s.content, s.token_count],
+        });
+      }
+    }
+
+    // 4. Insert medium_chunks
+    if (Array.isArray(data.medium_chunks)) {
+      for (const m of data.medium_chunks) {
+        await db.cloudClient.execute({
+          sql: "INSERT INTO medium_chunks (id, section_id, doc_id, content, block_type, token_count, created_at) VALUES (?, ?, ?, ?, ?, ?, ?);",
+          args: [m.id, m.section_id, doc.id, m.content, m.block_type, m.token_count, m.created_at || Date.now()],
+        });
+      }
+    }
+
+    // 5. Insert micro_chunks & FTS
+    if (Array.isArray(data.micro_chunks)) {
+      for (const mc of data.micro_chunks) {
+        let vecBuf = Buffer.alloc(0);
+        if (mc.vector) {
+          if (Buffer.isBuffer(mc.vector)) {
+            vecBuf = mc.vector;
+          } else if (typeof mc.vector === "string") {
+            vecBuf = Buffer.from(mc.vector, "base64");
+          } else if (mc.vector.type === "Buffer" && Array.isArray(mc.vector.data)) {
+            vecBuf = Buffer.from(mc.vector.data);
+          } else if (Array.isArray(mc.vector)) {
+            vecBuf = Buffer.from(mc.vector);
+          }
+        }
+        await db.cloudClient.execute({
+          sql: "INSERT INTO micro_chunks (id, section_id, doc_id, content, vector, token_count, medium_id) VALUES (?, ?, ?, ?, ?, ?, ?);",
+          args: [mc.id, mc.section_id, doc.id, mc.content, vecBuf, mc.token_count, mc.medium_id || null],
+        });
+
+        // Insert into remote FTS
+        try {
+          await db.cloudClient.execute({
+            sql: "INSERT INTO micro_chunks_fts (id, content, breadcrumbs) VALUES (?, ?, ?);",
+            args: [mc.id, mc.content, mc.breadcrumbs || ""],
+          });
+        } catch (ftsErr) {
+          console.warn("FTS insertion failed on cloud:", ftsErr.message);
+        }
+      }
+    }
+
+    // 6. Insert graph_edges
+    if (Array.isArray(data.graph_edges)) {
+      for (const e of data.graph_edges) {
+        await db.cloudClient.execute({
+          sql: "INSERT OR IGNORE INTO graph_edges (source_id, target_id, relation_type, metadata_json, created_at) VALUES (?, ?, ?, ?, ?);",
+          args: [e.source_id, e.target_id, e.relation_type, e.metadata_json ? (typeof e.metadata_json === "string" ? e.metadata_json : JSON.stringify(e.metadata_json)) : null, e.created_at || Date.now()],
+        });
+      }
+    }
+  }
+}
+
+export async function enqueueSyncTask(action, keyOrId, payload = null) {
+  const { getDatabase } = await import("./database.js");
+  const db = await getDatabase();
+  if (db.mode === "only-cloud") return; // No need to queue in only-cloud mode
+
+  // Ensure queue table exists
+  await db.exec(`
+    CREATE TABLE IF NOT EXISTS sync_queue (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        action TEXT NOT NULL,
+        key_or_id TEXT NOT NULL,
+        payload TEXT,
+        created_at INTEGER NOT NULL
+    );
+  `);
+
+  await db.prepare(`
+    INSERT INTO sync_queue (action, key_or_id, payload, created_at)
+    VALUES (?, ?, ?, ?);
+  `).run(action, keyOrId, payload ? (typeof payload === "string" ? payload : JSON.stringify(payload)) : null, Date.now());
+
+  // Trigger background sync worker asynchronously
+  triggerBackgroundSync().catch((err) => {
+    console.error("Background sync trigger error:", err.message);
+  });
+}
+
+export async function triggerBackgroundSync() {
+  if (isSyncing) return;
+  isSyncing = true;
+
+  try {
+    const { getDatabase } = await import("./database.js");
+    const db = await getDatabase();
+    if (db.mode !== "hybrid-sync" || !db.cloudClient) {
+      isSyncing = false;
+      return;
+    }
+
+    // Ensure table exists
+    await db.exec(`
+      CREATE TABLE IF NOT EXISTS sync_queue (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          action TEXT NOT NULL,
+          key_or_id TEXT NOT NULL,
+          payload TEXT,
+          created_at INTEGER NOT NULL
+      );
+    `);
+
+    // Fetch tasks ordered by id
+    const tasks = await db.prepare("SELECT * FROM sync_queue ORDER BY id ASC LIMIT 50;").all();
+    if (tasks.length === 0) {
+      isSyncing = false;
+      return;
+    }
+
+    for (const task of tasks) {
+      try {
+        await processSyncTask(db, task);
+        // On success, delete from queue
+        await db.prepare("DELETE FROM sync_queue WHERE id = ?;").run(task.id);
+      } catch (err) {
+        console.error(`Failed to process sync task ${task.id} (${task.action}):`, err.message, err.stack);
+        // Stop processing this batch to preserve order on error, retry next time
+        break;
+      }
+    }
+  } catch (err) {
+    console.error("Error during background sync execution:", err.message);
+  } finally {
+    isSyncing = false;
+  }
+}

package/mcp-server/graph/graph_extractor.js

@@ -1,20 +1,43 @@
+const ignoredKeywords = new Set([
+  "const", "let", "var", "function", "class", "import", "export", "from", "return", "if", "for", "while", "switch", "case", "default",
+  "def", "self", "lambda", "pass", "yield", "async", "await", "with", "except", "try", "catch", "finally",
+  "func", "type", "struct", "interface", "chan", "map", "go", "defer", "package", "range",
+  "fn", "enum", "trait", "impl", "pub", "mut", "ref", "self", "Self", "match", "use", "mod", "crate",
+  "namespace", "template", "typename", "public", "private", "protected", "virtual", "override", "using", "inline", "static", "constexpr", "extern", "explicit", "friend", "operator", "throw",
+  "record", "synchronized", "final", "void", "throws", "new", "this", "super", "fun", "val", "null", "true", "false",
+  "internal", "readonly", "base", "get", "set", "echo", "exit", "die", "require", "include",
+  "module", "end", "extend", "attr_accessor", "attr_reader", "attr_writer", "nil", "puts", "raise"
+]);
+
 export function extractSymbolsFromContent(content) {
+  if (!content) return [];
   const symbols = new Set();
 
-  const jsTsRegex = /(?:function|class|interface|type|enum|const|let|var)\s+([a-zA-Z0-9_$]+)/g;
-  let match;
-  while ((match = jsTsRegex.exec(content)) !== null) {
-    const symbol = match[1];
-    if (symbol.length > 2 && !["const", "let", "var", "function", "class", "import", "export", "from", "return", "if", "for", "while"].includes(symbol)) {
-      symbols.add(symbol);
-    }
-  }
+  const patterns = [
+    // JS/TS
+    /(?:function|class|interface|type|enum|const|let|var)\s+([a-zA-Z0-9_$]+)/g,
+    // Python / PHP / Ruby
+    /(?:def|class|function|module|trait)\s+([a-zA-Z0-9_!?=]+)/g,
+    // Go / Rust / C++ / Java / Kotlin / C# / PHP (Types)
+    /\b(?:class|struct|interface|record|enum|trait|type|namespace)\s+([a-zA-Z0-9_]+)/g,
+    // Go (Functions)
+    /\bfunc\s+(?:\([^)]+\)\s+)?([a-zA-Z0-9_]+)\s*\(/g,
+    // Rust (Functions)
+    /\bfn\s+([a-zA-Z0-9_]+)/g,
+    // Kotlin (Functions)
+    /\bfun\s+([a-zA-Z0-9_]+)/g,
+    // Java / C# / C++ (Methods/Functions)
+    /\b(?:public|protected|private|static|synchronized|final|async|virtual|override|readonly)*\s*[\w<>\[\]]+\s+([a-zA-Z0-9_]+)\s*\([^)]*\)\s*(?:const|override|noexcept|throws\s+[\w,\s]+|\s)*\s*[{;]/g
+  ];
 
-  const pyRegex = /(?:def|class)\s+([a-zA-Z0-9_]+)/g;
-  while ((match = pyRegex.exec(content)) !== null) {
-    const symbol = match[1];
-    if (symbol.length > 2 && !["def", "class", "self", "return", "import", "from"].includes(symbol)) {
-      symbols.add(symbol);
+  for (const pattern of patterns) {
+    let match;
+    pattern.lastIndex = 0;
+    while ((match = pattern.exec(content)) !== null) {
+      const symbol = match[1];
+      if (symbol && symbol.length > 2 && !ignoredKeywords.has(symbol)) {
+        symbols.add(symbol);
+      }
     }
   }
 
@@ -52,21 +75,21 @@ export function buildGraphEdges(docId, hierarchy) {
   return edges;
 }
 
-export function saveGraphEdges(db, edges) {
+export async function saveGraphEdges(db, edges) {
   const stmt = db.prepare(`
     INSERT OR IGNORE INTO graph_edges (source_id, target_id, relation_type)
     VALUES (?, ?, ?);
   `);
   for (const edge of edges) {
-    stmt.run(edge.source_id, edge.target_id, edge.relation_type);
+    await stmt.run(edge.source_id, edge.target_id, edge.relation_type);
   }
 }
 
-export function getRelatedSymbols(db, sectionId) {
+export async function getRelatedSymbols(db, sectionId) {
   const stmt = db.prepare(`
     SELECT target_id, relation_type FROM graph_edges
     WHERE source_id = ? AND relation_type = 'DEFINES_SYMBOL';
   `);
-  const rows = stmt.all(sectionId);
+  const rows = await stmt.all(sectionId);
   return rows.map((r) => r.target_id.replace("symbol:", ""));
 }

package/mcp-server/graph/knowledge_linker.js

@@ -1,7 +1,7 @@
 import { getDatabase } from "../db/database.js";
 import { randomUUID } from "node:crypto";
 
-export function linkFactToDocument({
+export async function linkFactToDocument({
   factKey,
   factText,
   docId,
@@ -11,11 +11,11 @@ export function linkFactToDocument({
   relationType = "LINKS_TO",
   metadata = null,
 }) {
-  const db = getDatabase();
+  const db = await getDatabase();
   const id = `link_${randomUUID().substring(0, 12)}`;
   const now = Date.now();
 
-  const doc = db
+  const doc = await db
     .prepare("SELECT id, title, path FROM documents WHERE id = ? OR path = ? OR title = ?")
     .get(docId, docId, docId);
 
@@ -28,7 +28,7 @@ export function linkFactToDocument({
     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   `);
 
-  stmt.run(
+  await stmt.run(
     id,
     factKey,
     factText,
@@ -46,7 +46,7 @@ export function linkFactToDocument({
     VALUES (?, ?, ?, ?, ?)
   `);
   const targetSpec = startLine ? `${doc.id}:L${startLine}-${endLine || startLine}` : doc.id;
-  edgeStmt.run(`fact:${factKey}:${factText.substring(0, 30)}`, targetSpec, relationType, JSON.stringify({ linkId: id }), now);
+  await edgeStmt.run(`fact:${factKey}:${factText.substring(0, 30)}`, targetSpec, relationType, JSON.stringify({ linkId: id }), now);
 
   return {
     linkId: id,
@@ -60,8 +60,8 @@ export function linkFactToDocument({
   };
 }
 
-export function getLinksForFact(factKey, factText) {
-  const db = getDatabase();
+export async function getLinksForFact(factKey, factText) {
+  const db = await getDatabase();
   const stmt = db.prepare(`
     SELECT k.*, d.title as doc_title, d.path as doc_path
     FROM knowledge_links k
@@ -70,11 +70,11 @@ export function getLinksForFact(factKey, factText) {
     ORDER BY k.created_at DESC
   `);
   const queryPattern = `%${factText.substring(0, 20)}%`;
-  return stmt.all(factKey, queryPattern, factText);
+  return await stmt.all(factKey, queryPattern, factText);
 }
 
-export function getLinksForDoc(docId) {
-  const db = getDatabase();
+export async function getLinksForDoc(docId) {
+  const db = await getDatabase();
   const stmt = db.prepare(`
     SELECT k.*, d.title as doc_title, d.path as doc_path
     FROM knowledge_links k
@@ -82,11 +82,11 @@ export function getLinksForDoc(docId) {
     WHERE k.doc_id = ? OR d.path = ? OR d.title = ?
     ORDER BY k.created_at DESC
   `);
-  return stmt.all(docId, docId, docId);
+  return await stmt.all(docId, docId, docId);
 }
 
-export function listAllLinks(factKey = null) {
-  const db = getDatabase();
+export async function listAllLinks(factKey = null) {
+  const db = await getDatabase();
   let sql = `
     SELECT k.*, d.title as doc_title, d.path as doc_path
     FROM knowledge_links k
@@ -98,5 +98,5 @@ export function listAllLinks(factKey = null) {
     params.push(factKey);
   }
   sql += " ORDER BY k.created_at DESC";
-  return db.prepare(sql).all(...params);
+  return await db.prepare(sql).all(...params);
 }