@lotargo/memory_plugin 1.6.3 → 1.6.5

package/mcp-server/db/sync_queue.js

@@ -3,7 +3,8 @@ import { join } from "path";
 import { MEMORY_DIR, GLOBAL_KEY, buildMemoryContent, extractFacts, writeMemoryFile } from "../memory.js";
 import { toVectorBytes } from "../retrieval/retriever.js";
 
-let isSyncing = false;
+let isSyncing = false;
+let syncRequested = false;
 
 // Reverse sync (cloud -> local) throttling: only pull at most once per window
 // even if readMemory triggers it frequently (recall hits every keystroke).
@@ -30,13 +31,24 @@ async function processSyncTask(db, task) {
       sql: "SELECT id FROM documents WHERE id = ? OR path = ?;",
       args: [docId, docId],
     });
-    if (docRow.rows.length > 0) {
-      const realDocId = docRow.rows[0].id;
-      await db.cloudClient.execute({ sql: "DELETE FROM micro_chunks_fts WHERE id IN (SELECT id FROM micro_chunks WHERE doc_id = ?);", args: [realDocId] });
+    if (docRow.rows.length > 0) {
+      const realDocId = docRow.rows[0].id;
+      await db.cloudClient.execute({
+        sql: `DELETE FROM graph_edges
+              WHERE source_id = ? OR target_id = ?
+                 OR target_id GLOB ?
+                 OR source_id IN (SELECT id FROM sections WHERE doc_id = ?)
+                 OR target_id IN (SELECT id FROM sections WHERE doc_id = ?)
+                 OR source_id IN (SELECT id FROM medium_chunks WHERE doc_id = ?)
+                 OR target_id IN (SELECT id FROM medium_chunks WHERE doc_id = ?)
+                 OR source_id IN (SELECT id FROM micro_chunks WHERE doc_id = ?)
+                 OR target_id IN (SELECT id FROM micro_chunks WHERE doc_id = ?);`,
+        args: [realDocId, realDocId, `${realDocId}:L*`, realDocId, realDocId, realDocId, realDocId, realDocId, realDocId],
+      });
+      await db.cloudClient.execute({ sql: "DELETE FROM micro_chunks_fts WHERE id IN (SELECT id FROM micro_chunks WHERE doc_id = ?);", args: [realDocId] });
       await db.cloudClient.execute({ sql: "DELETE FROM micro_chunks WHERE doc_id = ?;", args: [realDocId] });
       await db.cloudClient.execute({ sql: "DELETE FROM medium_chunks WHERE doc_id = ?;", args: [realDocId] });
       await db.cloudClient.execute({ sql: "DELETE FROM sections WHERE doc_id = ?;", args: [realDocId] });
-      await db.cloudClient.execute({ sql: "DELETE FROM graph_edges WHERE source_id = ? OR target_id = ?;", args: [realDocId, realDocId] });
       await db.cloudClient.execute({ sql: "DELETE FROM knowledge_links WHERE doc_id = ?;", args: [realDocId] });
       await db.cloudClient.execute({ sql: "DELETE FROM documents WHERE id = ?;", args: [realDocId] });
     }
@@ -52,14 +64,25 @@ async function processSyncTask(db, task) {
       sql: "SELECT id FROM documents WHERE id = ? OR path = ?;",
       args: [doc.id, doc.path],
     });
-    if (existingDocRow.rows.length > 0) {
-      const realDocId = existingDocRow.rows[0].id;
-      await db.cloudClient.execute({ sql: "DELETE FROM micro_chunks_fts WHERE id IN (SELECT id FROM micro_chunks WHERE doc_id = ?);", args: [realDocId] });
+    if (existingDocRow.rows.length > 0) {
+      const realDocId = existingDocRow.rows[0].id;
+      await db.cloudClient.execute({
+        sql: `DELETE FROM graph_edges
+              WHERE source_id = ? OR target_id = ?
+                 OR target_id GLOB ?
+                 OR source_id IN (SELECT id FROM sections WHERE doc_id = ?)
+                 OR target_id IN (SELECT id FROM sections WHERE doc_id = ?)
+                 OR source_id IN (SELECT id FROM medium_chunks WHERE doc_id = ?)
+                 OR target_id IN (SELECT id FROM medium_chunks WHERE doc_id = ?)
+                 OR source_id IN (SELECT id FROM micro_chunks WHERE doc_id = ?)
+                 OR target_id IN (SELECT id FROM micro_chunks WHERE doc_id = ?);`,
+        args: [realDocId, realDocId, `${realDocId}:L*`, realDocId, realDocId, realDocId, realDocId, realDocId, realDocId],
+      });
+      await db.cloudClient.execute({ sql: "DELETE FROM micro_chunks_fts WHERE id IN (SELECT id FROM micro_chunks WHERE doc_id = ?);", args: [realDocId] });
       await db.cloudClient.execute({ sql: "DELETE FROM micro_chunks WHERE doc_id = ?;", args: [realDocId] });
       await db.cloudClient.execute({ sql: "DELETE FROM medium_chunks WHERE doc_id = ?;", args: [realDocId] });
       await db.cloudClient.execute({ sql: "DELETE FROM sections WHERE doc_id = ?;", args: [realDocId] });
-      await db.cloudClient.execute({ sql: "DELETE FROM graph_edges WHERE source_id = ? OR target_id = ?;", args: [realDocId, realDocId] });
-      await db.cloudClient.execute({ sql: "DELETE FROM knowledge_links WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM knowledge_links WHERE doc_id = ?;", args: [realDocId] });
       await db.cloudClient.execute({ sql: "DELETE FROM documents WHERE id = ?;", args: [realDocId] });
     }
 
@@ -79,8 +102,18 @@ async function processSyncTask(db, task) {
         doc.metadata_json ? (typeof doc.metadata_json === "string" ? doc.metadata_json : JSON.stringify(doc.metadata_json)) : null,
         doc.created_at,
         doc.updated_at,
-      ],
-    });
+      ],
+    });
+
+    const scopes = Array.isArray(data.document_scopes) && data.document_scopes.length
+      ? data.document_scopes
+      : [{ scope_key: "global", created_at: doc.created_at }];
+    for (const scope of scopes) {
+      await db.cloudClient.execute({
+        sql: "INSERT OR IGNORE INTO document_scopes (doc_id, scope_key, created_at) VALUES (?, ?, ?);",
+        args: [doc.id, scope.scope_key || "global", scope.created_at || Date.now()],
+      });
+    }
 
     // 3. Insert sections
     if (Array.isArray(data.sections)) {
@@ -112,8 +145,18 @@ async function processSyncTask(db, task) {
           ? Buffer.from(vecBytes.buffer, vecBytes.byteOffset, vecBytes.byteLength)
           : Buffer.alloc(0);
         await db.cloudClient.execute({
-          sql: "INSERT INTO micro_chunks (id, section_id, doc_id, content, vector, token_count, medium_id) VALUES (?, ?, ?, ?, ?, ?, ?);",
-          args: [mc.id, mc.section_id, doc.id, mc.content, vecBuf, mc.token_count, mc.medium_id || null],
+          sql: "INSERT INTO micro_chunks (id, section_id, doc_id, content, vector, token_count, medium_id, retrieval_policy, policy_source_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);",
+          args: [
+            mc.id,
+            mc.section_id,
+            doc.id,
+            mc.content,
+            vecBuf,
+            mc.token_count,
+            mc.medium_id || null,
+            mc.retrieval_policy || "micro_chunk",
+            mc.policy_source_id || null,
+          ],
         });
 
         // Insert into remote FTS
@@ -129,16 +172,38 @@ async function processSyncTask(db, task) {
     }
 
     // 6. Insert graph_edges
-    if (Array.isArray(data.graph_edges)) {
+    if (Array.isArray(data.graph_edges)) {
       for (const e of data.graph_edges) {
         await db.cloudClient.execute({
           sql: "INSERT OR IGNORE INTO graph_edges (source_id, target_id, relation_type, metadata_json, created_at) VALUES (?, ?, ?, ?, ?);",
           args: [e.source_id, e.target_id, e.relation_type, e.metadata_json ? (typeof e.metadata_json === "string" ? e.metadata_json : JSON.stringify(e.metadata_json)) : null, e.created_at || Date.now()],
         });
-      }
-    }
-  }
-}
+      }
+    }
+
+    if (Array.isArray(data.knowledge_links)) {
+      for (const link of data.knowledge_links) {
+        await db.cloudClient.execute({
+          sql: `INSERT OR REPLACE INTO knowledge_links
+                (id, fact_key, fact_text, doc_id, section_id, start_line, end_line, relation_type, metadata_json, created_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);`,
+          args: [
+            link.id,
+            link.fact_key,
+            link.fact_text,
+            doc.id,
+            link.section_id || null,
+            link.start_line || null,
+            link.end_line || null,
+            link.relation_type || "LINKS_TO",
+            link.metadata_json || null,
+            link.created_at || Date.now(),
+          ],
+        });
+      }
+    }
+  }
+}
 
 export async function enqueueSyncTask(action, keyOrId, payload = null) {
   const { getDatabase } = await import("./database.js");
@@ -317,9 +382,13 @@ export function resetReverseSyncThrottle() {
   lastReverseSync = 0;
 }
 
-export async function triggerBackgroundSync() {
-  if (isSyncing) return;
-  isSyncing = true;
+export async function triggerBackgroundSync() {
+  if (isSyncing) {
+    syncRequested = true;
+    return;
+  }
+  isSyncing = true;
+  syncRequested = false;
 
   try {
     const { getDatabase } = await import("./database.js");
@@ -340,30 +409,35 @@ export async function triggerBackgroundSync() {
       );
     `);
 
-    // Fetch tasks ordered by id
-    const tasks = await db.prepare("SELECT * FROM sync_queue ORDER BY id ASC LIMIT 50;").all();
-    if (tasks.length === 0) {
-      isSyncing = false;
-      return;
-    }
-
-    for (const task of tasks) {
-      try {
-        await processSyncTask(db, task);
-        // On success, delete from queue
-        await db.prepare("DELETE FROM sync_queue WHERE id = ?;").run(task.id);
-      } catch (err) {
-        console.error(`Failed to process sync task ${task.id} (${task.action}):`, err.message, err.stack);
-        // Stop processing this batch to preserve order on error, retry next time
-        break;
-      }
-    }
+    // Drain until empty. New tasks can be enqueued while a batch is running;
+    // stopping after one snapshot of the queue left those tasks stranded until
+    // an unrelated future write happened to wake the worker again.
+    let syncFailed = false;
+    while (!syncFailed) {
+      const tasks = await db.prepare("SELECT * FROM sync_queue ORDER BY id ASC LIMIT 50;").all();
+      if (tasks.length === 0) break;
+
+      for (const task of tasks) {
+        try {
+          await processSyncTask(db, task);
+          await db.prepare("DELETE FROM sync_queue WHERE id = ?;").run(task.id);
+        } catch (err) {
+          console.error(`Failed to process sync task ${task.id} (${task.action}):`, err.message, err.stack);
+          syncFailed = true;
+          break;
+        }
+      }
+    }
 
     // Push queue drained — now pull cloud state back down (reverse sync).
     await syncFromCloud();
   } catch (err) {
     console.error("Error during background sync execution:", err.message);
-  } finally {
-    isSyncing = false;
-  }
-}
+  } finally {
+    isSyncing = false;
+    if (syncRequested) {
+      syncRequested = false;
+      await triggerBackgroundSync();
+    }
+  }
+}

package/mcp-server/graph/knowledge_linker.js

@@ -1,5 +1,18 @@
 import { getDatabase } from "../db/database.js";
-import { randomUUID } from "node:crypto";
+import { randomUUID } from "node:crypto";
+import { GLOBAL_KEY } from "../memory.js";
+
+export async function queueDocumentSyncIfNeeded(db, docId) {
+  try {
+    const { getConfig } = await import("../config/config_manager.js");
+    if (getConfig().mode !== "hybrid-sync") return;
+    const { exportDocumentData } = await import("../ingest/exporter.js");
+    const { enqueueSyncTask } = await import("../db/sync_queue.js");
+    await enqueueSyncTask("ingest_document", docId, await exportDocumentData(docId, db));
+  } catch (err) {
+    console.warn(`Failed to queue linked RAG document sync for ${docId}: ${err.message}`);
+  }
+}
 
 export async function linkFactToDocument({
   factKey,
@@ -15,9 +28,19 @@ export async function linkFactToDocument({
   const id = `link_${randomUUID().substring(0, 12)}`;
   const now = Date.now();
 
-  const doc = await db
-    .prepare("SELECT id, title, path FROM documents WHERE id = ? OR path = ? OR title = ?")
-    .get(docId, docId, docId);
+  const allowedScopes = factKey === GLOBAL_KEY ? [GLOBAL_KEY] : [GLOBAL_KEY, factKey];
+  const placeholders = allowedScopes.map(() => "?").join(",");
+  const doc = await db
+    .prepare(`
+      SELECT d.id, d.title, d.path
+      FROM documents d
+      WHERE (d.id = ? OR d.path = ? OR d.title = ?)
+        AND EXISTS (
+          SELECT 1 FROM document_scopes ds
+          WHERE ds.doc_id = d.id AND ds.scope_key IN (${placeholders})
+        )
+    `)
+    .get(docId, docId, docId, ...allowedScopes);
 
   if (!doc) {
     throw new Error(`Target document not found in knowledge base for link: ${docId}`);
@@ -46,7 +69,8 @@ export async function linkFactToDocument({
     VALUES (?, ?, ?, ?, ?)
   `);
   const targetSpec = startLine ? `${doc.id}:L${startLine}-${endLine || startLine}` : doc.id;
-  await edgeStmt.run(`fact:${factKey}:${factText.substring(0, 30)}`, targetSpec, relationType, JSON.stringify({ linkId: id }), now);
+  await edgeStmt.run(`fact:${factKey}:${factText.substring(0, 30)}`, targetSpec, relationType, JSON.stringify({ linkId: id }), now);
+  await queueDocumentSyncIfNeeded(db, doc.id);
 
   return {
     linkId: id,
@@ -73,19 +97,33 @@ export async function getLinksForFact(factKey, factText) {
   return await stmt.all(factKey, queryPattern, factText);
 }
 
-export async function getLinksForDoc(docId) {
-  const db = await getDatabase();
-  const stmt = db.prepare(`
-    SELECT k.*, d.title as doc_title, d.path as doc_path
-    FROM knowledge_links k
-    JOIN documents d ON k.doc_id = d.id
-    WHERE k.doc_id = ? OR d.path = ? OR d.title = ?
-    ORDER BY k.created_at DESC
-  `);
-  return await stmt.all(docId, docId, docId);
-}
+export async function getLinksForDoc(docId, scopeKeys = null) {
+  const db = await getDatabase();
+  const scoped = Array.isArray(scopeKeys) && scopeKeys.length > 0;
+  const scopeClause = scoped
+    ? `AND k.fact_key IN (${scopeKeys.map(() => "?").join(",")})
+      AND EXISTS (
+        SELECT 1 FROM document_scopes ds
+        WHERE ds.doc_id = d.id AND ds.scope_key IN (${scopeKeys.map(() => "?").join(",")})
+      )`
+    : "";
+  const stmt = db.prepare(`
+    SELECT k.*, d.title as doc_title, d.path as doc_path
+    FROM knowledge_links k
+    JOIN documents d ON k.doc_id = d.id
+    WHERE (k.doc_id = ? OR d.path = ? OR d.title = ?)
+    ${scopeClause}
+    ORDER BY k.created_at DESC
+  `);
+  return await stmt.all(
+    docId,
+    docId,
+    docId,
+    ...(scoped ? [...scopeKeys, ...scopeKeys] : [])
+  );
+}
 
-export async function listAllLinks(factKey = null) {
+export async function listAllLinks(factKey = null) {
   const db = await getDatabase();
   let sql = `
     SELECT k.*, d.title as doc_title, d.path as doc_path
@@ -98,5 +136,74 @@ export async function listAllLinks(factKey = null) {
     params.push(factKey);
   }
   sql += " ORDER BY k.created_at DESC";
-  return await db.prepare(sql).all(...params);
-}
+  return await db.prepare(sql).all(...params);
+}
+
+export async function moveKnowledgeScope(db, sourceKey, targetKey) {
+  if (!sourceKey || !targetKey || sourceKey === targetKey) {
+    return { movedLinks: 0, movedDocuments: 0 };
+  }
+
+  const links = await db.prepare("SELECT * FROM knowledge_links WHERE fact_key = ?").all(sourceKey);
+  const scopedDocs = await db.prepare("SELECT doc_id, created_at FROM document_scopes WHERE scope_key = ?").all(sourceKey);
+  const docIds = new Set([...scopedDocs.map((row) => row.doc_id), ...links.map((link) => link.doc_id)]);
+
+  for (const docId of docIds) {
+    await db
+      .prepare("INSERT OR IGNORE INTO document_scopes (doc_id, scope_key, created_at) VALUES (?, ?, ?)")
+      .run(docId, targetKey, Date.now());
+  }
+  await db.prepare("DELETE FROM document_scopes WHERE scope_key = ?").run(sourceKey);
+
+  for (const link of links) {
+    const oldSource = `fact:${sourceKey}:${link.fact_text.substring(0, 30)}`;
+    const targetSpec = link.start_line
+      ? `${link.doc_id}:L${link.start_line}-${link.end_line || link.start_line}`
+      : link.doc_id;
+    await db
+      .prepare("DELETE FROM graph_edges WHERE source_id = ? AND target_id = ? AND relation_type = ?")
+      .run(oldSource, targetSpec, link.relation_type || "LINKS_TO");
+    const newSource = `fact:${targetKey}:${link.fact_text.substring(0, 30)}`;
+    await db.prepare(`
+      INSERT OR IGNORE INTO graph_edges (source_id, target_id, relation_type, metadata_json, created_at)
+      VALUES (?, ?, ?, ?, ?)
+    `).run(
+      newSource,
+      targetSpec,
+      link.relation_type || "LINKS_TO",
+      JSON.stringify({ linkId: link.id }),
+      link.created_at || Date.now()
+    );
+  }
+  await db.prepare("UPDATE knowledge_links SET fact_key = ? WHERE fact_key = ?").run(targetKey, sourceKey);
+
+  for (const docId of docIds) {
+    await queueDocumentSyncIfNeeded(db, docId);
+  }
+
+  return { movedLinks: links.length, movedDocuments: docIds.size };
+}
+
+export async function deleteLinksForFacts(db, factKey, factTexts) {
+  const texts = [...new Set((factTexts || []).filter(Boolean))];
+  if (!factKey || texts.length === 0) return { deletedLinks: 0, affectedDocuments: 0 };
+  const placeholders = texts.map(() => "?").join(",");
+  const links = await db
+    .prepare(`SELECT * FROM knowledge_links WHERE fact_key = ? AND fact_text IN (${placeholders})`)
+    .all(factKey, ...texts);
+  for (const link of links) {
+    const source = `fact:${factKey}:${link.fact_text.substring(0, 30)}`;
+    const target = link.start_line
+      ? `${link.doc_id}:L${link.start_line}-${link.end_line || link.start_line}`
+      : link.doc_id;
+    await db
+      .prepare("DELETE FROM graph_edges WHERE source_id = ? AND target_id = ? AND relation_type = ?")
+      .run(source, target, link.relation_type || "LINKS_TO");
+  }
+  await db
+    .prepare(`DELETE FROM knowledge_links WHERE fact_key = ? AND fact_text IN (${placeholders})`)
+    .run(factKey, ...texts);
+  const docIds = [...new Set(links.map((link) => link.doc_id))];
+  for (const docId of docIds) await queueDocumentSyncIfNeeded(db, docId);
+  return { deletedLinks: links.length, affectedDocuments: docIds.length };
+}

package/mcp-server/ingest/exporter.js

@@ -1,7 +1,8 @@
 import { getDatabase } from "../db/database.js";
 import { writeFileSync, mkdirSync, existsSync } from "node:fs";
 import { join } from "node:path";
-import { MEMORY_DIR } from "../memory.js";
+import { MEMORY_DIR } from "../memory.js";
+import { toVectorBytes } from "../retrieval/retriever.js";
 
 export const EXPORTS_DIR = join(MEMORY_DIR, "exports");
 
@@ -35,8 +36,32 @@ export async function exportDocumentData(docIdOrPath, customDb = null) {
 
   const sections = await db.prepare("SELECT id, heading, breadcrumbs, content, token_count FROM sections WHERE doc_id = ?").all(doc.id);
   const mediumChunks = await db.prepare("SELECT id, section_id, content, block_type, token_count, created_at FROM medium_chunks WHERE doc_id = ?").all(doc.id);
-  const microChunks = await db.prepare("SELECT id, medium_id, section_id, content, token_count FROM micro_chunks WHERE doc_id = ?").all(doc.id);
-  const graphEdges = await db.prepare("SELECT source_id, target_id, relation_type, metadata_json FROM graph_edges WHERE source_id = ? OR target_id = ?").all(doc.id, doc.id);
+  const rawMicroChunks = await db.prepare("SELECT id, medium_id, section_id, content, vector, token_count, retrieval_policy, policy_source_id FROM micro_chunks WHERE doc_id = ?").all(doc.id);
+  const microChunks = rawMicroChunks.map((chunk) => {
+    const bytes = toVectorBytes(chunk.vector);
+    return {
+      ...chunk,
+      vector: bytes && bytes.byteLength
+        ? Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64")
+        : "",
+    };
+  });
+  const ownedRows = await db.prepare(`
+    SELECT id FROM sections WHERE doc_id = ?
+    UNION SELECT id FROM medium_chunks WHERE doc_id = ?
+    UNION SELECT id FROM micro_chunks WHERE doc_id = ?;
+  `).all(doc.id, doc.id, doc.id);
+  const ownedIds = [doc.id, ...ownedRows.map((row) => row.id)];
+  const placeholders = ownedIds.map(() => "?").join(",");
+  const graphEdges = await db.prepare(`
+    SELECT source_id, target_id, relation_type, metadata_json, created_at
+    FROM graph_edges
+    WHERE source_id IN (${placeholders})
+       OR target_id IN (${placeholders})
+       OR target_id GLOB ?;
+  `).all(...ownedIds, ...ownedIds, `${doc.id}:L*`);
+  const knowledgeLinks = await db.prepare("SELECT * FROM knowledge_links WHERE doc_id = ?").all(doc.id);
+  const documentScopes = await db.prepare("SELECT scope_key, created_at FROM document_scopes WHERE doc_id = ?").all(doc.id);
 
   return {
     document: {
@@ -44,9 +69,11 @@ export async function exportDocumentData(docIdOrPath, customDb = null) {
       path: doc.path,
       title: doc.title,
       blob_hash: doc.blob_hash,
-      checksum: doc.checksum,
-      toc,
-      metadata,
+      checksum: doc.checksum,
+      toc_json: doc.toc_json,
+      metadata_json: doc.metadata_json,
+      toc,
+      metadata,
       created_at: doc.created_at,
       updated_at: doc.updated_at,
     },
@@ -59,9 +86,11 @@ export async function exportDocumentData(docIdOrPath, customDb = null) {
     sections,
     medium_chunks: mediumChunks,
     micro_chunks: microChunks,
-    graph_edges: graphEdges,
-  };
-}
+    graph_edges: graphEdges,
+    knowledge_links: knowledgeLinks,
+    document_scopes: documentScopes,
+  };
+}
 
 export async function exportDocumentToJsonString(docIdOrPath, customDb = null) {
   const data = await exportDocumentData(docIdOrPath, customDb);