@lotargo/memory_plugin 1.5.0 → 1.5.2

package/mcp-server/admin/snapshot.js

@@ -1,10 +1,24 @@
 import { readFileSync, writeFileSync, existsSync, readdirSync, rmSync, statSync } from "node:fs";
 import { gzipSync, gunzipSync } from "node:zlib";
-import { join } from "node:path";
+import { join, resolve } from "node:path";
 import { getDatabase, BLOBS_DIR } from "../db/database.js";
 import { readBlob, saveBlob } from "../storage/blob_store.js";
 import { ensureExportsDir } from "../ingest/exporter.js";
 
+export function validateSnapshotPath(pathStr, isExport = false) {
+  if (typeof pathStr !== "string" || !pathStr.trim()) {
+    throw new Error("Snapshot path cannot be empty.");
+  }
+  const resolved = resolve(pathStr.trim());
+  if (!resolved.endsWith(".json") && !resolved.endsWith(".json.gz")) {
+    throw new Error(`Invalid snapshot file extension for path '${pathStr}'. Path must end with .json or .json.gz`);
+  }
+  if (!isExport && !existsSync(resolved)) {
+    throw new Error(`Snapshot file not found: ${pathStr}`);
+  }
+  return resolved;
+}
+
 export function listAvailableSnapshots() {
   const exportsDir = ensureExportsDir();
   if (!existsSync(exportsDir)) return [];
@@ -79,7 +93,7 @@ export async function exportSnapshot({ customDb = null, customBlobDir = BLOBS_DI
   };
 
   const jsonStr = JSON.stringify(snapshot, null, 2);
-  const targetPath = outputPath || join(ensureExportsDir(), `rag_snapshot_${Date.now()}.json.gz`);
+  const targetPath = outputPath ? validateSnapshotPath(outputPath, true) : join(ensureExportsDir(), `rag_snapshot_${Date.now()}.json.gz`);
 
   if (targetPath.endsWith(".gz")) {
     const gzipped = gzipSync(Buffer.from(jsonStr, "utf-8"));
@@ -96,11 +110,9 @@ export async function importSnapshot({ customDb = null, customBlobDir = BLOBS_DI
   let snapshot;
 
   if (typeof snapshotPathOrData === "string") {
-    if (!existsSync(snapshotPathOrData)) {
-      throw new Error(`Snapshot file not found: ${snapshotPathOrData}`);
-    }
-    const raw = readFileSync(snapshotPathOrData);
-    if (snapshotPathOrData.endsWith(".gz")) {
+    const validPath = validateSnapshotPath(snapshotPathOrData, false);
+    const raw = readFileSync(validPath);
+    if (validPath.endsWith(".gz")) {
       const decompressed = gunzipSync(raw);
       snapshot = JSON.parse(decompressed.toString("utf-8"));
     } else {

package/mcp-server/graph/graph_extractor.js

@@ -85,11 +85,26 @@ export async function saveGraphEdges(db, edges) {
   }
 }
 
-export async function getRelatedSymbols(db, sectionId) {
+export async function getRelatedSymbolsBatch(db, sectionIds) {
+  if (!sectionIds || sectionIds.length === 0) return new Map();
+  const placeholders = sectionIds.map(() => "?").join(",");
   const stmt = db.prepare(`
-    SELECT target_id, relation_type FROM graph_edges
-    WHERE source_id = ? AND relation_type = 'DEFINES_SYMBOL';
+    SELECT source_id, target_id FROM graph_edges
+    WHERE source_id IN (${placeholders}) AND relation_type = 'DEFINES_SYMBOL';
   `);
-  const rows = await stmt.all(sectionId);
-  return rows.map((r) => r.target_id.replace("symbol:", ""));
+  const rows = await stmt.all(...sectionIds);
+  const result = new Map();
+  for (const r of rows) {
+    const list = result.get(r.source_id) || [];
+    list.push(r.target_id.replace("symbol:", ""));
+    result.set(r.source_id, list);
+  }
+  return result;
+}
+
+export async function getRelatedSymbols(db, sectionId) {
+  if (!sectionId) return [];
+  const map = await getRelatedSymbolsBatch(db, [sectionId]);
+  return map.get(sectionId) || [];
 }
+

package/mcp-server/index.js

@@ -32,7 +32,7 @@ await ensureDir();
 
 const server = new McpServer({
   name: "memory-agent",
-  version: "1.5.0",
+  version: "1.5.2",
 });
 
 registerAllTools(server);

package/mcp-server/ingest/normalizer.js

@@ -30,15 +30,51 @@ export function cleanHtml(html) {
   return cleaned;
 }
 
+export function validateUrlForSsrf(urlStr) {
+  if (typeof urlStr !== "string" || !urlStr.trim()) {
+    throw new Error(`Unsupported URL for ingestion: '${urlStr}'. Only http/https URLs are supported.`);
+  }
+  let parsed;
+  try {
+    parsed = new URL(urlStr.trim());
+  } catch {
+    throw new Error(`Invalid URL format for ingestion: '${urlStr}'`);
+  }
+
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    throw new Error(`Unsupported URL scheme '${parsed.protocol}'. Only http/https are allowed.`);
+  }
+
+  const hostname = parsed.hostname.toLowerCase();
+
+  const isBlocked =
+    hostname === "localhost" ||
+    hostname === "127.0.0.1" ||
+    hostname === "::1" ||
+    hostname === "169.254.169.254" ||
+    hostname === "metadata.google.internal" ||
+    /^127\./.test(hostname) ||
+    /^10\./.test(hostname) ||
+    /^192\.168\./.test(hostname) ||
+    /^172\.(1[6-9]|2[0-9]|3[0-1])\./.test(hostname) ||
+    /^169\.254\./.test(hostname) ||
+    /^0\./.test(hostname);
+
+  if (isBlocked) {
+    throw new Error(`Ingestion blocked: URL '${urlStr}' targets a private/local IP address or metadata service.`);
+  }
+
+  return parsed;
+}
+
 // Fetch a web page and convert it to Markdown/text. Used by the 'url' ingestion type
 // so the RAG store gets the page CONTENT, not just the URL string.
 export async function fetchUrlContent(url) {
-  if (typeof url !== "string" || !/^https?:\/\//i.test(url.trim())) {
-    throw new Error(`Unsupported URL for ingestion: '${url}'. Only http/https URLs are supported.`);
-  }
+  const parsed = validateUrlForSsrf(url);
+  const targetUrl = parsed.toString();
   let res;
   try {
-    res = await fetch(url.trim(), {
+    res = await fetch(targetUrl, {
       headers: {
         "User-Agent": "memory-agent-rag/1.0",
         Accept: "text/html,application/xhtml+xml,application/json,text/plain,*/*",

package/mcp-server/ingest/pipeline.js

@@ -85,12 +85,9 @@ export async function ingestDocument({
   try {
     const existingDoc = await db.prepare("SELECT id FROM documents WHERE path = ?").get(docPath);
     if (existingDoc) {
-      const microChunks = await db.prepare("SELECT id FROM micro_chunks WHERE doc_id = ?").all(existingDoc.id);
-      for (const mc of microChunks) {
-        try {
-          await db.prepare("DELETE FROM micro_chunks_fts WHERE id = ?").run(mc.id);
-        } catch {}
-      }
+      try {
+        await db.prepare("DELETE FROM micro_chunks_fts WHERE id IN (SELECT id FROM micro_chunks WHERE doc_id = ?);").run(existingDoc.id);
+      } catch {}
       await db.prepare("DELETE FROM graph_edges WHERE source_id = ? OR target_id = ?").run(existingDoc.id, existingDoc.id);
       await db.prepare("DELETE FROM documents WHERE id = ?").run(existingDoc.id);
     }
@@ -184,8 +181,6 @@ export async function deleteDocument(docIdOrPath, customDb = null, customBlobDir
     return { deleted: false, reason: "Document not found" };
   }
 
-  const microChunks = await db.prepare("SELECT id FROM micro_chunks WHERE doc_id = ?").all(doc.id);
-
   // Collect every id owned by this document so we can purge dangling graph edges
   // (graph_edges has no FK constraints, so section/chunk/doc references would otherwise leak).
   const ownedIds = [doc.id];
@@ -196,11 +191,9 @@ export async function deleteDocument(docIdOrPath, customDb = null, customBlobDir
 
   await db.exec("BEGIN IMMEDIATE;");
   try {
-    for (const mc of microChunks) {
-      try {
-        await db.prepare("DELETE FROM micro_chunks_fts WHERE id = ?").run(mc.id);
-      } catch {}
-    }
+    try {
+      await db.prepare("DELETE FROM micro_chunks_fts WHERE id IN (SELECT id FROM micro_chunks WHERE doc_id = ?);").run(doc.id);
+    } catch {}
 
     // Auto-clean Agent knowledge graph links pointing at this document.
     await db.prepare("DELETE FROM knowledge_links WHERE doc_id = ?").run(doc.id);

package/mcp-server/retrieval/retriever.js

@@ -263,57 +263,74 @@ export async function hybridQuery({
 
   // Parent-Child Rollup: Deduplicate hits sharing the same medium_id or section_id to prevent noise
   const parentDeduplicatedHits = [];
-  const seenParents = new Set();
-  const parentLookupStmt = db.prepare(`
-    SELECT medium_id, section_id FROM micro_chunks WHERE id = ?;
-  `);
-
-  for (const hit of fusedHits) {
-    const row = await parentLookupStmt.get(hit.id);
-    const parentKey = row ? (row.medium_id || row.section_id) : hit.id;
-    if (!seenParents.has(parentKey)) {
-      seenParents.add(parentKey);
-      parentDeduplicatedHits.push(hit);
+  if (fusedHits.length > 0) {
+    const hitIds = fusedHits.map((h) => h.id);
+    const placeholders = hitIds.map(() => "?").join(",");
+    const rows = await db.prepare(`
+      SELECT id, medium_id, section_id FROM micro_chunks WHERE id IN (${placeholders});
+    `).all(...hitIds);
+    const parentMap = new Map(rows.map((r) => [r.id, r]));
+
+    const seenParents = new Set();
+    for (const hit of fusedHits) {
+      const row = parentMap.get(hit.id);
+      const parentKey = row ? (row.medium_id || row.section_id) : hit.id;
+      if (!seenParents.has(parentKey)) {
+        seenParents.add(parentKey);
+        parentDeduplicatedHits.push(hit);
+      }
     }
   }
 
   const topHits = parentDeduplicatedHits.slice(0, limit);
   const results = [];
 
-  const secStmt = db.prepare(`
-    SELECT s.id as section_id, s.heading, s.breadcrumbs, s.content as section_content,
-           med.content as medium_content, d.title as doc_title, d.path as doc_path
-    FROM micro_chunks m
-    JOIN sections s ON m.section_id = s.id
-    JOIN documents d ON m.doc_id = d.id
-    LEFT JOIN medium_chunks med ON m.medium_id = med.id
-    WHERE m.id = ?;
-  `);
-
-  for (const hit of topHits) {
-    const detail = await secStmt.get(hit.id);
-    if (!detail) continue;
-
-    let symbols = [];
+  if (topHits.length > 0) {
+    const topIds = topHits.map((h) => h.id);
+    const placeholders = topIds.map(() => "?").join(",");
+    const details = await db.prepare(`
+      SELECT m.id as micro_id, s.id as section_id, s.heading, s.breadcrumbs, s.content as section_content,
+             med.content as medium_content, d.title as doc_title, d.path as doc_path
+      FROM micro_chunks m
+      JOIN sections s ON m.section_id = s.id
+      JOIN documents d ON m.doc_id = d.id
+      LEFT JOIN medium_chunks med ON m.medium_id = med.id
+      WHERE m.id IN (${placeholders});
+    `).all(...topIds);
+
+    const detailMap = new Map(details.map((d) => [d.micro_id, d]));
+
+    let symbolsBySection = new Map();
     if (includeGraphContext) {
-      symbols = await getRelatedSymbols(db, detail.section_id);
+      const sectionIds = [...new Set(details.map((d) => d.section_id).filter(Boolean))];
+      if (sectionIds.length > 0) {
+        const { getRelatedSymbolsBatch } = await import("../graph/graph_extractor.js");
+        symbolsBySection = await getRelatedSymbolsBatch(db, sectionIds);
+      }
     }
 
-    results.push({
-      chunk_id: hit.id,
-      doc_title: detail.doc_title,
-      doc_path: detail.doc_path,
-      heading: detail.heading,
-      breadcrumbs: detail.breadcrumbs,
-      snippet: hit.content,
-      paragraph_context: detail.medium_content || hit.content,
-      full_section_content: detail.section_content,
-      score: parseFloat((hit.score || 0).toFixed(4)),
-      rsf_score: hit.rsf_score ? parseFloat(hit.rsf_score.toFixed(4)) : null,
-      rrf_score: hit.rrf_score ? parseFloat(hit.rrf_score.toFixed(4)) : null,
-      cosine_sim: hit.cosine_sim ? parseFloat(hit.cosine_sim.toFixed(4)) : null,
-      defined_symbols: symbols,
-    });
+    for (const hit of topHits) {
+      const detail = detailMap.get(hit.id);
+      if (!detail) continue;
+
+      const symbols = symbolsBySection.get(detail.section_id) || [];
+
+      results.push({
+        chunk_id: hit.id,
+        doc_title: detail.doc_title,
+        doc_path: detail.doc_path,
+        heading: detail.heading,
+        breadcrumbs: detail.breadcrumbs,
+        snippet: hit.content,
+        paragraph_context: detail.medium_content || hit.content,
+        full_section_content: detail.section_content,
+        score: parseFloat((hit.score || 0).toFixed(4)),
+        rsf_score: hit.rsf_score ? parseFloat(hit.rsf_score.toFixed(4)) : null,
+        rrf_score: hit.rrf_score ? parseFloat(hit.rrf_score.toFixed(4)) : null,
+        cosine_sim: hit.cosine_sim ? parseFloat(hit.cosine_sim.toFixed(4)) : null,
+        defined_symbols: symbols,
+      });
+    }
   }
 
   return results;

package/mcp-server/tools/identity_tools.js

@@ -33,7 +33,7 @@ export function registerIdentityTools(server) {
         if (!factText || !docId) {
           throw new Error("factText and docId are required parameters for link action");
         }
-        const res = linkFactToDocument({
+        const res = await linkFactToDocument({
           factKey: key,
           factText,
           docId,
@@ -48,14 +48,14 @@ export function registerIdentityTools(server) {
 
       if (action === "get_doc_links") {
         if (!docId) throw new Error("docId parameter is required for get_doc_links action");
-        const links = getLinksForDoc(docId);
+        const links = await getLinksForDoc(docId);
         return {
           content: [{ type: "text", text: JSON.stringify(links, null, 2) }],
         };
       }
 
       if (action === "list_links") {
-        const links = listAllLinks(key);
+        const links = await listAllLinks(key);
         return {
           content: [{ type: "text", text: JSON.stringify(links, null, 2) }],
         };

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@lotargo/memory_plugin",
-  "version": "1.5.0",
+  "version": "1.5.2",
   "description": "100% local hybrid RAG memory for AI coding agents (OpenCode, Claude Code, Codex, Antigravity). MCP server + plugin: persistent user facts, document ingestion, vector + SQLite FTS5 retrieval across sessions.",
   "type": "module",
   "main": "opencode-plugin/index.js",
   "scripts": {
     "preinstall": "node mcp-server/preinstall.js || true",
-    "test": "node mcp-server/test.js",
+    "test": "node tests/run_all.js",
     "benchmark": "node mcp-server/benchmarks/run_benchmarks.js"
   },
   "bin": {