@lotargo/memory_plugin 1.6.1 → 1.6.2

package/CHANGELOG.md

@@ -5,6 +5,20 @@ All notable changes to `@lotargo/memory_plugin` are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.6.2] - 2026-08-12
+
+### Added
+
+- **Batch retrieval API** (`batch_query_knowledge_base`): execute multiple search queries in a single MCP call. All query embeddings computed in one ONNX pass, queries run in parallel via `Promise.all`. Ideal for cross-document comparisons and multi-part analysis — significantly reduces API overhead vs N separate `query_knowledge_base` calls.
+- **Policy expansion toggle** (`config.policyExpansion`, default: `true`): table summaries and code signatures are automatically expanded to full content for better recall (~+5-10% recall, slight MRR trade-off). Disable per-call or via config for pure micro_chunk precision.
+- **RAG evaluation test** (`tests/unit/rag_evaluation.test.js`): 10 analytical queries with expected-fact verification (100% pass rate on financial reports). Includes raw-question vs optimized-query comparison demonstrating +33% fact retrieval improvement.
+
+### Changed
+
+- `hybridQuery()` accepts `_precomputedVector` internal parameter for batch embedding reuse.
+- `PROMPT_BLOCK` (injected into AGENTS.md/CLAUDE.md) updated with query optimization and batch usage directives.
+- SKILL.md updated with batch query tool and query formulation guidance.
+
 ## [1.6.1] - 2026-08-10
 
 ### Fixed

package/README.md

@@ -176,6 +176,7 @@ The MCP server registers **14 MCP tools** accessible across all connected AI env
 | :--- | :------------- | :---------- |
 | `ingest_document` | `content`, `type`, `title`, `path`, `generateEmbeddings` | Ingest local files, URLs, or raw text into the 3-tier index (Big/Medium/Small) with ONNX vector embeddings and GraphRAG symbol extraction. |
 | `query_knowledge_base` | `query`, `limit`, `instruction`, `generateEmbeddings` | Perform hybrid search (RSF/RRF BM25 + dense vector similarity) to retrieve candidate document sections with defined code symbols. |
+| `batch_query_knowledge_base` | `queries` (array), `limit`, `instruction`, `generateEmbeddings` | Execute multiple queries in a single batch call. More efficient than separate `query_knowledge_base` calls — all embeddings computed in one ONNX pass, queries run in parallel. Ideal for comparisons and multi-topic analysis. |
 | `manage_knowledge_base` | `action`, `docId`, `snapshotPath` | Inspect DB stats (`stats`), list documents (`list`), read full raw document (`read_document`), delete document (`delete`), or export/import snapshots (`export_snapshot` / `import_snapshot`). |
 | `reindex_knowledge_base` | `model`, `dimension` | Re-embed all stored vectors with the active (or specified) embedding model and vector dimension. Use after switching the embedding model or vector dimension so previously indexed documents remain retrievable. Preserves documents, FTS index, graph edges, and fact links. |
 | `link_knowledge` | `action`, `factText`, `docId`, `scope`, `startLine`, `endLine`, `relationType` | Create, list, or retrieve semantic graph links connecting Notebook facts to Knowledge Base documents, sections, or line ranges. Actions: `link`, `list_links`, `get_doc_links`. |
@@ -319,6 +320,7 @@ The engine is configured through `<memory-dir>/config.json` (created with defaul
 | `onnxThreads` | `0` | ONNX WASM threads: `0` auto-detect, or `1-16` |
 | `executionDevice` | `cpu` | `cpu` or `webgpu` (experimental) |
 | `vectorScanLimit` | `50000` | Max micro-chunks scanned per vector query (`0` = unlimited) |
+| `policyExpansion` | `true` | Expand table_summary/code_signature policy chunks for better recall (slight MRR trade-off). Disable for pure micro_chunk precision. |
 | `injectLimit` | `10` | Max facts injected into the agent's system prompt |
 | `conflictStrategy` | `merge` | Hybrid-sync conflict resolution: `merge`, `cloud-wins`, or `local-wins` |
 | `tursoUrl` | `""` | Primary Turso endpoint URL (set by `login`) |

package/mcp-server/config/config_manager.js

@@ -25,6 +25,7 @@ export const DEFAULT_CONFIG = {
   username: "",           // Account username from the Turso OAuth profile
   ingestAllowedPaths: [], // Extra directories ingest_document(type:"file") may read from
   ingestAllowAnyPath: false, // Escape hatch: allow reading ANY path from disk (unsafe)
+  policyExpansion: true,    // Expand table_summary/code_signature policy chunks (boosts recall, slight MRR trade-off)
 };
 
 let cachedConfig = null;

package/mcp-server/db/migrations.js

@@ -138,6 +138,21 @@ const MIGRATIONS = [
       `);
     },
   },
+  {
+    version: 5,
+    name: "005_retrieval_policy",
+    up: async (db) => {
+      try {
+        await db.exec(`ALTER TABLE micro_chunks ADD COLUMN retrieval_policy TEXT DEFAULT 'micro_chunk';`);
+      } catch (e) {}
+      try {
+        await db.exec(`ALTER TABLE micro_chunks ADD COLUMN policy_source_id TEXT;`);
+      } catch (e) {}
+      await db.exec(`
+        CREATE INDEX IF NOT EXISTS idx_micro_chunks_retrieval_policy ON micro_chunks(retrieval_policy);
+      `);
+    },
+  },
 ];
 
 export async function runMigrations(db) {

package/mcp-server/ingest/chunker.js

@@ -5,6 +5,159 @@ export function estimateTokens(text) {
   return Math.ceil(text.length / 4);
 }
 
+const CODE_SIGNATURE_REGEX = /^\s*(?:export\s+|async\s+)?(?:function|class|def|pub\s+fn|fn|struct|interface|enum)\s+/;
+
+function _classifyLine(line) {
+  const t = line.trimStart();
+  if (t.startsWith("/**")) return "jsdoc_start";
+  if (t.startsWith("*/")) return "jsdoc_end";
+  if (t.startsWith("*")) return "jsdoc_mid";
+  if (t.startsWith("//")) return "line_comment";
+  if (t.startsWith("#")) return "hash_comment";
+  if (t.startsWith("'''") || t.startsWith('"""')) return "py_docstring";
+  return "code";
+}
+
+export function extractCodeSignatures(codeContent) {
+  const lines = codeContent.split("\n");
+  const signatures = [];
+
+  const fenceMatch = lines[0] && lines[0].match(/^(\s*)(```|~~~)/);
+  const bodyStart = fenceMatch ? 1 : 0
+  const bodyEnd = fenceMatch && (lines[lines.length - 1].startsWith("```") || lines[lines.length - 1].startsWith("~~~")) ? lines.length - 1 : lines.length;
+  const bodyLines = lines.slice(bodyStart, bodyEnd);
+
+  let i = 0;
+  while (i < bodyLines.length) {
+    const line = bodyLines[i];
+    const type = _classifyLine(line);
+
+    if (type === "jsdoc_start") {
+      const jsdocBlock = [line];
+      let j = i + 1;
+      while (j < bodyLines.length) {
+        jsdocBlock.push(bodyLines[j]);
+        if (_classifyLine(bodyLines[j]) === "jsdoc_end") {
+          j++;
+          break;
+        }
+        j++;
+      }
+
+      if (j < bodyLines.length && CODE_SIGNATURE_REGEX.test(bodyLines[j])) {
+        const sigLines = [bodyLines[j]];
+        const pyDocResult = _tryPyDocstring(bodyLines, j + 1);
+        if (pyDocResult.docLines.length > 0) {
+          sigLines.push(...pyDocResult.docLines);
+        }
+        const endIdx = pyDocResult.docLines.length > 0 ? pyDocResult.endIdx : j;
+
+        signatures.push({
+          signature: [...jsdocBlock, ...sigLines].join("\n").trim(),
+          line_number: j + bodyStart + 1,
+        });
+        i = endIdx + 1;
+        continue;
+      }
+
+      i = j;
+      continue;
+    }
+
+    if (type === "line_comment" || type === "hash_comment") {
+      const commentBlock = [line];
+      let j = i + 1;
+      while (j < bodyLines.length && _classifyLine(bodyLines[j]) === type) {
+        commentBlock.push(bodyLines[j]);
+        j++;
+      }
+
+      if (j < bodyLines.length && CODE_SIGNATURE_REGEX.test(bodyLines[j])) {
+        const sigLines = [bodyLines[j]];
+        const pyDocResult = _tryPyDocstring(bodyLines, j + 1);
+        if (pyDocResult.docLines.length > 0) {
+          sigLines.push(...pyDocResult.docLines);
+        }
+        const endIdx = pyDocResult.docLines.length > 0 ? pyDocResult.endIdx : j;
+
+        signatures.push({
+          signature: [...commentBlock, ...sigLines].join("\n").trim(),
+          line_number: j + bodyStart + 1,
+        });
+        i = endIdx + 1;
+        continue;
+      }
+
+      i = j;
+      continue;
+    }
+
+    if (CODE_SIGNATURE_REGEX.test(line)) {
+      const sigLines = [line];
+      const pyDocResult = _tryPyDocstring(bodyLines, i + 1);
+      if (pyDocResult.docLines.length > 0) {
+        sigLines.push(...pyDocResult.docLines);
+      }
+      const endIdx = pyDocResult.docLines.length > 0 ? pyDocResult.endIdx : i;
+
+      signatures.push({
+        signature: sigLines.join("\n").trim(),
+        line_number: i + bodyStart + 1,
+      });
+      i = endIdx + 1;
+      continue;
+    }
+
+    i++;
+  }
+
+  return signatures;
+}
+
+function _tryPyDocstring(bodyLines, startIdx) {
+  let k = startIdx;
+  while (k < bodyLines.length && bodyLines[k].trim() === "") k++;
+  if (k >= bodyLines.length) return { docLines: [], endIdx: startIdx - 1 };
+
+  const line = bodyLines[k];
+  const tripleDouble = /^\s*"""/.test(line);
+  const tripleSingle = /^\s*'''/.test(line);
+  const marker = tripleDouble ? '"""' : tripleSingle ? "'''" : null;
+  if (!marker) return { docLines: [], endIdx: startIdx - 1 };
+
+  const docLines = [line];
+  if (line.includes(marker.repeat(2)) && line.indexOf(marker) !== line.lastIndexOf(marker)) {
+    return { docLines, endIdx: k };
+  }
+
+  for (let m = k + 1; m < bodyLines.length; m++) {
+    docLines.push(bodyLines[m]);
+    if (bodyLines[m].includes(marker)) {
+      return { docLines, endIdx: m };
+    }
+  }
+  return { docLines, endIdx: k };
+}
+
+export function generateTableSummary(tableContent, breadcrumbs = "") {
+  const lines = tableContent.split("\n").filter((l) => l.trim().length > 0);
+  if (lines.length === 0) return null;
+
+  const headerLine = lines[0];
+  const columns = headerLine
+    .split("|")
+    .map((c) => c.trim())
+    .filter((c) => c.length > 0);
+
+  const separatorLine = lines[1] || "";
+  const hasSeparator = /^\s*\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)+\|?\s*$/.test(separatorLine);
+  const dataLines = hasSeparator ? lines.slice(2) : lines.slice(1);
+  const rowCount = dataLines.length;
+
+  const contextPart = breadcrumbs ? ` Context: ${breadcrumbs}.` : "";
+  return `Table with columns [${columns.join(", ")}] containing ${rowCount} row${rowCount !== 1 ? "s" : ""}.${contextPart}`;
+}
+
 // 1. BIG LEVEL: Heading & Section Hierarchy Parser
 export function parseSections(markdown, docTitle = "Document") {
   const lines = markdown.split("\n");
@@ -72,7 +225,7 @@ export function extractMediumBlocks(section, sectionId, docId) {
   let blockIndex = 0;
 
   let currentLines = [];
-  let currentBlockType = "paragraph"; // 'paragraph', 'code', 'table', 'list', 'blockquote'
+  let currentBlockType = "paragraph";
 
   function pushCurrentBlock() {
     const blockContent = currentLines.join("\n").trim();
@@ -97,7 +250,6 @@ export function extractMediumBlocks(section, sectionId, docId) {
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i];
 
-    // Check code fence
     const fenceMatch = line.match(/^(\s*)(```|~~~)/);
     if (fenceMatch) {
       if (!inFencedCode) {
@@ -121,7 +273,6 @@ export function extractMediumBlocks(section, sectionId, docId) {
       continue;
     }
 
-    // Check table line
     const isTableLine = /^\s*\|.*\|\s*$/.test(line);
     if (isTableLine) {
       if (currentBlockType !== "table" && currentLines.length > 0) {
@@ -134,7 +285,6 @@ export function extractMediumBlocks(section, sectionId, docId) {
       pushCurrentBlock();
     }
 
-    // Check list item line
     const isListLine = /^\s*([*+-]|\d+\.)\s+/.test(line);
     if (isListLine) {
       if (currentBlockType !== "list" && currentBlockType !== "paragraph" && currentLines.length > 0) {
@@ -145,7 +295,6 @@ export function extractMediumBlocks(section, sectionId, docId) {
       continue;
     }
 
-    // Check empty line
     if (line.trim().length === 0) {
       if (currentLines.length > 0) {
         pushCurrentBlock();
@@ -169,6 +318,7 @@ export function createSmallChunks(mediumBlock, sectionId, docId) {
 
   function makeChunk(chunkText, extraMeta = {}) {
     if (!chunkText || chunkText.trim().length === 0) return;
+    const { retrieval_policy, policy_source_id, ...rest } = extraMeta;
     smallChunks.push({
       id: `${mediumBlock.id}_s${smallIdx++}`,
       medium_id: mediumBlock.id,
@@ -177,14 +327,24 @@ export function createSmallChunks(mediumBlock, sectionId, docId) {
       content: chunkText.trim(),
       breadcrumbs: mediumBlock.breadcrumbs,
       token_count: estimateTokens(chunkText),
-      ...extraMeta,
+      retrieval_policy: retrieval_policy || "micro_chunk",
+      policy_source_id: policy_source_id || null,
+      ...rest,
     });
   }
 
   // RULE FOR TABLES
   if (mediumBlock.block_type === "table") {
+    const summary = generateTableSummary(content, mediumBlock.breadcrumbs);
+    if (summary) {
+      makeChunk(summary, {
+        retrieval_policy: "table_summary",
+        policy_source_id: mediumBlock.id,
+      });
+    }
+
     if (tokenCount <= 350) {
-      makeChunk(content);
+      makeChunk(content, { retrieval_policy: "micro_chunk" });
       return smallChunks;
     }
 
@@ -205,15 +365,23 @@ export function createSmallChunks(mediumBlock, sectionId, docId) {
     for (let i = 0; i < dataLines.length; i += chunkSize) {
       const rowBatch = dataLines.slice(i, i + chunkSize);
       const tableChunkText = `${headerStr}\n${rowBatch.join("\n")}`;
-      makeChunk(tableChunkText);
+      makeChunk(tableChunkText, { retrieval_policy: "micro_chunk" });
     }
     return smallChunks;
   }
 
   // RULE FOR CODE BLOCKS
   if (mediumBlock.block_type === "code") {
+    const signatures = extractCodeSignatures(content);
+    for (const sig of signatures) {
+      makeChunk(sig.signature, {
+        retrieval_policy: "code_signature",
+        policy_source_id: mediumBlock.id,
+      });
+    }
+
     if (tokenCount <= 350) {
-      makeChunk(content);
+      makeChunk(content, { retrieval_policy: "micro_chunk" });
       return smallChunks;
     }
 
@@ -243,7 +411,7 @@ export function createSmallChunks(mediumBlock, sectionId, docId) {
 
     for (const block of astBlocks) {
       const fullChunk = fenceHeader ? `${fenceHeader}\n${block}\n${fenceFooter}` : block;
-      makeChunk(fullChunk);
+      makeChunk(fullChunk, { retrieval_policy: "micro_chunk" });
     }
     return smallChunks;
   }
@@ -265,8 +433,7 @@ export function createSmallChunks(mediumBlock, sectionId, docId) {
 
     if (currentTokens + sTokens > TARGET_WINDOW_TOKENS && currentWindow.length > 0) {
       makeChunk(currentWindow.join(" "));
-      
-      // Safe Overlap: Keep the last sentence of the previous window if available
+
       const lastSentence = currentWindow[currentWindow.length - 1];
       currentWindow = [lastSentence, sentence];
       currentTokens = estimateTokens(lastSentence) + sTokens;

package/mcp-server/ingest/pipeline.js

@@ -129,8 +129,8 @@ export async function ingestDocument({
     }
 
     const insertMicroStmt = db.prepare(`
-      INSERT INTO micro_chunks (id, section_id, doc_id, content, vector, token_count, medium_id)
-      VALUES (?, ?, ?, ?, ?, ?, ?);
+      INSERT INTO micro_chunks (id, section_id, doc_id, content, vector, token_count, medium_id, retrieval_policy, policy_source_id)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);
     `);
     const insertFtsStmt = db.prepare(`
       INSERT INTO micro_chunks_fts (id, content, breadcrumbs)
@@ -138,7 +138,7 @@ export async function ingestDocument({
     `);
 
     for (const micro of hierarchy.microChunks) {
-      await insertMicroStmt.run(micro.id, micro.section_id, micro.doc_id, micro.content, micro.vector, micro.token_count, micro.medium_id || null);
+      await insertMicroStmt.run(micro.id, micro.section_id, micro.doc_id, micro.content, micro.vector, micro.token_count, micro.medium_id || null, micro.retrieval_policy || "micro_chunk", micro.policy_source_id || null);
       await insertFtsStmt.run(micro.id, micro.content, micro.breadcrumbs);
     }