@lotargo/memory_plugin 1.1.4 → 1.1.6

package/mcp-server/ingest/exporter.js

@@ -1,80 +1,80 @@
-import { getDatabase } from "../db/database.js";
-import { writeFileSync, mkdirSync, existsSync } from "node:fs";
-import { join } from "node:path";
-import { MEMORY_DIR } from "../memory.js";
-
-export const EXPORTS_DIR = join(MEMORY_DIR, "exports");
-
-export function ensureExportsDir() {
-  if (!existsSync(EXPORTS_DIR)) {
-    mkdirSync(EXPORTS_DIR, { recursive: true });
-  }
-  return EXPORTS_DIR;
-}
-
-export function exportDocumentData(docIdOrPath, customDb = null) {
-  const db = customDb || getDatabase();
-  const doc = db.prepare("SELECT * FROM documents WHERE id = ? OR path = ?").get(docIdOrPath, docIdOrPath);
-  if (!doc) {
-    throw new Error(`Document not found for ID or path: ${docIdOrPath}`);
-  }
-
-  let toc = null;
-  try {
-    toc = doc.toc_json ? JSON.parse(doc.toc_json) : null;
-  } catch {
-    toc = doc.toc_json;
-  }
-
-  let metadata = null;
-  try {
-    metadata = doc.metadata_json ? JSON.parse(doc.metadata_json) : null;
-  } catch {
-    metadata = doc.metadata_json;
-  }
-
-  const sections = db.prepare("SELECT id, heading, breadcrumbs, content, token_count FROM sections WHERE doc_id = ?").all(doc.id);
-  const mediumChunks = db.prepare("SELECT id, section_id, content, block_type, token_count, created_at FROM medium_chunks WHERE doc_id = ?").all(doc.id);
-  const microChunks = db.prepare("SELECT id, medium_id, section_id, content, token_count FROM micro_chunks WHERE doc_id = ?").all(doc.id);
-  const graphEdges = db.prepare("SELECT source_id, target_id, relation_type, metadata_json FROM graph_edges WHERE source_id = ? OR target_id = ?").all(doc.id, doc.id);
-
-  return {
-    document: {
-      id: doc.id,
-      path: doc.path,
-      title: doc.title,
-      blob_hash: doc.blob_hash,
-      checksum: doc.checksum,
-      toc,
-      metadata,
-      created_at: doc.created_at,
-      updated_at: doc.updated_at,
-    },
-    counts: {
-      sections: sections.length,
-      medium_chunks: mediumChunks.length,
-      micro_chunks: microChunks.length,
-      graph_edges: graphEdges.length,
-    },
-    sections,
-    medium_chunks: mediumChunks,
-    micro_chunks: microChunks,
-    graph_edges: graphEdges,
-  };
-}
-
-export function exportDocumentToJsonString(docIdOrPath, customDb = null) {
-  const data = exportDocumentData(docIdOrPath, customDb);
-  return JSON.stringify(data, null, 2);
-}
-
-export function exportDocumentToFile(docIdOrPath, outputPath = null, customDb = null) {
-  const targetDir = ensureExportsDir();
-  const db = customDb || getDatabase();
-  const data = exportDocumentData(docIdOrPath, db);
-  const jsonStr = JSON.stringify(data, null, 2);
-
-  const finalPath = outputPath || join(targetDir, `doc_export_${data.document.id}.json`);
-  writeFileSync(finalPath, jsonStr, "utf-8");
-  return finalPath;
-}
+import { getDatabase } from "../db/database.js";
+import { writeFileSync, mkdirSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { MEMORY_DIR } from "../memory.js";
+
+export const EXPORTS_DIR = join(MEMORY_DIR, "exports");
+
+export function ensureExportsDir() {
+  if (!existsSync(EXPORTS_DIR)) {
+    mkdirSync(EXPORTS_DIR, { recursive: true });
+  }
+  return EXPORTS_DIR;
+}
+
+export function exportDocumentData(docIdOrPath, customDb = null) {
+  const db = customDb || getDatabase();
+  const doc = db.prepare("SELECT * FROM documents WHERE id = ? OR path = ?").get(docIdOrPath, docIdOrPath);
+  if (!doc) {
+    throw new Error(`Document not found for ID or path: ${docIdOrPath}`);
+  }
+
+  let toc = null;
+  try {
+    toc = doc.toc_json ? JSON.parse(doc.toc_json) : null;
+  } catch {
+    toc = doc.toc_json;
+  }
+
+  let metadata = null;
+  try {
+    metadata = doc.metadata_json ? JSON.parse(doc.metadata_json) : null;
+  } catch {
+    metadata = doc.metadata_json;
+  }
+
+  const sections = db.prepare("SELECT id, heading, breadcrumbs, content, token_count FROM sections WHERE doc_id = ?").all(doc.id);
+  const mediumChunks = db.prepare("SELECT id, section_id, content, block_type, token_count, created_at FROM medium_chunks WHERE doc_id = ?").all(doc.id);
+  const microChunks = db.prepare("SELECT id, medium_id, section_id, content, token_count FROM micro_chunks WHERE doc_id = ?").all(doc.id);
+  const graphEdges = db.prepare("SELECT source_id, target_id, relation_type, metadata_json FROM graph_edges WHERE source_id = ? OR target_id = ?").all(doc.id, doc.id);
+
+  return {
+    document: {
+      id: doc.id,
+      path: doc.path,
+      title: doc.title,
+      blob_hash: doc.blob_hash,
+      checksum: doc.checksum,
+      toc,
+      metadata,
+      created_at: doc.created_at,
+      updated_at: doc.updated_at,
+    },
+    counts: {
+      sections: sections.length,
+      medium_chunks: mediumChunks.length,
+      micro_chunks: microChunks.length,
+      graph_edges: graphEdges.length,
+    },
+    sections,
+    medium_chunks: mediumChunks,
+    micro_chunks: microChunks,
+    graph_edges: graphEdges,
+  };
+}
+
+export function exportDocumentToJsonString(docIdOrPath, customDb = null) {
+  const data = exportDocumentData(docIdOrPath, customDb);
+  return JSON.stringify(data, null, 2);
+}
+
+export function exportDocumentToFile(docIdOrPath, outputPath = null, customDb = null) {
+  const targetDir = ensureExportsDir();
+  const db = customDb || getDatabase();
+  const data = exportDocumentData(docIdOrPath, db);
+  const jsonStr = JSON.stringify(data, null, 2);
+
+  const finalPath = outputPath || join(targetDir, `doc_export_${data.document.id}.json`);
+  writeFileSync(finalPath, jsonStr, "utf-8");
+  return finalPath;
+}

package/mcp-server/ingest/normalizer.js

@@ -1,104 +1,104 @@
-import { basename, extname } from "node:path";
-
-export function cleanHtml(html) {
-  if (!html) return "";
-
-  let cleaned = html.replace(/<(script|style|nav|header|footer|svg|noscript)[^>]*>[\s\S]*?<\/\1>/gi, "");
-
-  cleaned = cleaned.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, "\n# $1\n");
-  cleaned = cleaned.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, "\n## $1\n");
-  cleaned = cleaned.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, "\n### $1\n");
-  cleaned = cleaned.replace(/<h[4-6][^>]*>([\s\S]*?)<\/h[4-6]>/gi, "\n#### $1\n");
-
-  cleaned = cleaned.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, "\n- $1");
-  cleaned = cleaned.replace(/<(p|div|br)[^>]*>/gi, "\n");
-
-  cleaned = cleaned.replace(/<[^>]+>/g, "");
-
-  cleaned = cleaned
-    .replace(/&nbsp;/g, " ")
-    .replace(/&amp;/g, "&")
-    .replace(/&lt;/g, "<")
-    .replace(/&gt;/g, ">")
-    .replace(/&quot;/g, '"')
-    .replace(/&#39;/g, "'");
-
-  cleaned = cleaned.replace(/\n\s*\n\s*\n/g, "\n\n").trim();
-  return cleaned;
-}
-
-export function extractTitle(markdown, fallbackName = "Untitled Document") {
-  const h1Match = markdown.match(/^#\s+(.+)$/m);
-  if (h1Match && h1Match[1].trim()) {
-    return h1Match[1].trim();
-  }
-  return fallbackName;
-}
-
-export function stripMarkdownBadgesAndNoise(text) {
-  if (!text) return "";
-  let cleaned = text;
-
-  // 1. Remove markdown link-wrapped badges: [![alt](image_url)](link_url)
-  cleaned = cleaned.replace(/\[\s*!\[[^\]]*\]\([^)]+\)\s*\]\([^)]+\)/g, "");
-
-  // 2. Remove standalone markdown image badges: ![alt](https://img.shields.io/...) or badge URLs
-  cleaned = cleaned.replace(/!\[[^\]]*\]\([^)]*(?:shields\.io|badge|actions\/workflows|codecov|travis-ci)[^)]*\)/gi, "");
-
-  // 3. Remove raw HTML img badge tags
-  cleaned = cleaned.replace(/<img[^>]*(?:shields\.io|badge|workflows|badge\.svg)[^>]*>/gi, "");
-
-  // 4. Remove empty HTML anchor containers often surrounding badges
-  cleaned = cleaned.replace(/<a[^>]*>\s*<\/a>/gi, "");
-
-  // 5. Normalize excessive blank lines
-  cleaned = cleaned.replace(/\n\s*\n\s*\n+/g, "\n\n").trim();
-  return cleaned;
-}
-
-export function normalizeContent({ content, type = "text", path = null, title = null }) {
-  let markdown = "";
-  let docTitle = title;
-  const fileName = path ? basename(path) : "document";
-
-  if (type === "url" || (typeof content === "string" && /<html|<body|<div/i.test(content))) {
-    markdown = cleanHtml(content);
-    docTitle = title || extractTitle(markdown, fileName);
-  } else if (type === "file" && path) {
-    const ext = extname(path).toLowerCase();
-    const codeLangs = {
-      ".js": "javascript",
-      ".ts": "typescript",
-      ".py": "python",
-      ".json": "json",
-      ".yaml": "yaml",
-      ".yml": "yaml",
-      ".css": "css",
-      ".html": "html",
-    };
-
-    if (codeLangs[ext]) {
-      markdown = `# ${fileName}\n\n\`\`\`${codeLangs[ext]}\n${content.trim()}\n\`\`\``;
-      docTitle = title || fileName;
-    } else {
-      markdown = content.trim();
-      docTitle = title || extractTitle(markdown, fileName);
-    }
-  } else {
-    markdown = typeof content === "string" ? content.trim() : String(content);
-    docTitle = title || extractTitle(markdown, "Direct Note");
-  }
-
-  // Apply noise and badge stripping to all developer documentation
-  markdown = stripMarkdownBadgesAndNoise(markdown);
-
-  return {
-    markdown,
-    title: docTitle,
-    metadata: {
-      source_type: type,
-      original_path: path || null,
-      char_count: markdown.length,
-    },
-  };
-}
+import { basename, extname } from "node:path";
+
+export function cleanHtml(html) {
+  if (!html) return "";
+
+  let cleaned = html.replace(/<(script|style|nav|header|footer|svg|noscript)[^>]*>[\s\S]*?<\/\1>/gi, "");
+
+  cleaned = cleaned.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, "\n# $1\n");
+  cleaned = cleaned.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, "\n## $1\n");
+  cleaned = cleaned.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, "\n### $1\n");
+  cleaned = cleaned.replace(/<h[4-6][^>]*>([\s\S]*?)<\/h[4-6]>/gi, "\n#### $1\n");
+
+  cleaned = cleaned.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, "\n- $1");
+  cleaned = cleaned.replace(/<(p|div|br)[^>]*>/gi, "\n");
+
+  cleaned = cleaned.replace(/<[^>]+>/g, "");
+
+  cleaned = cleaned
+    .replace(/&nbsp;/g, " ")
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'");
+
+  cleaned = cleaned.replace(/\n\s*\n\s*\n/g, "\n\n").trim();
+  return cleaned;
+}
+
+export function extractTitle(markdown, fallbackName = "Untitled Document") {
+  const h1Match = markdown.match(/^#\s+(.+)$/m);
+  if (h1Match && h1Match[1].trim()) {
+    return h1Match[1].trim();
+  }
+  return fallbackName;
+}
+
+export function stripMarkdownBadgesAndNoise(text) {
+  if (!text) return "";
+  let cleaned = text;
+
+  // 1. Remove markdown link-wrapped badges: [![alt](image_url)](link_url)
+  cleaned = cleaned.replace(/\[\s*!\[[^\]]*\]\([^)]+\)\s*\]\([^)]+\)/g, "");
+
+  // 2. Remove standalone markdown image badges: ![alt](https://img.shields.io/...) or badge URLs
+  cleaned = cleaned.replace(/!\[[^\]]*\]\([^)]*(?:shields\.io|badge|actions\/workflows|codecov|travis-ci)[^)]*\)/gi, "");
+
+  // 3. Remove raw HTML img badge tags
+  cleaned = cleaned.replace(/<img[^>]*(?:shields\.io|badge|workflows|badge\.svg)[^>]*>/gi, "");
+
+  // 4. Remove empty HTML anchor containers often surrounding badges
+  cleaned = cleaned.replace(/<a[^>]*>\s*<\/a>/gi, "");
+
+  // 5. Normalize excessive blank lines
+  cleaned = cleaned.replace(/\n\s*\n\s*\n+/g, "\n\n").trim();
+  return cleaned;
+}
+
+export function normalizeContent({ content, type = "text", path = null, title = null }) {
+  let markdown = "";
+  let docTitle = title;
+  const fileName = path ? basename(path) : "document";
+
+  if (type === "url" || (typeof content === "string" && /<html|<body|<div/i.test(content))) {
+    markdown = cleanHtml(content);
+    docTitle = title || extractTitle(markdown, fileName);
+  } else if (type === "file" && path) {
+    const ext = extname(path).toLowerCase();
+    const codeLangs = {
+      ".js": "javascript",
+      ".ts": "typescript",
+      ".py": "python",
+      ".json": "json",
+      ".yaml": "yaml",
+      ".yml": "yaml",
+      ".css": "css",
+      ".html": "html",
+    };
+
+    if (codeLangs[ext]) {
+      markdown = `# ${fileName}\n\n\`\`\`${codeLangs[ext]}\n${content.trim()}\n\`\`\``;
+      docTitle = title || fileName;
+    } else {
+      markdown = content.trim();
+      docTitle = title || extractTitle(markdown, fileName);
+    }
+  } else {
+    markdown = typeof content === "string" ? content.trim() : String(content);
+    docTitle = title || extractTitle(markdown, "Direct Note");
+  }
+
+  // Apply noise and badge stripping to all developer documentation
+  markdown = stripMarkdownBadgesAndNoise(markdown);
+
+  return {
+    markdown,
+    title: docTitle,
+    metadata: {
+      source_type: type,
+      original_path: path || null,
+      char_count: markdown.length,
+    },
+  };
+}

package/mcp-server/ingest/pipeline.js

@@ -3,8 +3,9 @@ import { getDatabase, BLOBS_DIR } from "../db/database.js";
 import { saveBlob, deleteBlob } from "../storage/blob_store.js";
 import { normalizeContent } from "./normalizer.js";
 import { buildTripleHierarchy } from "./chunker.js";
-import { embedText, vectorToBuffer } from "../ml/model_manager.js";
+import { embedText, embedBatch, vectorToBuffer } from "../ml/model_manager.js";
 import { buildGraphEdges, saveGraphEdges } from "../graph/graph_extractor.js";
+import { getConfig } from "../config/config_manager.js";
 
 export async function ingestDocument({
   content,
@@ -28,13 +29,27 @@ export async function ingestDocument({
 
   const hierarchy = buildTripleHierarchy(markdown, docId, docTitle);
 
-  if (generateEmbeddings) {
-    for (const micro of hierarchy.microChunks) {
-      const contextualText = micro.breadcrumbs
+  if (generateEmbeddings && hierarchy.microChunks.length > 0) {
+    const BATCH_SIZE = getConfig().batchSize || 12;
+
+    // Smart Batching: Sort micro-chunks by character/token length to minimize ONNX zero-padding overhead
+    const indexedItems = hierarchy.microChunks.map((micro, idx) => ({
+      index: idx,
+      text: micro.breadcrumbs
         ? `${micro.content}\n\nContext: ${docTitle} > ${micro.breadcrumbs}`
-        : `${micro.content}\n\nContext: ${docTitle}`;
-      const vec = await embedText(contextualText, false);
-      micro.vector = vectorToBuffer(vec);
+        : `${micro.content}\n\nContext: ${docTitle}`,
+    }));
+
+    indexedItems.sort((a, b) => a.text.length - b.text.length);
+
+    for (let i = 0; i < indexedItems.length; i += BATCH_SIZE) {
+      const batch = indexedItems.slice(i, i + BATCH_SIZE);
+      const batchTexts = batch.map((item) => item.text);
+      const batchVecs = await embedBatch(batchTexts, false);
+      for (let j = 0; j < batchVecs.length; j++) {
+        const origIdx = batch[j].index;
+        hierarchy.microChunks[origIdx].vector = vectorToBuffer(batchVecs[j]);
+      }
     }
   } else {
     for (const micro of hierarchy.microChunks) {

package/mcp-server/ingest/sentence_segmenter.js

@@ -1,74 +1,74 @@
-// Multilingual Sentence Segmenter using Intl.Segmenter with Abbreviation & Boundary Protection
-
-const RU_ABBREVIATIONS = new Set([
-  "т.д", "т.п", "т.е", "и др", "г", "гг", "ул", "руб", "коп", "стр", "рис", "см", "им", "пер", "д", "к", "п", "в"
-]);
-
-const EN_ABBREVIATIONS = new Set([
-  "e.g", "i.e", "etc", "vs", "dr", "mr", "mrs", "ms", "prof", "inc", "ltd", "co", "vol", "no", "p", "pp", "fig"
-]);
-
-function isAbbreviation(word) {
-  if (!word) return false;
-  const clean = word.trim().toLowerCase().replace(/\.$/, "");
-  return RU_ABBREVIATIONS.has(clean) || EN_ABBREVIATIONS.has(clean);
-}
-
-export function splitSentencesMultilingual(text, langHint = "ru") {
-  if (!text || typeof text !== "string") return [];
-  const trimmed = text.trim();
-  if (trimmed.length === 0) return [];
-
-  let rawSegments = [];
-
-  if (typeof Intl !== "undefined" && Intl.Segmenter) {
-    try {
-      const segmenter = new Intl.Segmenter([langHint, "ru", "en"], { granularity: "sentence" });
-      const iter = segmenter.segment(trimmed);
-      for (const seg of iter) {
-        if (seg.segment.trim().length > 0) {
-          rawSegments.push(seg.segment);
-        }
-      }
-    } catch {
-      rawSegments = fallbackSentenceSplit(trimmed);
-    }
-  } else {
-    rawSegments = fallbackSentenceSplit(trimmed);
-  }
-
-  // Post-process & merge false splits caused by abbreviations or numbers (e.g., "v1.0", "e.g.", "т. д.")
-  const sentences = [];
-  let current = "";
-
-  for (let i = 0; i < rawSegments.length; i++) {
-    const seg = rawSegments[i];
-    if (!current) {
-      current = seg;
-    } else {
-      const lastWordMatch = current.trim().match(/([a-zA-Zа-яА-Я0-9._-]+)\s*\.?$/);
-      const lastWord = lastWordMatch ? lastWordMatch[1] : "";
-      
-      const isNumDot = /\b\d+\.$/.test(current.trim());
-      const isAbbr = isAbbreviation(lastWord) || isNumDot;
-      const startsWithLowercase = /^[a-zа-я]/.test(seg.trim());
-
-      if (isAbbr || startsWithLowercase) {
-        current += seg;
-      } else {
-        sentences.push(current.trim());
-        current = seg;
-      }
-    }
-  }
-
-  if (current && current.trim().length > 0) {
-    sentences.push(current.trim());
-  }
-
-  return sentences;
-}
-
-function fallbackSentenceSplit(text) {
-  return text.split(/(?<=[.!?])\s+/).filter((s) => s.trim().length > 0);
-}
+// Multilingual Sentence Segmenter using Intl.Segmenter with Abbreviation & Boundary Protection
+
+const RU_ABBREVIATIONS = new Set([
+  "т.д", "т.п", "т.е", "и др", "г", "гг", "ул", "руб", "коп", "стр", "рис", "см", "им", "пер", "д", "к", "п", "в"
+]);
+
+const EN_ABBREVIATIONS = new Set([
+  "e.g", "i.e", "etc", "vs", "dr", "mr", "mrs", "ms", "prof", "inc", "ltd", "co", "vol", "no", "p", "pp", "fig"
+]);
+
+function isAbbreviation(word) {
+  if (!word) return false;
+  const clean = word.trim().toLowerCase().replace(/\.$/, "");
+  return RU_ABBREVIATIONS.has(clean) || EN_ABBREVIATIONS.has(clean);
+}
+
+export function splitSentencesMultilingual(text, langHint = "ru") {
+  if (!text || typeof text !== "string") return [];
+  const trimmed = text.trim();
+  if (trimmed.length === 0) return [];
+
+  let rawSegments = [];
+
+  if (typeof Intl !== "undefined" && Intl.Segmenter) {
+    try {
+      const segmenter = new Intl.Segmenter([langHint, "ru", "en"], { granularity: "sentence" });
+      const iter = segmenter.segment(trimmed);
+      for (const seg of iter) {
+        if (seg.segment.trim().length > 0) {
+          rawSegments.push(seg.segment);
+        }
+      }
+    } catch {
+      rawSegments = fallbackSentenceSplit(trimmed);
+    }
+  } else {
+    rawSegments = fallbackSentenceSplit(trimmed);
+  }
+
+  // Post-process & merge false splits caused by abbreviations or numbers (e.g., "v1.0", "e.g.", "т. д.")
+  const sentences = [];
+  let current = "";
+
+  for (let i = 0; i < rawSegments.length; i++) {
+    const seg = rawSegments[i];
+    if (!current) {
+      current = seg;
+    } else {
+      const lastWordMatch = current.trim().match(/([a-zA-Zа-яА-Я0-9._-]+)\s*\.?$/);
+      const lastWord = lastWordMatch ? lastWordMatch[1] : "";
+      
+      const isNumDot = /\b\d+\.$/.test(current.trim());
+      const isAbbr = isAbbreviation(lastWord) || isNumDot;
+      const startsWithLowercase = /^[a-zа-я]/.test(seg.trim());
+
+      if (isAbbr || startsWithLowercase) {
+        current += seg;
+      } else {
+        sentences.push(current.trim());
+        current = seg;
+      }
+    }
+  }
+
+  if (current && current.trim().length > 0) {
+    sentences.push(current.trim());
+  }
+
+  return sentences;
+}
+
+function fallbackSentenceSplit(text) {
+  return text.split(/(?<=[.!?])\s+/).filter((s) => s.trim().length > 0);
+}