@lojban/semantic-search-mcp 1.0.9 → 1.0.11

package/README.md

@@ -111,6 +111,8 @@ Indexing uses **adaptive batch size** based on free system RAM so the OS doesn
 
 Example: `SEMANTIC_SEARCH_RESERVE_MB=800 SEMANTIC_SEARCH_MAX_BATCH=256` to leave more headroom and use smaller batches.
 
+- **`SEMANTIC_SEARCH_GC`** — explicit GC after each batch is **on by default** when Node is run with `--expose-gc` (helps avoid OS freezes during long indexing). In MCP use e.g. `"args": ["--expose-gc", "-y", "@lojban/semantic-search-mcp"]`. Set to `0` or `false` to disable.
+
 ## Example: Lojban dictionary gaps
 
 1. Put your dictionary TSV (e.g. `jbo-eng.tsv`) in a folder.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lojban/semantic-search-mcp",
-  "version": "1.0.9",
+  "version": "1.0.11",
   "description": "Local-first MCP server for semantic search using transformers.js and SQLite",
   "type": "module",
   "scripts": {

package/src/index.ts

@@ -45,7 +45,9 @@ let currentJobId = 0;
 const RESERVE_MB = Number(process.env.SEMANTIC_SEARCH_RESERVE_MB) || 400;
 const RESERVE_BYTES = RESERVE_MB * 1024 * 1024;
 const MIN_BATCH = Number(process.env.SEMANTIC_SEARCH_MIN_BATCH) || 32;
-const MAX_BATCH = Number(process.env.SEMANTIC_SEARCH_MAX_BATCH) || 512;
+const MAX_BATCH = Number(process.env.SEMANTIC_SEARCH_MAX_BATCH) || 128;
+// Explicit GC after each batch (when --expose-gc is available). Default on; set SEMANTIC_SEARCH_GC=0 or false to disable.
+const ENABLE_GC = process.env.SEMANTIC_SEARCH_GC !== '0' && process.env.SEMANTIC_SEARCH_GC !== 'false';
 
 /** Rough bytes per indexed line in memory: line text + path + embedding (384 floats) + overhead */
 const BYTES_PER_LINE_ESTIMATE = 4000;
@@ -127,9 +129,14 @@ async function startIndexing(
       indexedCount += batchToProcess.length;
       if (isCurrentJob()) indexStatus.indexedLines = indexedCount;
       console.error(`Indexed ${indexedCount} lines...`);
+      // Explicit GC when enabled (default) and Node run with --expose-gc
+      if (ENABLE_GC && typeof (globalThis as { gc?: () => void }).gc === 'function') {
+        (globalThis as { gc: () => void }).gc();
+      }
     };
 
-    let processingPromise: Promise<void> | null = null;
+    // Single task queue: only one batch is processed at a time (no pipelining).
+    // We do not read the next batch until the current one is fully done, to avoid memory spikes and OS freezes.
     let batchSize = getAdaptiveBatchSize();
     console.error(`Adaptive batch size: ${batchSize} (free RAM: ${Math.round(os.freemem() / 1024 / 1024)}MB, reserve: ${RESERVE_MB}MB)`);
 
@@ -138,18 +145,12 @@ async function startIndexing(
 
       currentBatch.push(line);
       if (currentBatch.length >= batchSize) {
-        if (processingPromise) {
-          await processingPromise;
-        }
-        if (signal.aborted) break;
-
         const batchToProcess = currentBatch;
         currentBatch = [];
         batchSize = getAdaptiveBatchSize();
 
-        processingPromise = processBatch(batchToProcess).catch((err) => {
-          console.error('Error in background batch processing:', err);
-        });
+        await processBatch(batchToProcess);
+        if (signal.aborted) break;
       }
     }
 
@@ -158,9 +159,6 @@ async function startIndexing(
       return;
     }
 
-    if (processingPromise) {
-      await processingPromise;
-    }
     if (currentBatch.length > 0) {
       await processBatch(currentBatch);
     }

package/src/scanner.ts

@@ -38,6 +38,7 @@ export async function* scanDirectory(dirPath: string): AsyncGenerator<FileLine>
   for (const filePath of files) {
     if (!isTextFile(filePath)) continue;
 
+    let fileStream: ReturnType<typeof createReadStream> | null = null;
     try {
       const stats = statSync(filePath);
       if (stats.size > MAX_FILE_SIZE) {
@@ -45,26 +46,35 @@ export async function* scanDirectory(dirPath: string): AsyncGenerator<FileLine>
         continue;
       }
 
-      const fileStream = createReadStream(filePath);
+      fileStream = createReadStream(filePath);
       const rl = readline.createInterface({
         input: fileStream,
         crlfDelay: Infinity,
       });
 
-      let lineNumber = 0;
-      for await (const line of rl) {
-        lineNumber++;
-        const trimmed = line.trim();
-        if (trimmed.length >= MIN_LINE_LENGTH) {
-          yield {
-            filePath,
-            lineNumber,
-            content: trimmed,
-          };
+      try {
+        let lineNumber = 0;
+        for await (const line of rl) {
+          lineNumber++;
+          const trimmed = line.trim();
+          if (trimmed.length >= MIN_LINE_LENGTH) {
+            yield {
+              filePath,
+              lineNumber,
+              content: trimmed,
+            };
+          }
         }
+      } finally {
+        rl.close();
+        fileStream.destroy();
+        fileStream = null;
       }
     } catch (err) {
       console.error(`Error reading file ${filePath}:`, err);
+      if (fileStream) {
+        fileStream.destroy();
+      }
     }
   }
 }