@lojban/semantic-search-mcp 1.0.7 → 1.0.9

package/README.md

@@ -101,6 +101,16 @@ To replace the entire index with new content from several places:
 
 Paths can be anywhere on disk (e.g. different drives or projects); the server reads and indexes all supported text/TSV/CSV files under each directory recursively.
 
+### Memory and batch size
+
+Indexing uses **adaptive batch size** based on free system RAM so the OS doesn’t freeze on low-memory machines. The server reads `os.freemem()`, keeps a reserve (default 400MB), and caps batch size between 32 and 512 lines. You can tune this with env vars:
+
+- **`SEMANTIC_SEARCH_RESERVE_MB`** — MB of RAM to keep free (default `400`).
+- **`SEMANTIC_SEARCH_MIN_BATCH`** — minimum lines per batch (default `32`).
+- **`SEMANTIC_SEARCH_MAX_BATCH`** — maximum lines per batch (default `512`).
+
+Example: `SEMANTIC_SEARCH_RESERVE_MB=800 SEMANTIC_SEARCH_MAX_BATCH=256` to leave more headroom and use smaller batches.
+
 ## Example: Lojban dictionary gaps
 
 1. Put your dictionary TSV (e.g. `jbo-eng.tsv`) in a folder.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lojban/semantic-search-mcp",
-  "version": "1.0.7",
+  "version": "1.0.9",
   "description": "Local-first MCP server for semantic search using transformers.js and SQLite",
   "type": "module",
   "scripts": {

package/src/index.ts

@@ -5,6 +5,7 @@ import {
   CallToolRequestSchema,
   ListToolsRequestSchema,
 } from '@modelcontextprotocol/sdk/types.js';
+import os from 'node:os';
 import path from 'path';
 import { getEmbedding, getBatchEmbeddings } from './embeddings.js';
 import { createVectorStorage, type SearchResult, type VectorStorage } from './storage.js';
@@ -36,17 +37,51 @@ const indexStatus: IndexStatus = {
   directories: [],
 };
 
-async function startIndexing(storage: VectorStorage, directories: string[]): Promise<void> {
+// Single "mutex": only one indexing job is allowed to run. Starting a new job aborts the previous one.
+let currentIndexingAbortController: AbortController | null = null;
+let currentJobId = 0;
+
+// Adaptive batch size: reserve RAM so we don't freeze the OS (env overrides in bytes or MB)
+const RESERVE_MB = Number(process.env.SEMANTIC_SEARCH_RESERVE_MB) || 400;
+const RESERVE_BYTES = RESERVE_MB * 1024 * 1024;
+const MIN_BATCH = Number(process.env.SEMANTIC_SEARCH_MIN_BATCH) || 32;
+const MAX_BATCH = Number(process.env.SEMANTIC_SEARCH_MAX_BATCH) || 512;
+
+/** Rough bytes per indexed line in memory: line text + path + embedding (384 floats) + overhead */
+const BYTES_PER_LINE_ESTIMATE = 4000;
+
+/**
+ * Compute batch size from current free system RAM. Keeps reserve free to avoid freezing the OS.
+ */
+function getAdaptiveBatchSize(): number {
+  const free = os.freemem();
+  const available = free > RESERVE_BYTES ? free - RESERVE_BYTES : Math.floor(free / 2);
+  const batch = Math.floor(available / BYTES_PER_LINE_ESTIMATE);
+  const clamped = Math.max(MIN_BATCH, Math.min(MAX_BATCH, batch));
+  return clamped;
+}
+
+/**
+ * Request indexing of directories. If another indexing job is running, it is aborted first.
+ * Then a new job is started (clears index and rebuilds).
+ */
+function requestIndexing(storage: VectorStorage, directories: string[]): void {
   if (!directories.length) {
     console.error('No directories to index. Set SEMANTIC_SEARCH_INDEX_DIRS (comma-separated paths).');
     return;
   }
 
-  if (indexStatus.isIndexing) {
-    console.error('Indexing already in progress, not starting a new job.');
-    return;
+  // Abort any in-progress indexing so it doesn't conflict or flush this job's work.
+  if (currentIndexingAbortController) {
+    currentIndexingAbortController.abort();
+    currentIndexingAbortController = null;
   }
 
+  currentJobId += 1;
+  const jobId = currentJobId;
+  currentIndexingAbortController = new AbortController();
+  const signal = currentIndexingAbortController.signal;
+
   indexStatus.isIndexing = true;
   indexStatus.startedAt = Date.now();
   indexStatus.finishedAt = null;
@@ -55,13 +90,24 @@ async function startIndexing(storage: VectorStorage, directories: string[]): Pro
   indexStatus.indexedLines = 0;
   indexStatus.indexedFiles = 0;
 
+  void startIndexing(storage, directories, signal, jobId);
+}
+
+async function startIndexing(
+  storage: VectorStorage,
+  directories: string[],
+  signal: AbortSignal,
+  jobId: number
+): Promise<void> {
+  const isCurrentJob = (): boolean => currentJobId === jobId;
+
   try {
-    storage.clear();
+    if (signal.aborted) return;
 
+    storage.clear();
     console.error(`Scanning ${directories.length} directories (background indexing)...`);
 
     let indexedCount = 0;
-    const BATCH_SIZE = 512;
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     let currentBatch: any[] = [];
 
@@ -79,43 +125,48 @@ async function startIndexing(storage: VectorStorage, directories: string[]): Pro
 
       await storage.upsertLinesBatch(batchData);
       indexedCount += batchToProcess.length;
-      indexStatus.indexedLines = indexedCount;
+      if (isCurrentJob()) indexStatus.indexedLines = indexedCount;
       console.error(`Indexed ${indexedCount} lines...`);
     };
 
-    // Pipelining: Read next batch while processing current batch
-    // We allow ONE batch to be processed in parallel with reading the next one.
     let processingPromise: Promise<void> | null = null;
+    let batchSize = getAdaptiveBatchSize();
+    console.error(`Adaptive batch size: ${batchSize} (free RAM: ${Math.round(os.freemem() / 1024 / 1024)}MB, reserve: ${RESERVE_MB}MB)`);
 
     for await (const line of scanDirectories(directories)) {
+      if (signal.aborted) break;
+
       currentBatch.push(line);
-      if (currentBatch.length >= BATCH_SIZE) {
-        // If there's a previous batch still processing, wait for it
+      if (currentBatch.length >= batchSize) {
         if (processingPromise) {
           await processingPromise;
         }
+        if (signal.aborted) break;
 
         const batchToProcess = currentBatch;
         currentBatch = [];
+        batchSize = getAdaptiveBatchSize();
 
-        // Start processing this batch, but don't await it yet!
-        // This allows the loop to continue and read the next batch from disk.
         processingPromise = processBatch(batchToProcess).catch((err) => {
           console.error('Error in background batch processing:', err);
         });
       }
     }
 
-    // Wait for the last async batch
+    if (signal.aborted) {
+      console.error('Indexing aborted (new job started or cancelled).');
+      return;
+    }
+
     if (processingPromise) {
       await processingPromise;
     }
-
-    // Process any remaining lines
     if (currentBatch.length > 0) {
       await processBatch(currentBatch);
     }
 
+    if (!isCurrentJob()) return;
+
     const stats = await storage.getStats();
     indexStatus.indexedFiles = stats.totalFiles;
     indexStatus.indexedLines = stats.totalLines;
@@ -126,15 +177,22 @@ async function startIndexing(storage: VectorStorage, directories: string[]): Pro
     );
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);
-    indexStatus.lastError = message;
-    indexStatus.finishedAt = Date.now();
+    if (isCurrentJob()) {
+      indexStatus.lastError = message;
+      indexStatus.finishedAt = Date.now();
+    }
     console.error('Error during indexing job:', err);
   } finally {
-    indexStatus.isIndexing = false;
+    if (isCurrentJob()) {
+      indexStatus.isIndexing = false;
+    }
+    if (currentIndexingAbortController && currentJobId === jobId) {
+      currentIndexingAbortController = null;
+    }
   }
 }
 
-async function ensureInitialIndexing(storage: VectorStorage): Promise<void> {
+function ensureInitialIndexing(storage: VectorStorage): void {
   const envDirs = process.env.SEMANTIC_SEARCH_INDEX_DIRS;
   const directories = envDirs ? envDirs.split(',').map((d) => d.trim()).filter(Boolean) : [];
 
@@ -145,8 +203,7 @@ async function ensureInitialIndexing(storage: VectorStorage): Promise<void> {
     return;
   }
 
-  // Fire-and-forget; indexing runs in background.
-  void startIndexing(storage, directories);
+  requestIndexing(storage, directories);
 }
 
 async function main() {
@@ -222,10 +279,8 @@ async function main() {
             );
           }
 
-          // Trigger (or reuse) background indexing job.
-          if (!indexStatus.isIndexing) {
-            void startIndexing(storage, directories);
-          }
+          // Abort any in-progress indexing and start a new job (clears and rebuilds).
+          requestIndexing(storage, directories);
 
           const stats = await storage.getStats();
           return {
@@ -313,7 +368,7 @@ async function main() {
   console.error('Semantic Search MCP Server running on stdio');
 
   // Kick off initial background indexing when the MCP server is enabled.
-  await ensureInitialIndexing(storage);
+  ensureInitialIndexing(storage);
 }
 
 main().catch(console.error);