@lojban/semantic-search-mcp 1.0.12 → 1.0.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lojban/semantic-search-mcp",
-  "version": "1.0.12",
+  "version": "1.0.13",
   "description": "Local-first MCP server for semantic search using transformers.js and SQLite",
   "type": "module",
   "scripts": {

package/src/embeddings.ts

@@ -36,8 +36,8 @@ export async function getBatchEmbeddings(texts: string[]): Promise<Float32Array[
   const ext = await getExtractor();
   const results: Float32Array[] = [];
 
-  // Process in batches for memory; each batch is one model forward pass
-  const batchSize = 64;
+  // Process in batches for memory; each batch is one model forward pass (smaller = lower peak RAM)
+  const batchSize = 32;
   for (let i = 0; i < texts.length; i += batchSize) {
     const batch = texts.slice(i, i + batchSize);
     const output = await ext(batch, { pooling: 'mean', normalize: true });

package/src/index.ts

@@ -42,8 +42,8 @@ async function runBackgroundIndexing(
   try {
     let currentBatch: Array<{ filePath: string; lineNumber: number; content: string }> = [];
     let processingPromise: Promise<void> | null = null;
-    const seenFiles = new Set<string>();
 
+    let batchesProcessed = 0;
     const processBatch = async (
       batch: Array<{ filePath: string; lineNumber: number; content: string }>
     ): Promise<void> => {
@@ -57,9 +57,12 @@ async function runBackgroundIndexing(
         embedding: embeddings[idx],
       }));
       await storage.upsertLinesBatch(batchData);
-      for (const l of batch) seenFiles.add(l.filePath);
       indexingState.linesIndexed += batch.length;
-      indexingState.filesIndexed = seenFiles.size;
+      batchesProcessed++;
+      if (batchesProcessed === 1 || batchesProcessed % 10 === 0) {
+        const stats = await storage.getStats();
+        indexingState.filesIndexed = stats.totalFiles;
+      }
     };
 
     const yieldToEventLoop = (): Promise<void> =>

package/src/scanner.ts

@@ -1,5 +1,5 @@
 import { createReadStream, statSync } from 'fs';
-import { glob } from 'glob';
+import { globIterate } from 'glob';
 import path from 'path';
 import readline from 'readline';
 
@@ -18,6 +18,9 @@ const MIN_LINE_LENGTH = 5;
 // Maximum file size to process (skip very large files)
 const MAX_FILE_SIZE = 10 * 1024 * 1024; // 10MB
 
+// Cap line length to avoid unbounded readline buffer (e.g. file with no newlines)
+const MAX_LINE_LENGTH = 256 * 1024; // 256KB
+
 /**
  * Check if a file is a text file we should index
  */
@@ -27,17 +30,17 @@ function isTextFile(filePath: string): boolean {
 }
 
 /**
- * Scan a directory for text files and yield lines
+ * Scan a directory for text files and yield lines.
+ * Uses globIterate so file paths are streamed one-by-one (no full list in RAM).
+ * Readline has maxLineLength to avoid huge single-line buffers.
  */
 export async function* scanDirectory(dirPath: string): AsyncGenerator<FileLine> {
-  // Find all files in directory recursively
   const pattern = path.join(dirPath, '**/*');
-  // Nodir: true ensures we only get files
-  const files = await glob(pattern, { nodir: true, absolute: true });
-
-  for (const filePath of files) {
+  for await (const filePath of globIterate(pattern, { nodir: true, absolute: true }) as AsyncIterable<string>) {
     if (!isTextFile(filePath)) continue;
 
+    let fileStream: ReturnType<typeof createReadStream> | null = null;
+    let rl: readline.Interface | null = null;
     try {
       const stats = statSync(filePath);
       if (stats.size > MAX_FILE_SIZE) {
@@ -45,10 +48,11 @@ export async function* scanDirectory(dirPath: string): AsyncGenerator<FileLine>
         continue;
       }
 
-      const fileStream = createReadStream(filePath);
-      const rl = readline.createInterface({
+      fileStream = createReadStream(filePath);
+      rl = readline.createInterface({
         input: fileStream,
         crlfDelay: Infinity,
+        maxLineLength: MAX_LINE_LENGTH,
       });
 
       let lineNumber = 0;
@@ -65,6 +69,9 @@ export async function* scanDirectory(dirPath: string): AsyncGenerator<FileLine>
       }
     } catch (err) {
       console.error(`Error reading file ${filePath}:`, err);
+    } finally {
+      rl?.close();
+      fileStream?.destroy();
     }
   }
 }

package/src/storage.ts

@@ -31,6 +31,10 @@ export class VectorStorage {
   }
 
   private init(): void {
+    // Limit SQLite page cache to avoid unbounded RAM (negative = kibibytes)
+    try {
+      this.db.exec('PRAGMA cache_size = -65536'); // 64MB max
+    } catch {}
     this.db.exec(`
       CREATE TABLE IF NOT EXISTS lines (
         id INTEGER PRIMARY KEY AUTOINCREMENT,