@lojban/semantic-search-mcp 1.0.5 → 1.0.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lojban/semantic-search-mcp",
-  "version": "1.0.5",
+  "version": "1.0.7",
   "description": "Local-first MCP server for semantic search using transformers.js and SQLite",
   "type": "module",
   "scripts": {

package/src/index.ts

@@ -7,7 +7,7 @@ import {
 } from '@modelcontextprotocol/sdk/types.js';
 import path from 'path';
 import { getEmbedding, getBatchEmbeddings } from './embeddings.js';
-import { createVectorStorage, type SearchResult } from './storage.js';
+import { createVectorStorage, type SearchResult, type VectorStorage } from './storage.js';
 import { scanDirectories } from './scanner.js';
 
 // Data dir: use env, or project cwd so each workspace has its own index when run via npx from Cursor
@@ -16,6 +16,139 @@ const dataDir =
   path.join(process.cwd(), '.semantic-search', 'data');
 const DB_PATH = path.join(dataDir, 'vectors.db');
 
+type IndexStatus = {
+  isIndexing: boolean;
+  startedAt: number | null;
+  finishedAt: number | null;
+  lastError: string | null;
+  indexedLines: number;
+  indexedFiles: number;
+  directories: string[];
+};
+
+const indexStatus: IndexStatus = {
+  isIndexing: false,
+  startedAt: null,
+  finishedAt: null,
+  lastError: null,
+  indexedLines: 0,
+  indexedFiles: 0,
+  directories: [],
+};
+
+async function startIndexing(storage: VectorStorage, directories: string[]): Promise<void> {
+  if (!directories.length) {
+    console.error('No directories to index. Set SEMANTIC_SEARCH_INDEX_DIRS (comma-separated paths).');
+    return;
+  }
+
+  if (indexStatus.isIndexing) {
+    console.error('Indexing already in progress, not starting a new job.');
+    return;
+  }
+
+  indexStatus.isIndexing = true;
+  indexStatus.startedAt = Date.now();
+  indexStatus.finishedAt = null;
+  indexStatus.lastError = null;
+  indexStatus.directories = directories;
+  indexStatus.indexedLines = 0;
+  indexStatus.indexedFiles = 0;
+
+  try {
+    storage.clear();
+
+    console.error(`Scanning ${directories.length} directories (background indexing)...`);
+
+    let indexedCount = 0;
+    const BATCH_SIZE = 512;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    let currentBatch: any[] = [];
+
+    const processBatch = async (batchToProcess: any[]) => {
+      if (batchToProcess.length === 0) return;
+      const contents = batchToProcess.map((l) => l.content);
+      const embeddings = await getBatchEmbeddings(contents);
+
+      const batchData = batchToProcess.map((line, idx) => ({
+        filePath: line.filePath,
+        lineNumber: line.lineNumber,
+        content: line.content,
+        embedding: embeddings[idx],
+      }));
+
+      await storage.upsertLinesBatch(batchData);
+      indexedCount += batchToProcess.length;
+      indexStatus.indexedLines = indexedCount;
+      console.error(`Indexed ${indexedCount} lines...`);
+    };
+
+    // Pipelining: Read next batch while processing current batch
+    // We allow ONE batch to be processed in parallel with reading the next one.
+    let processingPromise: Promise<void> | null = null;
+
+    for await (const line of scanDirectories(directories)) {
+      currentBatch.push(line);
+      if (currentBatch.length >= BATCH_SIZE) {
+        // If there's a previous batch still processing, wait for it
+        if (processingPromise) {
+          await processingPromise;
+        }
+
+        const batchToProcess = currentBatch;
+        currentBatch = [];
+
+        // Start processing this batch, but don't await it yet!
+        // This allows the loop to continue and read the next batch from disk.
+        processingPromise = processBatch(batchToProcess).catch((err) => {
+          console.error('Error in background batch processing:', err);
+        });
+      }
+    }
+
+    // Wait for the last async batch
+    if (processingPromise) {
+      await processingPromise;
+    }
+
+    // Process any remaining lines
+    if (currentBatch.length > 0) {
+      await processBatch(currentBatch);
+    }
+
+    const stats = await storage.getStats();
+    indexStatus.indexedFiles = stats.totalFiles;
+    indexStatus.indexedLines = stats.totalLines;
+    indexStatus.finishedAt = Date.now();
+
+    console.error(
+      `Finished indexing ${stats.totalLines} lines from ${stats.totalFiles} files in background job.`
+    );
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    indexStatus.lastError = message;
+    indexStatus.finishedAt = Date.now();
+    console.error('Error during indexing job:', err);
+  } finally {
+    indexStatus.isIndexing = false;
+  }
+}
+
+async function ensureInitialIndexing(storage: VectorStorage): Promise<void> {
+  const envDirs = process.env.SEMANTIC_SEARCH_INDEX_DIRS;
+  const directories = envDirs ? envDirs.split(',').map((d) => d.trim()).filter(Boolean) : [];
+
+  if (!directories.length) {
+    console.error(
+      'Semantic Search MCP: SEMANTIC_SEARCH_INDEX_DIRS is not set; automatic indexing on startup is disabled.'
+    );
+    return;
+  }
+
+  // Fire-and-forget; indexing runs in background.
+  void startIndexing(storage, directories);
+}
+
 async function main() {
   const storage = await createVectorStorage(DB_PATH);
 
@@ -36,7 +169,8 @@ async function main() {
       tools: [
         {
           name: 'index_directories',
-          description: 'Scan directories from SEMANTIC_SEARCH_INDEX_DIRS (comma-separated) and index all text file lines for semantic search. Each line gets a vector embedding. Always clears the existing index first.',
+          description:
+            'Trigger background indexing of directories from SEMANTIC_SEARCH_INDEX_DIRS (comma-separated). Clears and rebuilds the index asynchronously.',
           inputSchema: {
             type: 'object',
             properties: {},
@@ -44,7 +178,8 @@ async function main() {
         },
         {
           name: 'search',
-          description: 'Search for lines semantically similar to the query. Returns the most relevant lines from indexed files.',
+          description:
+            'Search for lines semantically similar to the query. Returns the most relevant lines from indexed files.',
           inputSchema: {
             type: 'object',
             properties: {
@@ -63,7 +198,7 @@ async function main() {
         },
         {
           name: 'get_index_stats',
-          description: 'Get statistics about the current index (number of files and lines indexed)',
+          description: 'Get statistics and progress for the current index (files, lines, progress state)',
           inputSchema: {
             type: 'object',
             properties: {},
@@ -82,42 +217,14 @@ async function main() {
           const envDirs = process.env.SEMANTIC_SEARCH_INDEX_DIRS;
           const directories = envDirs ? envDirs.split(',').map((d) => d.trim()).filter(Boolean) : [];
           if (!directories.length) {
-            throw new Error('No directories to index. Set SEMANTIC_SEARCH_INDEX_DIRS (comma-separated paths).');
+            throw new Error(
+              'No directories to index. Set SEMANTIC_SEARCH_INDEX_DIRS (comma-separated paths).'
+            );
           }
-          storage.clear();
-
-          console.error(`Scanning ${directories.length} directories...`);
-          const lines = await scanDirectories(directories);
-          console.error(`Found ${lines.length} lines to index`);
-
-          const batchSize = 128;
-          let indexed = 0;
-
-          // Pipeline: compute embeddings for next batch while writing current batch to DB
-          let embedPromise: Promise<Float32Array[]> =
-            lines.length > 0
-              ? getBatchEmbeddings(lines.slice(0, batchSize).map((l) => l.content))
-              : Promise.resolve([]);
-
-          for (let i = 0; i < lines.length; i += batchSize) {
-            const batch = lines.slice(i, i + batchSize);
-            const embeddings = await embedPromise;
-
-            if (i + batchSize < lines.length) {
-              const nextTexts = lines.slice(i + batchSize, i + batchSize * 2).map((l) => l.content);
-              embedPromise = getBatchEmbeddings(nextTexts);
-            }
-
-            const batchData = batch.map((line, idx) => ({
-              filePath: line.filePath,
-              lineNumber: line.lineNumber,
-              content: line.content,
-              embedding: embeddings[idx],
-            }));
-
-            await storage.upsertLinesBatch(batchData);
-            indexed += batch.length;
-            console.error(`Indexed ${indexed}/${lines.length} lines`);
+
+          // Trigger (or reuse) background indexing job.
+          if (!indexStatus.isIndexing) {
+            void startIndexing(storage, directories);
           }
 
           const stats = await storage.getStats();
@@ -127,9 +234,15 @@ async function main() {
                 type: 'text',
                 text: JSON.stringify({
                   success: true,
+                  indexing: indexStatus.isIndexing,
                   indexed_lines: stats.totalLines,
                   indexed_files: stats.totalFiles,
-                  message: `Successfully indexed ${stats.totalLines} lines from ${stats.totalFiles} files`,
+                  started_at: indexStatus.startedAt,
+                  finished_at: indexStatus.finishedAt,
+                  last_error: indexStatus.lastError,
+                  message: indexStatus.isIndexing
+                    ? `Indexing started in background. Currently ${stats.totalLines} lines from ${stats.totalFiles} files in index.`
+                    : `Indexing completed. Indexed ${stats.totalLines} lines from ${stats.totalFiles} files.`,
                 }),
               },
             ],
@@ -170,6 +283,13 @@ async function main() {
                 text: JSON.stringify({
                   total_files: stats.totalFiles,
                   total_lines: stats.totalLines,
+                  is_indexing: indexStatus.isIndexing,
+                  indexed_lines: indexStatus.indexedLines,
+                  indexed_files: indexStatus.indexedFiles,
+                  started_at: indexStatus.startedAt,
+                  finished_at: indexStatus.finishedAt,
+                  last_error: indexStatus.lastError,
+                  directories: indexStatus.directories,
                 }),
               },
             ],
@@ -191,6 +311,9 @@ async function main() {
   const transport = new StdioServerTransport();
   await server.connect(transport);
   console.error('Semantic Search MCP Server running on stdio');
+
+  // Kick off initial background indexing when the MCP server is enabled.
+  await ensureInitialIndexing(storage);
 }
 
 main().catch(console.error);

package/src/scanner.ts

@@ -1,6 +1,7 @@
-import { readFileSync, statSync } from 'fs';
+import { createReadStream, statSync } from 'fs';
 import { glob } from 'glob';
 import path from 'path';
+import readline from 'readline';
 
 export interface FileLine {
   filePath: string;
@@ -9,7 +10,7 @@ export interface FileLine {
 }
 
 // File extensions to index
-const TEXT_EXTENSIONS = ['.txt', '.md', '.tsv', '.csv', '.json', '.html', '.xml'];
+const TEXT_EXTENSIONS = ['.txt', '.md', '.tsv', '.csv'];
 
 // Minimum line length to index (skip very short lines)
 const MIN_LINE_LENGTH = 5;
@@ -26,50 +27,53 @@ function isTextFile(filePath: string): boolean {
 }
 
 /**
- * Scan a directory for text files and extract lines
+ * Scan a directory for text files and yield lines
  */
-export async function scanDirectory(dirPath: string): Promise<FileLine[]> {
-  const lines: FileLine[] = [];
-  
+export async function* scanDirectory(dirPath: string): AsyncGenerator<FileLine> {
   // Find all files in directory recursively
   const pattern = path.join(dirPath, '**/*');
+  // Nodir: true ensures we only get files
   const files = await glob(pattern, { nodir: true, absolute: true });
-  
+
   for (const filePath of files) {
     if (!isTextFile(filePath)) continue;
-    
+
     try {
       const stats = statSync(filePath);
       if (stats.size > MAX_FILE_SIZE) {
         console.error(`Skipping large file: ${filePath}`);
         continue;
       }
-      
-      const content = readFileSync(filePath, 'utf-8');
-      const fileLines = content.split('\n');
-      
-      for (let i = 0; i < fileLines.length; i++) {
-        const line = fileLines[i].trim();
-        if (line.length >= MIN_LINE_LENGTH) {
-          lines.push({
+
+      const fileStream = createReadStream(filePath);
+      const rl = readline.createInterface({
+        input: fileStream,
+        crlfDelay: Infinity,
+      });
+
+      let lineNumber = 0;
+      for await (const line of rl) {
+        lineNumber++;
+        const trimmed = line.trim();
+        if (trimmed.length >= MIN_LINE_LENGTH) {
+          yield {
             filePath,
-            lineNumber: i + 1, // 1-indexed
-            content: line,
-          });
+            lineNumber,
+            content: trimmed,
+          };
         }
       }
     } catch (err) {
       console.error(`Error reading file ${filePath}:`, err);
     }
   }
-  
-  return lines;
 }
 
 /**
- * Scan multiple directories in parallel
+ * Scan multiple directories and yield lines
  */
-export async function scanDirectories(dirPaths: string[]): Promise<FileLine[]> {
-  const results = await Promise.all(dirPaths.map((dirPath) => scanDirectory(dirPath)));
-  return results.flat();
+export async function* scanDirectories(dirPaths: string[]): AsyncGenerator<FileLine> {
+  for (const dirPath of dirPaths) {
+    yield* scanDirectory(dirPath);
+  }
 }