@lojban/semantic-search-mcp 1.0.6 → 1.0.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lojban/semantic-search-mcp",
-  "version": "1.0.6",
+  "version": "1.0.7",
   "description": "Local-first MCP server for semantic search using transformers.js and SQLite",
   "type": "module",
   "scripts": {

package/src/index.ts

@@ -7,7 +7,7 @@ import {
 } from '@modelcontextprotocol/sdk/types.js';
 import path from 'path';
 import { getEmbedding, getBatchEmbeddings } from './embeddings.js';
-import { createVectorStorage, type SearchResult } from './storage.js';
+import { createVectorStorage, type SearchResult, type VectorStorage } from './storage.js';
 import { scanDirectories } from './scanner.js';
 
 // Data dir: use env, or project cwd so each workspace has its own index when run via npx from Cursor
@@ -16,6 +16,139 @@ const dataDir =
   path.join(process.cwd(), '.semantic-search', 'data');
 const DB_PATH = path.join(dataDir, 'vectors.db');
 
+type IndexStatus = {
+  isIndexing: boolean;
+  startedAt: number | null;
+  finishedAt: number | null;
+  lastError: string | null;
+  indexedLines: number;
+  indexedFiles: number;
+  directories: string[];
+};
+
+const indexStatus: IndexStatus = {
+  isIndexing: false,
+  startedAt: null,
+  finishedAt: null,
+  lastError: null,
+  indexedLines: 0,
+  indexedFiles: 0,
+  directories: [],
+};
+
+async function startIndexing(storage: VectorStorage, directories: string[]): Promise<void> {
+  if (!directories.length) {
+    console.error('No directories to index. Set SEMANTIC_SEARCH_INDEX_DIRS (comma-separated paths).');
+    return;
+  }
+
+  if (indexStatus.isIndexing) {
+    console.error('Indexing already in progress, not starting a new job.');
+    return;
+  }
+
+  indexStatus.isIndexing = true;
+  indexStatus.startedAt = Date.now();
+  indexStatus.finishedAt = null;
+  indexStatus.lastError = null;
+  indexStatus.directories = directories;
+  indexStatus.indexedLines = 0;
+  indexStatus.indexedFiles = 0;
+
+  try {
+    storage.clear();
+
+    console.error(`Scanning ${directories.length} directories (background indexing)...`);
+
+    let indexedCount = 0;
+    const BATCH_SIZE = 512;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    let currentBatch: any[] = [];
+
+    const processBatch = async (batchToProcess: any[]) => {
+      if (batchToProcess.length === 0) return;
+      const contents = batchToProcess.map((l) => l.content);
+      const embeddings = await getBatchEmbeddings(contents);
+
+      const batchData = batchToProcess.map((line, idx) => ({
+        filePath: line.filePath,
+        lineNumber: line.lineNumber,
+        content: line.content,
+        embedding: embeddings[idx],
+      }));
+
+      await storage.upsertLinesBatch(batchData);
+      indexedCount += batchToProcess.length;
+      indexStatus.indexedLines = indexedCount;
+      console.error(`Indexed ${indexedCount} lines...`);
+    };
+
+    // Pipelining: Read next batch while processing current batch
+    // We allow ONE batch to be processed in parallel with reading the next one.
+    let processingPromise: Promise<void> | null = null;
+
+    for await (const line of scanDirectories(directories)) {
+      currentBatch.push(line);
+      if (currentBatch.length >= BATCH_SIZE) {
+        // If there's a previous batch still processing, wait for it
+        if (processingPromise) {
+          await processingPromise;
+        }
+
+        const batchToProcess = currentBatch;
+        currentBatch = [];
+
+        // Start processing this batch, but don't await it yet!
+        // This allows the loop to continue and read the next batch from disk.
+        processingPromise = processBatch(batchToProcess).catch((err) => {
+          console.error('Error in background batch processing:', err);
+        });
+      }
+    }
+
+    // Wait for the last async batch
+    if (processingPromise) {
+      await processingPromise;
+    }
+
+    // Process any remaining lines
+    if (currentBatch.length > 0) {
+      await processBatch(currentBatch);
+    }
+
+    const stats = await storage.getStats();
+    indexStatus.indexedFiles = stats.totalFiles;
+    indexStatus.indexedLines = stats.totalLines;
+    indexStatus.finishedAt = Date.now();
+
+    console.error(
+      `Finished indexing ${stats.totalLines} lines from ${stats.totalFiles} files in background job.`
+    );
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    indexStatus.lastError = message;
+    indexStatus.finishedAt = Date.now();
+    console.error('Error during indexing job:', err);
+  } finally {
+    indexStatus.isIndexing = false;
+  }
+}
+
+async function ensureInitialIndexing(storage: VectorStorage): Promise<void> {
+  const envDirs = process.env.SEMANTIC_SEARCH_INDEX_DIRS;
+  const directories = envDirs ? envDirs.split(',').map((d) => d.trim()).filter(Boolean) : [];
+
+  if (!directories.length) {
+    console.error(
+      'Semantic Search MCP: SEMANTIC_SEARCH_INDEX_DIRS is not set; automatic indexing on startup is disabled.'
+    );
+    return;
+  }
+
+  // Fire-and-forget; indexing runs in background.
+  void startIndexing(storage, directories);
+}
+
 async function main() {
   const storage = await createVectorStorage(DB_PATH);
 
@@ -36,7 +169,8 @@ async function main() {
       tools: [
         {
           name: 'index_directories',
-          description: 'Scan directories from SEMANTIC_SEARCH_INDEX_DIRS (comma-separated) and index all text file lines for semantic search. Each line gets a vector embedding. Always clears the existing index first.',
+          description:
+            'Trigger background indexing of directories from SEMANTIC_SEARCH_INDEX_DIRS (comma-separated). Clears and rebuilds the index asynchronously.',
           inputSchema: {
             type: 'object',
             properties: {},
@@ -44,7 +178,8 @@ async function main() {
         },
         {
           name: 'search',
-          description: 'Search for lines semantically similar to the query. Returns the most relevant lines from indexed files.',
+          description:
+            'Search for lines semantically similar to the query. Returns the most relevant lines from indexed files.',
           inputSchema: {
             type: 'object',
             properties: {
@@ -63,7 +198,7 @@ async function main() {
         },
         {
           name: 'get_index_stats',
-          description: 'Get statistics about the current index (number of files and lines indexed)',
+          description: 'Get statistics and progress for the current index (files, lines, progress state)',
           inputSchema: {
             type: 'object',
             properties: {},
@@ -82,65 +217,14 @@ async function main() {
           const envDirs = process.env.SEMANTIC_SEARCH_INDEX_DIRS;
           const directories = envDirs ? envDirs.split(',').map((d) => d.trim()).filter(Boolean) : [];
           if (!directories.length) {
-            throw new Error('No directories to index. Set SEMANTIC_SEARCH_INDEX_DIRS (comma-separated paths).');
+            throw new Error(
+              'No directories to index. Set SEMANTIC_SEARCH_INDEX_DIRS (comma-separated paths).'
+            );
           }
-          storage.clear();
-
-          console.error(`Scanning ${directories.length} directories (streaming)...`);
-
-          let indexedCount = 0;
-          const BATCH_SIZE = 512;
-          // eslint-disable-next-line @typescript-eslint/no-explicit-any
-          let currentBatch: any[] = [];
-          
-          const processBatch = async (batchToProcess: any[]) => {
-            if (batchToProcess.length === 0) return;
-            const contents = batchToProcess.map((l) => l.content);
-            const embeddings = await getBatchEmbeddings(contents);
-
-            const batchData = batchToProcess.map((line, idx) => ({
-              filePath: line.filePath,
-              lineNumber: line.lineNumber,
-              content: line.content,
-              embedding: embeddings[idx],
-            }));
-
-            await storage.upsertLinesBatch(batchData);
-            indexedCount += batchToProcess.length;
-            console.error(`Indexed ${indexedCount} lines...`);
-          };
 
-          // Pipelining: Read next batch while processing current batch
-          // We allow ONE batch to be processed in parallel with reading the next one.
-          let processingPromise: Promise<void> | null = null;
-          
-          for await (const line of scanDirectories(directories)) {
-            currentBatch.push(line);
-            if (currentBatch.length >= BATCH_SIZE) {
-               // If there's a previous batch still processing, wait for it
-               if (processingPromise) {
-                 await processingPromise;
-               }
-
-               const batchToProcess = currentBatch;
-               currentBatch = [];
-               
-               // Start processing this batch, but don't await it yet!
-               // This allows the loop to continue and read the next batch from disk.
-               processingPromise = processBatch(batchToProcess).catch((err) => {
-                 console.error('Error in background batch processing:', err);
-               });
-            }
-          }
-
-          // Wait for the last async batch
-          if (processingPromise) {
-            await processingPromise;
-          }
-
-          // Process any remaining lines
-          if (currentBatch.length > 0) {
-            await processBatch(currentBatch);
+          // Trigger (or reuse) background indexing job.
+          if (!indexStatus.isIndexing) {
+            void startIndexing(storage, directories);
           }
 
           const stats = await storage.getStats();
@@ -150,9 +234,15 @@ async function main() {
                 type: 'text',
                 text: JSON.stringify({
                   success: true,
+                  indexing: indexStatus.isIndexing,
                   indexed_lines: stats.totalLines,
                   indexed_files: stats.totalFiles,
-                  message: `Successfully indexed ${stats.totalLines} lines from ${stats.totalFiles} files`,
+                  started_at: indexStatus.startedAt,
+                  finished_at: indexStatus.finishedAt,
+                  last_error: indexStatus.lastError,
+                  message: indexStatus.isIndexing
+                    ? `Indexing started in background. Currently ${stats.totalLines} lines from ${stats.totalFiles} files in index.`
+                    : `Indexing completed. Indexed ${stats.totalLines} lines from ${stats.totalFiles} files.`,
                 }),
               },
             ],
@@ -193,6 +283,13 @@ async function main() {
                 text: JSON.stringify({
                   total_files: stats.totalFiles,
                   total_lines: stats.totalLines,
+                  is_indexing: indexStatus.isIndexing,
+                  indexed_lines: indexStatus.indexedLines,
+                  indexed_files: indexStatus.indexedFiles,
+                  started_at: indexStatus.startedAt,
+                  finished_at: indexStatus.finishedAt,
+                  last_error: indexStatus.lastError,
+                  directories: indexStatus.directories,
                 }),
               },
             ],
@@ -214,6 +311,9 @@ async function main() {
   const transport = new StdioServerTransport();
   await server.connect(transport);
   console.error('Semantic Search MCP Server running on stdio');
+
+  // Kick off initial background indexing when the MCP server is enabled.
+  await ensureInitialIndexing(storage);
 }
 
 main().catch(console.error);

package/src/scanner.ts

@@ -10,7 +10,7 @@ export interface FileLine {
 }
 
 // File extensions to index
-const TEXT_EXTENSIONS = ['.txt', '.md', '.tsv', '.csv', '.json', '.html', '.xml', '.ts'];
+const TEXT_EXTENSIONS = ['.txt', '.md', '.tsv', '.csv'];
 
 // Minimum line length to index (skip very short lines)
 const MIN_LINE_LENGTH = 5;