codebasesearch 0.1.7 → 0.1.8

package/mcp.js

@@ -22,13 +22,8 @@ import { Server } from '@modelcontextprotocol/sdk/server/index.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
 import { cwd } from 'process';
-import { join, resolve } from 'path';
-import { existsSync, readFileSync, appendFileSync, writeFileSync } from 'fs';
-import { loadIgnorePatterns } from './src/ignore-parser.js';
-import { scanRepository } from './src/scanner.js';
-import { generateEmbeddings } from './src/embeddings.js';
-import { initStore, upsertChunks, closeStore } from './src/store.js';
-import { executeSearch } from './src/search.js';
+import { join, existsSync, readFileSync, appendFileSync, writeFileSync } from 'fs';
+import { supervisor } from './src/supervisor.js';
 
 async function ensureIgnoreEntry(rootPath) {
   const gitignorePath = join(rootPath, '.gitignore');
@@ -44,96 +39,10 @@ async function ensureIgnoreEntry(rootPath) {
       writeFileSync(gitignorePath, `${entry}\n`);
     }
   } catch (e) {
-    // Ignore write errors, proceed with search anyway
+    // Ignore write errors
   }
 }
 
-class CodeSearchManager {
-  async search(repositoryPath, query) {
-    const absolutePath = resolve(repositoryPath);
-
-    if (!existsSync(absolutePath)) {
-      return {
-        error: `Repository path not found: ${absolutePath}`,
-        results: [],
-      };
-    }
-
-    try {
-      // Ensure .code-search/ is in .gitignore
-      await ensureIgnoreEntry(absolutePath);
-
-      // Load ignore patterns
-      const ignorePatterns = loadIgnorePatterns(absolutePath);
-      const dbPath = join(absolutePath, '.code-search');
-
-      // Initialize store
-      await initStore(dbPath);
-
-      // Scan repository
-      const chunks = scanRepository(absolutePath, ignorePatterns);
-
-      if (chunks.length === 0) {
-        await closeStore();
-        return {
-          query,
-          results: [],
-          message: 'No code chunks found in repository',
-        };
-      }
-
-      // Generate embeddings in batches
-      const batchSize = 32;
-      const chunkTexts = chunks.map(c => c.content);
-      const allEmbeddings = [];
-
-      for (let i = 0; i < chunkTexts.length; i += batchSize) {
-        const batchTexts = chunkTexts.slice(i, i + batchSize);
-        const batchEmbeddings = await generateEmbeddings(batchTexts);
-        allEmbeddings.push(...batchEmbeddings);
-      }
-
-      // Create chunks with embeddings
-      const chunksWithEmbeddings = chunks.map((chunk, idx) => ({
-        ...chunk,
-        vector: allEmbeddings[idx],
-      }));
-
-      // Upsert to store
-      await upsertChunks(chunksWithEmbeddings);
-
-      // Execute search
-      const results = await executeSearch(query);
-
-      // Format results
-      const formattedResults = results.map((result, idx) => ({
-        rank: idx + 1,
-        file: result.file_path,
-        lines: `${result.line_start}-${result.line_end}`,
-        score: (result.score * 100).toFixed(1),
-        snippet: result.content.split('\n').slice(0, 3).join('\n'),
-      }));
-
-      await closeStore();
-
-      return {
-        query,
-        repository: absolutePath,
-        resultsCount: formattedResults.length,
-        results: formattedResults,
-      };
-    } catch (error) {
-      await closeStore().catch(() => {});
-      return {
-        error: error.message,
-        results: [],
-      };
-    }
-  }
-}
-
-const manager = new CodeSearchManager();
-
 const server = new Server(
   {
     name: 'code-search-mcp',
@@ -205,7 +114,12 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
   }
 
   try {
-    const result = await manager.search(repositoryPath, query);
+    await ensureIgnoreEntry(repositoryPath);
+    const result = await supervisor.sendRequest({
+      type: 'search',
+      query,
+      repositoryPath,
+    });
 
     if (result.error) {
       return {
@@ -267,10 +181,17 @@ const isMain = process.argv[1] && (
 if (isMain) {
   main().catch((error) => {
     console.error('Server error:', error);
-    process.exit(1);
   });
 }
 
+process.on('uncaughtException', (error) => {
+  console.error('Uncaught exception:', error);
+});
+
+process.on('unhandledRejection', (reason) => {
+  console.error('Unhandled rejection:', reason);
+});
+
 async function main() {
   await startMcpServer();
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codebasesearch",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "description": "Ultra-simple code search tool with Jina embeddings, LanceDB, and MCP protocol support",
   "type": "module",
   "bin": {

package/src/search-worker.js

@@ -0,0 +1,103 @@
+import { parentPort } from 'worker_threads';
+import { resolve } from 'path';
+import { existsSync } from 'fs';
+import { loadIgnorePatterns } from './ignore-parser.js';
+import { scanRepository } from './scanner.js';
+import { generateEmbeddings } from './embeddings.js';
+import { initStore, upsertChunks, closeStore } from './store.js';
+import { executeSearch } from './search.js';
+
+async function performSearch(repositoryPath, query) {
+  const absolutePath = resolve(repositoryPath);
+
+  if (!existsSync(absolutePath)) {
+    return { error: 'Repository path not found', results: [] };
+  }
+
+  try {
+    const ignorePatterns = loadIgnorePatterns(absolutePath);
+    const dbPath = resolve(absolutePath, '.code-search');
+
+    await initStore(dbPath);
+
+    const chunks = scanRepository(absolutePath, ignorePatterns);
+    if (chunks.length === 0) {
+      await closeStore();
+      return { query, results: [], message: 'No code chunks found' };
+    }
+
+    const batchSize = 32;
+    const allEmbeddings = [];
+
+    for (let i = 0; i < chunks.length; i += batchSize) {
+      const batchTexts = chunks.slice(i, i + batchSize).map(c => c.content);
+      const batchEmbeddings = await generateEmbeddings(batchTexts);
+      allEmbeddings.push(...batchEmbeddings);
+    }
+
+    const chunksWithEmbeddings = chunks.map((chunk, idx) => ({
+      ...chunk,
+      vector: allEmbeddings[idx],
+    }));
+
+    await upsertChunks(chunksWithEmbeddings);
+    const results = await executeSearch(query);
+    await closeStore();
+
+    return {
+      query,
+      repository: absolutePath,
+      resultsCount: results.length,
+      results: results.map((result, idx) => ({
+        rank: idx + 1,
+        file: result.file_path,
+        lines: `${result.line_start}-${result.line_end}`,
+        score: (result.score * 100).toFixed(1),
+        snippet: result.content.split('\n').slice(0, 3).join('\n'),
+      })),
+    };
+  } catch (error) {
+    await closeStore().catch(() => {});
+    return { error: error.message, results: [] };
+  }
+}
+
+if (parentPort) {
+  parentPort.on('message', async (msg) => {
+    try {
+      if (msg.type === 'health-check') {
+        parentPort.postMessage({ id: -1, type: 'pong' });
+        return;
+      }
+
+      if (msg.type === 'search') {
+        const result = await performSearch(msg.repositoryPath || process.cwd(), msg.query);
+        parentPort.postMessage({ id: msg.id, result });
+      }
+    } catch (error) {
+      console.error('[Worker] Uncaught error:', error.message);
+      try {
+        parentPort.postMessage({
+          id: msg?.id || -1,
+          result: { error: error.message, results: [] }
+        });
+      } catch (e) {
+        console.error('[Worker] Failed to send error response');
+      }
+    }
+  });
+
+  process.on('uncaughtException', (error) => {
+    console.error('[Worker] Uncaught exception:', error.message);
+    try {
+      parentPort.postMessage({
+        error: error.message,
+        results: []
+      });
+    } catch (e) {}
+  });
+
+  process.on('unhandledRejection', (reason) => {
+    console.error('[Worker] Unhandled rejection:', reason);
+  });
+}

package/src/supervisor.js

@@ -0,0 +1,124 @@
+import { Worker } from 'worker_threads';
+import { resolve, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+export class WorkerSupervisor {
+  constructor() {
+    this.worker = null;
+    this.restartDelay = 1000;
+    this.failureCount = 0;
+    this.requestQueue = new Map();
+    this.requestId = 0;
+    this.healthCheckInterval = null;
+    this.startWorker();
+  }
+
+  startWorker() {
+    try {
+      const workerPath = resolve(__dirname, 'search-worker.js');
+      this.worker = new Worker(workerPath);
+
+      this.worker.on('message', this.handleMessage.bind(this));
+      this.worker.on('error', (err) => {
+        console.error('[Supervisor] Worker error:', err.message);
+        this.scheduleRestart();
+      });
+      this.worker.on('exit', (code) => {
+        console.error('[Supervisor] Worker exited with code:', code);
+        this.worker = null;
+        if (code !== 0) {
+          this.failureCount++;
+          this.scheduleRestart();
+        }
+      });
+
+      this.restartDelay = 1000;
+      this.failureCount = 0;
+      console.error('[Supervisor] Worker started');
+      this.setupHealthCheck();
+    } catch (e) {
+      console.error('[Supervisor] Worker start failed:', e.message);
+      this.scheduleRestart();
+    }
+  }
+
+  setupHealthCheck() {
+    if (this.healthCheckInterval) clearInterval(this.healthCheckInterval);
+    this.healthCheckInterval = setInterval(() => {
+      if (this.worker) {
+        try {
+          this.worker.postMessage({ type: 'health-check', id: -1 });
+        } catch (e) {
+          console.error('[Supervisor] Health check failed:', e.message);
+        }
+      }
+    }, 30000);
+  }
+
+  scheduleRestart() {
+    const delay = Math.min(this.restartDelay, 60000);
+    console.error('[Supervisor] Restart scheduled in', delay, 'ms');
+    setTimeout(() => {
+      if (!this.worker) {
+        this.startWorker();
+      }
+    }, delay);
+    this.restartDelay = Math.min(this.restartDelay * 2, 60000);
+  }
+
+  async sendRequest(data) {
+    if (!this.worker) {
+      return { error: 'Worker unavailable, restarting...', results: [] };
+    }
+
+    return new Promise((resolve) => {
+      const id = this.requestId++;
+      const resolveWrapper = resolve;
+
+      const timeout = setTimeout(() => {
+        if (this.requestQueue.has(id)) {
+          this.requestQueue.delete(id);
+          resolveWrapper({ error: 'Request timeout', results: [] });
+        }
+      }, 600000);
+
+      this.requestQueue.set(id, (result) => {
+        clearTimeout(timeout);
+        resolveWrapper(result);
+      });
+
+      try {
+        this.worker.postMessage({ id, ...data });
+      } catch (e) {
+        clearTimeout(timeout);
+        this.requestQueue.delete(id);
+        resolveWrapper({ error: 'Worker communication failed', results: [] });
+        this.scheduleRestart();
+      }
+    });
+  }
+
+  handleMessage(msg) {
+    if (msg.id === -1) return;
+
+    const resolve = this.requestQueue.get(msg.id);
+    if (resolve) {
+      this.requestQueue.delete(msg.id);
+      resolve(msg.result || msg);
+    }
+  }
+
+  shutdown() {
+    if (this.healthCheckInterval) clearInterval(this.healthCheckInterval);
+    if (this.worker) {
+      try {
+        this.worker.terminate();
+      } catch (e) {}
+      this.worker = null;
+    }
+  }
+}
+
+export const supervisor = new WorkerSupervisor();

package/scripts/patch-transformers.js

@@ -1,42 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-
-// Patch @huggingface/transformers dist file for Windows compatibility
-const distPath = path.join(__dirname, '..', 'node_modules', '@huggingface', 'transformers', 'dist', 'transformers.node.mjs');
-
-if (!fs.existsSync(distPath)) {
-  console.log('transformers.node.mjs not found, skipping patch');
-  process.exit(0);
-}
-
-let distContent = fs.readFileSync(distPath, 'utf-8');
-
-// Check if already patched
-if (distContent.includes('SHARP_PATCHED_FOR_WINDOWS')) {
-  console.log('transformers.node.mjs already patched');
-  process.exit(0);
-}
-
-// Remove sharp import line
-distContent = distContent.replace(
-  /import \* as __WEBPACK_EXTERNAL_MODULE_sharp__ from "sharp";\n/,
-  '// SHARP_PATCHED_FOR_WINDOWS: sharp removed\n'
-);
-
-// Replace sharp module exports with stub
-distContent = distContent.replace(
-  /module\.exports = __WEBPACK_EXTERNAL_MODULE_sharp__;/g,
-  'module.exports = {};'
-);
-
-// Replace image processing error with fallback
-distContent = distContent.replace(
-  /} else \{\s*throw new Error\('Unable to load image processing library\.'\);\s*\}/,
-  '} else {\n    loadImageFunction = async () => { throw new Error(\'Image processing unavailable\'); };\n}'
-);
-
-fs.writeFileSync(distPath, distContent);
-console.log('Successfully patched transformers.node.mjs for Windows compatibility');