codebasesearch 0.1.4

package/mcp.js

@@ -0,0 +1,276 @@
+#!/usr/bin/env node
+
+// MUST patch sharp before any other imports
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const distPath = path.join(__dirname, 'node_modules', '@huggingface', 'transformers', 'dist', 'transformers.node.mjs');
+
+if (fs.existsSync(distPath)) {
+  let content = fs.readFileSync(distPath, 'utf-8');
+  if (!content.includes('SHARP_REMOVED_FOR_WINDOWS_COMPATIBILITY')) {
+    content = content.replace(/import \* as __WEBPACK_EXTERNAL_MODULE_sharp__ from "sharp";\n/, '// SHARP_REMOVED_FOR_WINDOWS_COMPATIBILITY\n');
+    content = content.replace(/module\.exports = __WEBPACK_EXTERNAL_MODULE_sharp__;/g, 'module.exports = {};');
+    content = content.replace(/} else \{\s*throw new Error\('Unable to load image processing library\.'\);\s*\}/, '} else {\n    loadImageFunction = async () => { throw new Error(\'Image processing unavailable\'); };\n}');
+    try { fs.writeFileSync(distPath, content); } catch (e) {}
+  }
+}
+
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
+import { cwd } from 'process';
+import { join, resolve } from 'path';
+import { existsSync, readFileSync, appendFileSync, writeFileSync } from 'fs';
+import { loadIgnorePatterns } from './src/ignore-parser.js';
+import { scanRepository } from './src/scanner.js';
+import { generateEmbeddings } from './src/embeddings.js';
+import { initStore, upsertChunks, closeStore } from './src/store.js';
+import { executeSearch } from './src/search.js';
+
+async function ensureIgnoreEntry(rootPath) {
+  const gitignorePath = join(rootPath, '.gitignore');
+  const entry = '.code-search/';
+
+  try {
+    if (existsSync(gitignorePath)) {
+      const content = readFileSync(gitignorePath, 'utf8');
+      if (!content.includes(entry)) {
+        appendFileSync(gitignorePath, `\n${entry}`);
+      }
+    } else {
+      writeFileSync(gitignorePath, `${entry}\n`);
+    }
+  } catch (e) {
+    // Ignore write errors, proceed with search anyway
+  }
+}
+
+class CodeSearchManager {
+  async search(repositoryPath, query) {
+    const absolutePath = resolve(repositoryPath);
+
+    if (!existsSync(absolutePath)) {
+      return {
+        error: `Repository path not found: ${absolutePath}`,
+        results: [],
+      };
+    }
+
+    try {
+      // Ensure .code-search/ is in .gitignore
+      await ensureIgnoreEntry(absolutePath);
+
+      // Load ignore patterns
+      const ignorePatterns = loadIgnorePatterns(absolutePath);
+      const dbPath = join(absolutePath, '.code-search');
+
+      // Initialize store
+      await initStore(dbPath);
+
+      // Scan repository
+      const chunks = scanRepository(absolutePath, ignorePatterns);
+
+      if (chunks.length === 0) {
+        await closeStore();
+        return {
+          query,
+          results: [],
+          message: 'No code chunks found in repository',
+        };
+      }
+
+      // Generate embeddings in batches
+      const batchSize = 32;
+      const chunkTexts = chunks.map(c => c.content);
+      const allEmbeddings = [];
+
+      for (let i = 0; i < chunkTexts.length; i += batchSize) {
+        const batchTexts = chunkTexts.slice(i, i + batchSize);
+        const batchEmbeddings = await generateEmbeddings(batchTexts);
+        allEmbeddings.push(...batchEmbeddings);
+      }
+
+      // Create chunks with embeddings
+      const chunksWithEmbeddings = chunks.map((chunk, idx) => ({
+        ...chunk,
+        vector: allEmbeddings[idx],
+      }));
+
+      // Upsert to store
+      await upsertChunks(chunksWithEmbeddings);
+
+      // Execute search
+      const results = await executeSearch(query);
+
+      // Format results
+      const formattedResults = results.map((result, idx) => ({
+        rank: idx + 1,
+        file: result.file_path,
+        lines: `${result.line_start}-${result.line_end}`,
+        score: (result.score * 100).toFixed(1),
+        snippet: result.content.split('\n').slice(0, 3).join('\n'),
+      }));
+
+      await closeStore();
+
+      return {
+        query,
+        repository: absolutePath,
+        resultsCount: formattedResults.length,
+        results: formattedResults,
+      };
+    } catch (error) {
+      await closeStore().catch(() => {});
+      return {
+        error: error.message,
+        results: [],
+      };
+    }
+  }
+}
+
+const manager = new CodeSearchManager();
+
+const server = new Server(
+  {
+    name: 'code-search-mcp',
+    version: '0.1.0',
+  },
+  {
+    capabilities: {
+      tools: {},
+    },
+  }
+);
+
+server.setRequestHandler(ListToolsRequestSchema, async () => {
+  return {
+    tools: [
+      {
+        name: 'search',
+        description:
+          'Search through a code repository using semantic search with Jina embeddings. Automatically indexes the repository before searching.',
+        inputSchema: {
+          type: 'object',
+          properties: {
+            repository_path: {
+              type: 'string',
+              description:
+                'Absolute or relative path to the repository to search in (defaults to current directory)',
+            },
+            query: {
+              type: 'string',
+              description:
+                'Natural language search query (e.g., "authentication middleware", "database connection")',
+            },
+          },
+          required: ['query'],
+        },
+      },
+    ],
+  };
+});
+
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+  const { name, arguments: args } = request.params;
+
+  if (name !== 'search') {
+    return {
+      content: [
+        {
+          type: 'text',
+          text: `Unknown tool: ${name}`,
+        },
+      ],
+      isError: true,
+    };
+  }
+
+  const query = args?.query;
+  const repositoryPath = args?.repository_path || cwd();
+
+  if (!query || typeof query !== 'string') {
+    return {
+      content: [
+        {
+          type: 'text',
+          text: 'Error: query is required and must be a string',
+        },
+      ],
+      isError: true,
+    };
+  }
+
+  try {
+    const result = await manager.search(repositoryPath, query);
+
+    if (result.error) {
+      return {
+        content: [
+          {
+            type: 'text',
+            text: `Error: ${result.error}`,
+          },
+        ],
+        isError: true,
+      };
+    }
+
+    const text =
+      result.resultsCount === 0
+        ? `No results found for query: "${query}"\nRepository: ${result.repository}`
+        : `Found ${result.resultsCount} result${result.resultsCount !== 1 ? 's' : ''} for query: "${query}"\nRepository: ${result.repository}\n\n${result.results
+            .map(
+              (r) =>
+                `${r.rank}. ${r.file}:${r.lines} (score: ${r.score}%)\n${r.snippet
+                  .split('\n')
+                  .map((line) => `   ${line}`)
+                  .join('\n')}`
+            )
+            .join('\n\n')}`;
+
+    return {
+      content: [
+        {
+          type: 'text',
+          text,
+        },
+      ],
+    };
+  } catch (error) {
+    return {
+      content: [
+        {
+          type: 'text',
+          text: `Error: ${error.message}`,
+        },
+      ],
+      isError: true,
+    };
+  }
+});
+
+export async function startMcpServer() {
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+}
+
+const isMain = process.argv[1] && (
+  process.argv[1] === fileURLToPath(import.meta.url) || 
+  process.argv[1].endsWith('mcp.js') || 
+  process.argv[1].endsWith('code-search-mcp')
+);
+
+if (isMain) {
+  main().catch((error) => {
+    console.error('Server error:', error);
+    process.exit(1);
+  });
+}
+
+async function main() {
+  await startMcpServer();
+}

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "codebasesearch",
+  "version": "0.1.4",
+  "description": "Ultra-simple code search tool with Jina embeddings, LanceDB, and MCP protocol support",
+  "type": "module",
+  "bin": {
+    "code-search": "./bin/code-search.js",
+    "code-search-mcp": "./mcp.js"
+  },
+  "main": "src/cli.js",
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "scripts": {
+    "start": "node ./bin/code-search.js",
+    "postinstall": "node scripts/patch-transformers.js"
+  },
+  "dependencies": {
+    "@huggingface/transformers": "^3.8.1",
+    "@modelcontextprotocol/sdk": "1.0.0",
+    "apache-arrow": "^14.0.0",
+    "fast-glob": "^3.3.2",
+    "onnxruntime-node": "^1.23.2",
+    "onnxruntime-web": "^1.19.0",
+    "vectordb": "^0.21.2"
+  },
+  "keywords": [
+    "code-search",
+    "embeddings",
+    "vector-db",
+    "jina",
+    "lancedb",
+    "mcp"
+  ]
+}

package/scripts/patch-transformers.js

@@ -0,0 +1,42 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+// Patch @huggingface/transformers dist file for Windows compatibility
+const distPath = path.join(__dirname, '..', 'node_modules', '@huggingface', 'transformers', 'dist', 'transformers.node.mjs');
+
+if (!fs.existsSync(distPath)) {
+  console.log('transformers.node.mjs not found, skipping patch');
+  process.exit(0);
+}
+
+let distContent = fs.readFileSync(distPath, 'utf-8');
+
+// Check if already patched
+if (distContent.includes('SHARP_PATCHED_FOR_WINDOWS')) {
+  console.log('transformers.node.mjs already patched');
+  process.exit(0);
+}
+
+// Remove sharp import line
+distContent = distContent.replace(
+  /import \* as __WEBPACK_EXTERNAL_MODULE_sharp__ from "sharp";\n/,
+  '// SHARP_PATCHED_FOR_WINDOWS: sharp removed\n'
+);
+
+// Replace sharp module exports with stub
+distContent = distContent.replace(
+  /module\.exports = __WEBPACK_EXTERNAL_MODULE_sharp__;/g,
+  'module.exports = {};'
+);
+
+// Replace image processing error with fallback
+distContent = distContent.replace(
+  /} else \{\s*throw new Error\('Unable to load image processing library\.'\);\s*\}/,
+  '} else {\n    loadImageFunction = async () => { throw new Error(\'Image processing unavailable\'); };\n}'
+);
+
+fs.writeFileSync(distPath, distContent);
+console.log('Successfully patched transformers.node.mjs for Windows compatibility');

package/src/cli.js

@@ -0,0 +1,113 @@
+import { cwd } from 'process';
+import { join } from 'path';
+import { existsSync, readFileSync, appendFileSync, writeFileSync } from 'fs';
+import { loadIgnorePatterns } from './ignore-parser.js';
+import { scanRepository } from './scanner.js';
+import { generateEmbeddings } from './embeddings.js';
+import { initStore, upsertChunks, closeStore } from './store.js';
+import { executeSearch, formatResults } from './search.js';
+import { startMcpServer } from '../mcp.js';
+
+async function isGitRepository(rootPath) {
+  const gitDir = join(rootPath, '.git');
+  try {
+    return existsSync(gitDir);
+  } catch {
+    return false;
+  }
+}
+
+async function ensureIgnoreEntry(rootPath) {
+  const gitignorePath = join(rootPath, '.gitignore');
+  const entry = '.code-search/';
+
+  try {
+    if (existsSync(gitignorePath)) {
+      const content = readFileSync(gitignorePath, 'utf8');
+      if (!content.includes(entry)) {
+        appendFileSync(gitignorePath, `\n${entry}`);
+      }
+    } else {
+      writeFileSync(gitignorePath, `${entry}\n`);
+    }
+  } catch (e) {
+    // Ignore write errors, proceed with search anyway
+  }
+}
+
+
+export async function run(args) {
+  try {
+    // Start MCP server if no arguments provided
+    if (args.length === 0) {
+      await startMcpServer();
+      return;
+    }
+
+    const query = args.join(' ');
+    const rootPath = cwd();
+
+    console.log(`Code Search Tool`);
+    console.log(`Root: ${rootPath}\n`);
+
+    // Check if git repo
+    const isGit = await isGitRepository(rootPath);
+    if (!isGit) {
+      console.warn('Warning: Not a git repository. Indexing current directory anyway.\n');
+    }
+
+    // Ensure .code-search/ is in .gitignore
+    await ensureIgnoreEntry(rootPath);
+
+    // Load ignore patterns
+    const ignorePatterns = loadIgnorePatterns(rootPath);
+    const dbPath = join(rootPath, '.code-search');
+
+    // Initialize store
+    await initStore(dbPath);
+
+    // Scan repository
+    console.log('Scanning repository...');
+    const chunks = scanRepository(rootPath, ignorePatterns);
+    console.log(`Found ${chunks.length} code chunks\n`);
+
+    // Always reindex to ensure freshness
+    console.log('Generating embeddings and indexing...');
+
+    // Generate embeddings in batches
+    const batchSize = 32;
+    const chunkTexts = chunks.map(c => c.content);
+    const allEmbeddings = [];
+
+    for (let i = 0; i < chunkTexts.length; i += batchSize) {
+      const batchTexts = chunkTexts.slice(i, i + batchSize);
+      const batchEmbeddings = await generateEmbeddings(batchTexts);
+      allEmbeddings.push(...batchEmbeddings);
+    }
+
+    // Create chunks with embeddings
+    const chunksWithEmbeddings = chunks.map((chunk, idx) => ({
+      ...chunk,
+      vector: allEmbeddings[idx]
+    }));
+
+    // Upsert to store
+    await upsertChunks(chunksWithEmbeddings);
+    console.log('Index created\n');
+
+    // Execute search
+    const results = await executeSearch(query);
+
+    // Format and display results
+    const output = formatResults(results);
+    console.log(output);
+
+    // Clean shutdown
+    await closeStore();
+    process.exit(0);
+  } catch (error) {
+    console.error('Error:', error.message);
+    await closeStore().catch(() => {});
+    process.exit(1);
+  }
+}

package/src/embeddings.js

@@ -0,0 +1,151 @@
+import { pipeline, env } from '@huggingface/transformers';
+import { rmSync, existsSync } from 'fs';
+import { homedir } from 'os';
+import { join } from 'path';
+
+// Force WASM backend only - disable onnxruntime-node to avoid memory issues on Windows
+try {
+  env.backends.onnx.wasm.numThreads = 1;
+  env.backends.onnx.ort = null;
+} catch (e) {
+  // Continue even if env config fails
+}
+
+let modelCache = null;
+let cacheCleared = false;
+
+function clearModelCache() {
+  const cacheDirs = [
+    join(homedir(), '.cache', 'huggingface', 'transformers'),
+    join(process.cwd(), 'node_modules', '@huggingface', 'transformers', '.cache'),
+  ];
+
+  for (const cacheDir of cacheDirs) {
+    try {
+      if (existsSync(cacheDir)) {
+        rmSync(cacheDir, { recursive: true, force: true });
+      }
+    } catch (e) {
+      // Ignore errors, continue with next cache dir
+    }
+  }
+  console.error('Cleared corrupted model cache');
+}
+
+async function getModel(retryOnError = true) {
+  if (modelCache) {
+    return modelCache;
+  }
+
+  console.error('Loading embeddings model (this may take a moment on first run)...');
+
+  const modelLoadPromise = pipeline(
+    'feature-extraction',
+    'Xenova/all-minilm-l6-v2'
+  );
+
+  const timeoutPromise = new Promise((_, reject) =>
+    setTimeout(() => reject(new Error('Model loading timeout after 5 minutes')), 300000)
+  );
+
+  try {
+    modelCache = await Promise.race([modelLoadPromise, timeoutPromise]);
+  } catch (e) {
+    if (retryOnError && !cacheCleared && (e.message.includes('Protobuf') || e.message.includes('parsing'))) {
+      console.error('Detected corrupted cache, clearing and retrying...');
+      cacheCleared = true;
+      clearModelCache();
+      modelCache = null;
+      return getModel(false);
+    }
+    console.error('Error loading model:', e.message);
+    throw e;
+  }
+
+  return modelCache;
+}
+
+async function meanPooling(modelOutput, attentionMask) {
+  // Get token embeddings from model output
+  const tokenEmbeddings = modelOutput.data;
+  const embeddingDim = modelOutput.dims[modelOutput.dims.length - 1];
+  const batchSize = modelOutput.dims[0];
+  const seqLength = modelOutput.dims[1];
+
+  const pooled = [];
+
+  for (let b = 0; b < batchSize; b++) {
+    let sum = new Array(embeddingDim).fill(0);
+    let count = 0;
+
+    for (let s = 0; s < seqLength; s++) {
+      const tokenIdx = b * seqLength + s;
+      const maskValue = attentionMask[tokenIdx] || 1;
+
+      if (maskValue > 0) {
+        const tokenStart = tokenIdx * embeddingDim;
+        for (let d = 0; d < embeddingDim; d++) {
+          sum[d] += tokenEmbeddings[tokenStart + d] * maskValue;
+        }
+        count += maskValue;
+      }
+    }
+
+    const normalized = sum.map(v => v / Math.max(count, 1e-9));
+    pooled.push(normalized);
+  }
+
+  return pooled;
+}
+
+export async function generateEmbeddings(texts) {
+  const model = await getModel();
+
+  if (!Array.isArray(texts)) {
+    texts = [texts];
+  }
+
+  // Generate embeddings for all texts
+  const embeddings = await model(texts, {
+    pooling: 'mean',
+    normalize: true
+  });
+
+  // Convert to regular arrays
+  const result = [];
+
+  // embeddings is a Tensor, convert to array
+  if (embeddings && embeddings.data) {
+    const data = Array.from(embeddings.data);
+    const shape = embeddings.dims;
+
+    // Shape is [batchSize, embeddingDim]
+    if (shape && shape.length === 2) {
+      const [batchSize, embeddingDim] = shape;
+      for (let i = 0; i < batchSize; i++) {
+        const start = i * embeddingDim;
+        const end = start + embeddingDim;
+        result.push(data.slice(start, end));
+      }
+    } else {
+      // Fallback: assume single embedding
+      result.push(data);
+    }
+  } else if (Array.isArray(embeddings)) {
+    // Already an array
+    for (const emb of embeddings) {
+      if (emb.data) {
+        result.push(Array.from(emb.data));
+      } else {
+        result.push(Array.from(emb));
+      }
+    }
+  }
+
+  return result;
+}
+
+export async function generateSingleEmbedding(text) {
+  const embeddings = await generateEmbeddings([text]);
+  return embeddings[0];
+}