codebasesearch 0.1.24 → 0.1.26

package/mcp.js

@@ -22,20 +22,9 @@ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'
 import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
 import { cwd } from 'process';
 import { join } from 'path';
-import { existsSync, readFileSync, appendFileSync, writeFileSync, readdirSync } from 'fs';
-import { homedir } from 'os';
+import { existsSync, readFileSync, appendFileSync, writeFileSync } from 'fs';
 import { supervisor } from './src/supervisor.js';
 
-const WORKSPACE_PATH = join(homedir(), 'workspace');
-
-function getWorkspaceFolders() {
-  try {
-    return readdirSync(WORKSPACE_PATH, { withFileTypes: true })
-      .filter(e => e.isDirectory() && !e.name.startsWith('.'))
-      .map(e => join(WORKSPACE_PATH, e.name));
-  } catch { return []; }
-}
-
 function ensureIgnoreEntry(rootPath) {
   const gitignorePath = join(rootPath, '.gitignore');
   const entry = '.code-search/';
@@ -49,10 +38,10 @@ function ensureIgnoreEntry(rootPath) {
   } catch (e) {}
 }
 
-function formatResults(result, query, scope) {
-  if (result.resultsCount === 0) return `No results found${scope ? ` across ${scope}` : ''} for: "${query}"`;
+function formatResults(result, query) {
+  if (result.resultsCount === 0) return `No results found for: "${query}"`;
   const plural = result.resultsCount !== 1 ? 's' : '';
-  const header = `Found ${result.resultsCount} result${plural}${scope ? ` across ${scope}` : ''} for: "${query}"\n\n`;
+  const header = `Found ${result.resultsCount} result${plural} for: "${query}"\n\n`;
   const body = result.results.map((r) => {
     const pathPart = r.relativePath || r.absolutePath;
     const lineCount = r.totalLines ? ` [${r.totalLines}L]` : '';
@@ -88,18 +77,6 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({
         required: ['query'],
       },
     },
-    {
-      name: 'search_workspace',
-      description: 'Search across ALL repositories in ~/workspace simultaneously. Returns ranked results with repo name prefix.',
-      inputSchema: {
-        type: 'object',
-        properties: {
-          query: { type: 'string', description: 'Natural language search query' },
-          limit: { type: 'number', description: 'Max results to return (default: 10)' },
-        },
-        required: ['query'],
-      },
-    },
   ],
 }));
 
@@ -107,26 +84,15 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
   const { name, arguments: args } = request.params;
   const query = args?.query;
 
-  if (!['search', 'search_workspace'].includes(name)) return errResponse(`Unknown tool: ${name}`);
+  if (name !== 'search') return errResponse(`Unknown tool: ${name}`);
   if (!query || typeof query !== 'string') return errResponse('Error: query is required and must be a string');
 
   try {
-    if (name === 'search_workspace') {
-      const result = await supervisor.sendRequest({
-        type: 'search-all',
-        query,
-        workspacePaths: getWorkspaceFolders(),
-        limit: args?.limit || 10,
-      });
-      if (result.error) return errResponse(`Error: ${result.error}`);
-      return okResponse(formatResults(result, query, 'workspace'));
-    }
-
     const repositoryPath = args?.repository_path || cwd();
     ensureIgnoreEntry(repositoryPath);
     const result = await supervisor.sendRequest({ type: 'search', query, repositoryPath });
     if (result.error) return errResponse(`Error: ${result.error}`);
-    return okResponse(formatResults(result, query, null));
+    return okResponse(formatResults(result, query));
   } catch (error) {
     return errResponse(`Error: ${error.message}`);
   }
@@ -135,13 +101,6 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
 export async function startMcpServer() {
   const transport = new StdioServerTransport();
   await server.connect(transport);
-
-  const workspacePaths = getWorkspaceFolders();
-  if (workspacePaths.length > 0) {
-    supervisor.sendRequest({ type: 'index-all', workspacePaths })
-      .then(r => console.error(`[MCP] Pre-indexed workspace: ${r.message || JSON.stringify(r)}`))
-      .catch(e => console.error(`[MCP] Pre-index warning: ${e.message}`));
-  }
 }
 
 const isMain = process.argv[1] && (

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codebasesearch",
-  "version": "0.1.24",
+  "version": "0.1.26",
   "description": "Ultra-simple code search tool with Jina embeddings, LanceDB, and MCP protocol support",
   "type": "module",
   "bin": {

package/src/ignore-parser.js

@@ -197,6 +197,8 @@ const IGNORED_DIRECTORIES = new Set([
   'node_modules', 'bower_components', 'jspm_packages', 'web_modules',
   // Version control
   '.git', '.svn', '.hg', '.bzr',
+  // Tool config (AI assistants, editors)
+  '.claude', '.cursor', '.aider',
   // IDE
   '.vscode', '.idea', '.vs', '.atom',
   // Build outputs (unambiguous names only)

package/src/search-worker.js

@@ -1,6 +1,6 @@
 import { parentPort } from 'worker_threads';
 import { resolve, relative } from 'path';
-import { existsSync, readFileSync, readdirSync, statSync } from 'fs';
+import { existsSync, readFileSync, statSync } from 'fs';
 import { loadIgnorePatterns } from './ignore-parser.js';
 import { scanRepository } from './scanner.js';
 import { buildTextIndex, searchText } from './text-search.js';
@@ -21,26 +21,22 @@ function findEnclosingContext(content, lineStart) {
 }
 
 function getFileTotalLines(absoluteFilePath) {
+  if (fileLineCountCache.has(absoluteFilePath)) {
+    return fileLineCountCache.get(absoluteFilePath);
+  }
   try {
     const content = readFileSync(absoluteFilePath, 'utf8');
-    return content.split('\n').length;
+    const count = content.split('\n').length;
+    fileLineCountCache.set(absoluteFilePath, count);
+    return count;
   } catch {
     return null;
   }
 }
 
 let indexCache = new Map();
-
-function getWorkspaceFolders(workspacePath) {
-  try {
-    const entries = readdirSync(workspacePath, { withFileTypes: true });
-    return entries
-      .filter(e => e.isDirectory() && !e.name.startsWith('.'))
-      .map(e => resolve(workspacePath, e.name));
-  } catch {
-    return [];
-  }
-}
+// Cache file line counts to avoid repeated disk reads on every search
+const fileLineCountCache = new Map();
 
 async function initializeIndex(repositoryPath) {
   const absolutePath = resolve(repositoryPath);
@@ -87,7 +83,17 @@ async function performSearch(repositoryPath, query) {
       return { error: indexData.error, results: [] };
     }
 
-    const results = searchText(query, indexData.chunks, indexData.indexData);
+    const rawResults = searchText(query, indexData.chunks, indexData.indexData);
+
+    // Deduplicate: keep best-scoring chunk per file, then take top results
+    const bestPerFile = new Map();
+    for (const r of rawResults) {
+      const existing = bestPerFile.get(r.file_path);
+      if (!existing || r.score > existing.score) {
+        bestPerFile.set(r.file_path, r);
+      }
+    }
+    const results = Array.from(bestPerFile.values()).sort((a, b) => b.score - a.score);
 
     return {
       query,
@@ -115,53 +121,6 @@ async function performSearch(repositoryPath, query) {
   }
 }
 
-async function performSearchAll(workspacePaths, query, limit = 10) {
-  const allResults = [];
-
-  for (const repoPath of workspacePaths) {
-    const absolutePath = resolve(repoPath);
-    if (!existsSync(absolutePath)) continue;
-
-    const indexData = await initializeIndex(absolutePath);
-    if (indexData.error || !indexData.chunks) continue;
-
-    const results = searchText(query, indexData.chunks, indexData.indexData);
-    const repoName = absolutePath.split('/').pop();
-
-    const seenFiles = new Set();
-    for (const r of results) {
-      if (!seenFiles.has(r.file_path)) {
-        seenFiles.add(r.file_path);
-        allResults.push({ ...r, repoName, repoPath: absolutePath });
-      }
-      if (seenFiles.size >= limit) break;
-    }
-  }
-
-  allResults.sort((a, b) => b.score - a.score);
-  const top = allResults.slice(0, limit);
-
-  return {
-    query,
-    resultsCount: top.length,
-    results: top.map((r, idx) => {
-      const absoluteFilePath = resolve(r.repoPath, r.file_path);
-      const totalLines = getFileTotalLines(absoluteFilePath);
-      const enclosingContext = findEnclosingContext(r.content, r.line_start);
-      return {
-        rank: idx + 1,
-        absolutePath: absoluteFilePath,
-        relativePath: `${r.repoName}/${r.file_path}`,
-        lines: `${r.line_start}-${r.line_end}`,
-        totalLines,
-        enclosingContext,
-        score: (r.score * 100).toFixed(1),
-        snippet: r.content.split('\n').slice(0, 30).join('\n'),
-      };
-    }),
-  };
-}
-
 if (parentPort) {
   parentPort.on('message', async (msg) => {
     try {
@@ -170,26 +129,6 @@ if (parentPort) {
         return;
       }
 
-      if (msg.type === 'index-all') {
-        const folders = msg.workspacePaths || getWorkspaceFolders(msg.workspacePath || '');
-        let indexed = 0;
-        for (const folder of folders) {
-          if (existsSync(folder)) {
-            await initializeIndex(folder);
-            indexed++;
-          }
-        }
-        parentPort.postMessage({ id: msg.id, result: { indexed, message: `Indexed ${indexed} repositories` } });
-        return;
-      }
-
-      if (msg.type === 'search-all') {
-        const folders = msg.workspacePaths || getWorkspaceFolders(msg.workspacePath || '');
-        const result = await performSearchAll(folders, msg.query, msg.limit || 10);
-        parentPort.postMessage({ id: msg.id, result });
-        return;
-      }
-
       if (msg.type === 'search') {
         const result = await performSearch(msg.repositoryPath || process.cwd(), msg.query);
         parentPort.postMessage({ id: msg.id, result });

package/src/text-search.js

@@ -1,35 +1,71 @@
 export function buildTextIndex(chunks) {
   const index = new Map();
-  const chunkMetadata = [];
-
-  chunks.forEach((chunk, idx) => {
-    const tokens = tokenize(chunk.content);
-    const fileNameTokens = tokenize(chunk.file_path);
-    const symbols = extractSymbols(chunk.content);
-    const frequency = new Map();
-
-    tokens.forEach(token => {
-      frequency.set(token, (frequency.get(token) || 0) + 1);
-      if (!index.has(token)) {
-        index.set(token, new Set());
-      }
-      index.get(token).add(idx);
-    });
+  const chunkMetadata = new Array(chunks.length);
+
+  for (let idx = 0; idx < chunks.length; idx++) {
+    const chunk = chunks[idx];
+    const frequency = tokenizeToFrequency(chunk.content, index, idx);
+    const fileNameTokens = new Set(tokenize(chunk.file_path));
+    const symbols = new Set(extractSymbols(chunk.content));
 
     chunkMetadata[idx] = {
-      tokens,
       fileNameTokens,
       symbols,
       frequency,
       isCode: isCodeFile(chunk.file_path),
+      contentLower: chunk.content.toLowerCase(),
     };
-  });
+  }
+
+  // Precompute IDF for each token: log((N+1)/(df+1))
+  const N = chunks.length;
+  const idf = new Map();
+  for (const [token, docSet] of index) {
+    idf.set(token, Math.log((N + 1) / (docSet.size + 1)) + 1);
+  }
+
+  return { index, chunkMetadata, idf };
+}
+
+function tokenizeToFrequency(text, index, chunkIdx) {
+  const frequency = new Map();
+
+  for (const word of text.split(/\s+/)) {
+    if (word.length === 0) continue;
+
+    const hasUpperCase = word !== word.toLowerCase();
+    if (hasUpperCase) {
+      const camelTokens = word.match(/[A-Z]?[a-z]+|[A-Z]+(?=[A-Z][a-z]|\d|\W|$)|[0-9]+/g);
+      if (camelTokens) {
+        for (const t of camelTokens) {
+          if (t.length > 1) frequency.set(t.toLowerCase(), (frequency.get(t.toLowerCase()) || 0) + 1);
+        }
+      }
+    }
+
+    const cleaned = word.replace(/[^\w]/g, '').toLowerCase();
+    if (cleaned.length > 1) {
+      frequency.set(cleaned, (frequency.get(cleaned) || 0) + 1);
+      if (word.includes('-') || word.includes('_') || word.includes('.')) {
+        for (const part of word.split(/[-_.]/)) {
+          const partCleaned = part.replace(/[^\w]/g, '').toLowerCase();
+          if (partCleaned.length > 1 && partCleaned !== cleaned) frequency.set(partCleaned, (frequency.get(partCleaned) || 0) + 1);
+        }
+      }
+    }
+  }
 
-  return { index, chunkMetadata };
+  for (const token of frequency.keys()) {
+    let docSet = index.get(token);
+    if (!docSet) { docSet = new Set(); index.set(token, docSet); }
+    docSet.add(chunkIdx);
+  }
+
+  return frequency;
 }
 
 export function searchText(query, chunks, indexData) {
-  const { index, chunkMetadata } = indexData;
+  const { index, chunkMetadata, idf } = indexData;
   const queryTokens = tokenize(query);
   const querySymbols = extractSymbols(query);
   const chunkScores = new Map();
@@ -47,36 +83,53 @@ export function searchText(query, chunks, indexData) {
     }
   });
 
-  for (const idx of candidates) {
+  const queryLower = query.toLowerCase();
+
+  let scoringCandidates = candidates;
+  if (candidates.size > 500) {
+    const ranked = Array.from(candidates).sort((a, b) => {
+      let aSum = 0, bSum = 0;
+      for (const token of queryTokens) {
+        if (index.has(token)) {
+          if (index.get(token).has(a)) aSum += idf.get(token) || 1;
+          if (index.get(token).has(b)) bSum += idf.get(token) || 1;
+        }
+      }
+      return bSum - aSum;
+    });
+    scoringCandidates = new Set(ranked.slice(0, 500));
+  }
+
+  for (const idx of scoringCandidates) {
     const chunk = chunks[idx];
     const meta = chunkMetadata[idx];
     let score = 0;
 
-    // Exact phrase match - highest priority (saves embedding cost)
-    if (chunk.content.toLowerCase().includes(query.toLowerCase())) {
+    if (queryTokens.length > 1 && meta.contentLower.includes(queryLower)) {
       score += 30;
     }
 
     // Symbol match in content - function/class named after query terms
     querySymbols.forEach(symbol => {
-      if (meta.symbols.includes(symbol)) score += 10;
+      if (meta.symbols.has(symbol)) score += 10;
     });
 
     // Filename token match - strong signal that this file is about the query topic
     let fileNameMatches = 0;
     queryTokens.forEach(token => {
-      if (meta.fileNameTokens.includes(token)) fileNameMatches++;
+      if (meta.fileNameTokens.has(token)) fileNameMatches++;
     });
     if (fileNameMatches > 0) {
       score += fileNameMatches * 10;
     }
 
-    // Token frequency scoring
+    // TF-IDF scoring: reward rare tokens that appear in this chunk
     queryTokens.forEach(token => {
       if (index.has(token) && index.get(token).has(idx)) {
-        const freq = meta.frequency.get(token) || 1;
+        const tf = Math.min(meta.frequency.get(token) || 1, 5);
+        const tokenIdf = idf ? (idf.get(token) || 1) : 1;
         const lengthBoost = token.length > 4 ? 1.5 : 1;
-        score += lengthBoost * Math.min(freq, 5);
+        score += lengthBoost * tf * tokenIdf;
       }
     });
 

package/.prd

@@ -1,78 +0,0 @@
-{
-  "project": "code-search",
-  "created": "2026-03-12",
-  "objective": "Profile and improve code-search speed and result quality",
-  "items": [
-    {
-      "id": "fix-dedup-buildtextindex",
-      "subject": "Remove duplicate buildTextIndex from search.js",
-      "status": "pending",
-      "description": "search.js has a private copy of buildTextIndex and tokenize/extractSymbols/isCodeFile that duplicates text-search.js. Import the exported buildTextIndex from text-search.js instead.",
-      "category": "refactor",
-      "effort": "small",
-      "blocking": ["fix-score-normalization", "fix-hybrid-weights"],
-      "blockedBy": []
-    },
-    {
-      "id": "fix-chunk-size",
-      "subject": "Reduce chunk size from 300 to 60 lines for better semantic granularity",
-      "status": "pending",
-      "description": "scanner.js uses 300-line chunks. Embeddings work best on 50-100 line chunks. Reduce to 60-line chunks with 15-line overlap for better vector search quality.",
-      "category": "feature",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": []
-    },
-    {
-      "id": "fix-score-normalization",
-      "subject": "Fix text search score normalization so top result is always 1.0",
-      "status": "pending",
-      "description": "Text scores divide raw by 100 but scores can exceed 100. Use dynamic max-score scaling. Lower hasGoodTextResults threshold from 0.5 to 0.3.",
-      "category": "bug",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": ["fix-dedup-buildtextindex"]
-    },
-    {
-      "id": "fix-hybrid-weights",
-      "subject": "Boost text-only exact-match results in hybrid merge",
-      "status": "pending",
-      "description": "Text-only results are capped at 20% weight. Give high-scoring text-only results a floor finalScore of 0.4.",
-      "category": "feature",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": ["fix-dedup-buildtextindex"]
-    },
-    {
-      "id": "fix-vector-cache-key",
-      "subject": "Strengthen vector search cache key to 20 dimensions",
-      "status": "pending",
-      "description": "Cache key uses only first 5 embedding dims. Use 20 dims for near-zero collision rate.",
-      "category": "bug",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": []
-    },
-    {
-      "id": "remove-dead-meanpooling",
-      "subject": "Remove dead meanPooling function from embeddings.js",
-      "status": "pending",
-      "description": "meanPooling is defined but never called. Remove dead code.",
-      "category": "refactor",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": []
-    },
-    {
-      "id": "verify-and-commit",
-      "subject": "Verify improvements and commit all changes",
-      "status": "pending",
-      "description": "Run end-to-end search logic test inline. Commit and push all changes.",
-      "category": "infra",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": ["fix-dedup-buildtextindex", "fix-chunk-size", "fix-score-normalization", "fix-hybrid-weights", "fix-vector-cache-key", "remove-dead-meanpooling"]
-    }
-  ],
-  "completed": []
-}