codebasesearch 0.1.24 → 0.1.25

package/.prd

@@ -1,78 +1 @@
-{
-  "project": "code-search",
-  "created": "2026-03-12",
-  "objective": "Profile and improve code-search speed and result quality",
-  "items": [
-    {
-      "id": "fix-dedup-buildtextindex",
-      "subject": "Remove duplicate buildTextIndex from search.js",
-      "status": "pending",
-      "description": "search.js has a private copy of buildTextIndex and tokenize/extractSymbols/isCodeFile that duplicates text-search.js. Import the exported buildTextIndex from text-search.js instead.",
-      "category": "refactor",
-      "effort": "small",
-      "blocking": ["fix-score-normalization", "fix-hybrid-weights"],
-      "blockedBy": []
-    },
-    {
-      "id": "fix-chunk-size",
-      "subject": "Reduce chunk size from 300 to 60 lines for better semantic granularity",
-      "status": "pending",
-      "description": "scanner.js uses 300-line chunks. Embeddings work best on 50-100 line chunks. Reduce to 60-line chunks with 15-line overlap for better vector search quality.",
-      "category": "feature",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": []
-    },
-    {
-      "id": "fix-score-normalization",
-      "subject": "Fix text search score normalization so top result is always 1.0",
-      "status": "pending",
-      "description": "Text scores divide raw by 100 but scores can exceed 100. Use dynamic max-score scaling. Lower hasGoodTextResults threshold from 0.5 to 0.3.",
-      "category": "bug",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": ["fix-dedup-buildtextindex"]
-    },
-    {
-      "id": "fix-hybrid-weights",
-      "subject": "Boost text-only exact-match results in hybrid merge",
-      "status": "pending",
-      "description": "Text-only results are capped at 20% weight. Give high-scoring text-only results a floor finalScore of 0.4.",
-      "category": "feature",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": ["fix-dedup-buildtextindex"]
-    },
-    {
-      "id": "fix-vector-cache-key",
-      "subject": "Strengthen vector search cache key to 20 dimensions",
-      "status": "pending",
-      "description": "Cache key uses only first 5 embedding dims. Use 20 dims for near-zero collision rate.",
-      "category": "bug",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": []
-    },
-    {
-      "id": "remove-dead-meanpooling",
-      "subject": "Remove dead meanPooling function from embeddings.js",
-      "status": "pending",
-      "description": "meanPooling is defined but never called. Remove dead code.",
-      "category": "refactor",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": []
-    },
-    {
-      "id": "verify-and-commit",
-      "subject": "Verify improvements and commit all changes",
-      "status": "pending",
-      "description": "Run end-to-end search logic test inline. Commit and push all changes.",
-      "category": "infra",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": ["fix-dedup-buildtextindex", "fix-chunk-size", "fix-score-normalization", "fix-hybrid-weights", "fix-vector-cache-key", "remove-dead-meanpooling"]
-    }
-  ],
-  "completed": []
-}
+{}

package/mcp.js

@@ -22,20 +22,9 @@ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'
 import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
 import { cwd } from 'process';
 import { join } from 'path';
-import { existsSync, readFileSync, appendFileSync, writeFileSync, readdirSync } from 'fs';
-import { homedir } from 'os';
+import { existsSync, readFileSync, appendFileSync, writeFileSync } from 'fs';
 import { supervisor } from './src/supervisor.js';
 
-const WORKSPACE_PATH = join(homedir(), 'workspace');
-
-function getWorkspaceFolders() {
-  try {
-    return readdirSync(WORKSPACE_PATH, { withFileTypes: true })
-      .filter(e => e.isDirectory() && !e.name.startsWith('.'))
-      .map(e => join(WORKSPACE_PATH, e.name));
-  } catch { return []; }
-}
-
 function ensureIgnoreEntry(rootPath) {
   const gitignorePath = join(rootPath, '.gitignore');
   const entry = '.code-search/';
@@ -49,10 +38,10 @@ function ensureIgnoreEntry(rootPath) {
   } catch (e) {}
 }
 
-function formatResults(result, query, scope) {
-  if (result.resultsCount === 0) return `No results found${scope ? ` across ${scope}` : ''} for: "${query}"`;
+function formatResults(result, query) {
+  if (result.resultsCount === 0) return `No results found for: "${query}"`;
   const plural = result.resultsCount !== 1 ? 's' : '';
-  const header = `Found ${result.resultsCount} result${plural}${scope ? ` across ${scope}` : ''} for: "${query}"\n\n`;
+  const header = `Found ${result.resultsCount} result${plural} for: "${query}"\n\n`;
   const body = result.results.map((r) => {
     const pathPart = r.relativePath || r.absolutePath;
     const lineCount = r.totalLines ? ` [${r.totalLines}L]` : '';
@@ -88,18 +77,6 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({
         required: ['query'],
       },
     },
-    {
-      name: 'search_workspace',
-      description: 'Search across ALL repositories in ~/workspace simultaneously. Returns ranked results with repo name prefix.',
-      inputSchema: {
-        type: 'object',
-        properties: {
-          query: { type: 'string', description: 'Natural language search query' },
-          limit: { type: 'number', description: 'Max results to return (default: 10)' },
-        },
-        required: ['query'],
-      },
-    },
   ],
 }));
 
@@ -107,26 +84,15 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
   const { name, arguments: args } = request.params;
   const query = args?.query;
 
-  if (!['search', 'search_workspace'].includes(name)) return errResponse(`Unknown tool: ${name}`);
+  if (name !== 'search') return errResponse(`Unknown tool: ${name}`);
   if (!query || typeof query !== 'string') return errResponse('Error: query is required and must be a string');
 
   try {
-    if (name === 'search_workspace') {
-      const result = await supervisor.sendRequest({
-        type: 'search-all',
-        query,
-        workspacePaths: getWorkspaceFolders(),
-        limit: args?.limit || 10,
-      });
-      if (result.error) return errResponse(`Error: ${result.error}`);
-      return okResponse(formatResults(result, query, 'workspace'));
-    }
-
     const repositoryPath = args?.repository_path || cwd();
     ensureIgnoreEntry(repositoryPath);
     const result = await supervisor.sendRequest({ type: 'search', query, repositoryPath });
     if (result.error) return errResponse(`Error: ${result.error}`);
-    return okResponse(formatResults(result, query, null));
+    return okResponse(formatResults(result, query));
   } catch (error) {
     return errResponse(`Error: ${error.message}`);
   }
@@ -135,13 +101,6 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
 export async function startMcpServer() {
   const transport = new StdioServerTransport();
   await server.connect(transport);
-
-  const workspacePaths = getWorkspaceFolders();
-  if (workspacePaths.length > 0) {
-    supervisor.sendRequest({ type: 'index-all', workspacePaths })
-      .then(r => console.error(`[MCP] Pre-indexed workspace: ${r.message || JSON.stringify(r)}`))
-      .catch(e => console.error(`[MCP] Pre-index warning: ${e.message}`));
-  }
 }
 
 const isMain = process.argv[1] && (

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codebasesearch",
-  "version": "0.1.24",
+  "version": "0.1.25",
   "description": "Ultra-simple code search tool with Jina embeddings, LanceDB, and MCP protocol support",
   "type": "module",
   "bin": {

package/profile-results.txt

@@ -0,0 +1,248 @@
+Performance profiling across codebases...
+This will take several minutes.
+
+
+▶ Profiling xbot...
+  Scanned: 7 chunks in 8ms
+Vector store initialized
+Loading embeddings model (this may take a moment on first run)...
+dtype not specified for "model". Using the default dtype (fp32) for this device (cpu).
+Indexed 7 chunks
+  Embeddings: 7/7 chunks in 1530ms
+Searching for: "function class method"
+  Text search: 7 results in 88ms
+
+▶ Profiling fsbrowse...
+  Scanned: 6 chunks in 7ms
+Vector store initialized
+Indexed 6 chunks
+  Embeddings: 6/6 chunks in 1126ms
+Searching for: "function class method"
+  Text search: 6 results in 26ms
+
+▶ Profiling agentauth...
+  Scanned: 11 chunks in 2ms
+Vector store initialized
+Indexed 11 chunks
+  Embeddings: 11/11 chunks in 1957ms
+Searching for: "function class method"
+  Text search: 10 results in 40ms
+
+▶ Profiling webtalk...
+  Scanned: 40 chunks in 28ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 8 chunks
+  Embeddings: 40/40 chunks in 6680ms
+Searching for: "function class method"
+  Text search: 10 results in 80ms
+
+▶ Profiling plugforge...
+  Scanned: 66 chunks in 11ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 2 chunks
+  Embeddings: 66/66 chunks in 10923ms
+Searching for: "function class method"
+  Text search: 10 results in 83ms
+
+▶ Profiling agentgui...
+  Scanned: 83 chunks in 10ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 19 chunks
+  Embeddings: 83/83 chunks in 16535ms
+Searching for: "function class method"
+  Text search: 10 results in 146ms
+
+▶ Profiling seqos...
+  Scanned: 76 chunks in 9ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 12 chunks
+  Embeddings: 76/76 chunks in 13730ms
+Searching for: "function class method"
+  Text search: 10 results in 113ms
+
+▶ Profiling docmcp...
+  Scanned: 74 chunks in 53ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 10 chunks
+  Embeddings: 74/74 chunks in 13526ms
+Searching for: "function class method"
+  Text search: 10 results in 95ms
+
+▶ Profiling zellous...
+  Scanned: 77 chunks in 54ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 13 chunks
+  Embeddings: 77/77 chunks in 13463ms
+Searching for: "function class method"
+  Text search: 10 results in 120ms
+
+▶ Profiling teatree...
+  Scanned: 93 chunks in 10ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 29 chunks
+  Embeddings: 93/93 chunks in 15931ms
+Searching for: "function class method"
+  Text search: 10 results in 82ms
+
+▶ Profiling moonlanding...
+  Scanned: 323 chunks in 35ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 3 chunks
+  Embeddings: 323/323 chunks in 69519ms
+Searching for: "function class method"
+  Text search: 10 results in 672ms
+
+▶ Profiling pp...
+  Scanned: 1041 chunks in 461ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 17 chunks
+  Embeddings: 1041/1041 chunks in 286137ms
+Searching for: "function class method"
+  Text search: 10 results in 1312ms
+
+================================================================================
+PERFORMANCE PROFILE
+================================================================================
+
+scan:
+  Total: 687.90ms
+  Avg:   57.32ms
+  Min:   2.05ms
+  Max:   461.34ms
+  Count: 12
+  Per-chunk: 8.189ms/chunk
+
+embeddings:
+  Total: 451057.24ms
+  Avg:   37588.10ms
+  Min:   1125.83ms
+  Max:   286136.65ms
+  Count: 12
+  Per-chunk: 5369.729ms/chunk
+
+search-text:
+  Total: 2856.57ms
+  Avg:   238.05ms
+  Min:   26.30ms
+  Max:   1311.63ms
+  Count: 12
+  Per-chunk: 34.007ms/chunk
+
+--------------------------------------------------------------------------------
+CODEBASE TIMING
+--------------------------------------------------------------------------------
+
+xbot (7 chunks): 1625ms
+  scan: 8ms (0.5%)
+  embeddings: 1530ms (94.1%)
+  search-text: 88ms (5.4%)
+
+fsbrowse (6 chunks): 1159ms
+  scan: 7ms (0.6%)
+  embeddings: 1126ms (97.1%)
+  search-text: 26ms (2.3%)
+
+agentauth (11 chunks): 1999ms
+  scan: 2ms (0.1%)
+  embeddings: 1957ms (97.9%)
+  search-text: 40ms (2.0%)
+
+webtalk (40 chunks): 6788ms
+  scan: 28ms (0.4%)
+  embeddings: 6680ms (98.4%)
+  search-text: 80ms (1.2%)
+
+plugforge (66 chunks): 11018ms
+  scan: 11ms (0.1%)
+  embeddings: 10923ms (99.1%)
+  search-text: 83ms (0.8%)
+
+agentgui (83 chunks): 16691ms
+  scan: 10ms (0.1%)
+  embeddings: 16535ms (99.1%)
+  search-text: 146ms (0.9%)
+
+seqos (76 chunks): 13852ms
+  scan: 9ms (0.1%)
+  embeddings: 13730ms (99.1%)
+  search-text: 113ms (0.8%)
+
+docmcp (74 chunks): 13674ms
+  scan: 53ms (0.4%)
+  embeddings: 13526ms (98.9%)
+  search-text: 95ms (0.7%)
+
+zellous (77 chunks): 13637ms
+  scan: 54ms (0.4%)
+  embeddings: 13463ms (98.7%)
+  search-text: 120ms (0.9%)
+
+teatree (93 chunks): 16023ms
+  scan: 10ms (0.1%)
+  embeddings: 15931ms (99.4%)
+  search-text: 82ms (0.5%)
+
+moonlanding (323 chunks): 70226ms
+  scan: 35ms (0.0%)
+  embeddings: 69519ms (99.0%)
+  search-text: 672ms (1.0%)
+
+pp (1041 chunks): 287910ms
+  scan: 461ms (0.2%)
+  embeddings: 286137ms (99.4%)
+  search-text: 1312ms (0.5%)

package/profile.js

@@ -0,0 +1,152 @@
+#!/usr/bin/env node
+
+import { existsSync } from 'fs';
+import { loadIgnorePatterns } from './src/ignore-parser.js';
+import { scanRepository } from './src/scanner.js';
+import { buildTextIndex, searchText } from './src/text-search.js';
+
+const CODEBASES = [
+  '~/workspace/agentauth',
+  '~/workspace/agentgui',
+  '~/workspace/docmcp',
+  '~/workspace/friday-staging',
+  '~/workspace/fsbrowse',
+  '~/workspace/gmweb',
+  '~/workspace/hookie',
+  '~/workspace/invoic',
+  '~/workspace/mcp-thorns',
+  '~/workspace/models',
+  '~/workspace/moonlanding',
+  '~/workspace/myworkreview-staging',
+  '~/workspace/opencode-source',
+  '~/workspace/plugforge',
+  '~/workspace/pp',
+  '~/workspace/proxypilot-setup',
+  '~/workspace/seqos',
+  '~/workspace/sttttsmodels',
+  '~/workspace/teatree',
+  '~/workspace/webtalk',
+  '~/workspace/webtalk-repo',
+  '~/workspace/xbot',
+  '~/workspace/zellous',
+];
+
+// Realistic queries that represent actual usage
+const TEST_QUERIES = [
+  'authentication',
+  'database connection',
+  'error handling',
+  'HTTP request',
+  'user session',
+];
+
+class Profiler {
+  constructor() {
+    this.marks = {};
+    this.measurements = [];
+  }
+
+  start(name) {
+    this.marks[name] = performance.now();
+  }
+
+  end(name, extra = null) {
+    if (!this.marks[name]) return 0;
+    const duration = performance.now() - this.marks[name];
+    this.measurements.push({ name, duration, extra });
+    delete this.marks[name];
+    return duration;
+  }
+
+  report(results) {
+    console.log('\n' + '='.repeat(80));
+    console.log('PERFORMANCE SUMMARY');
+    console.log('='.repeat(80));
+
+    const scanTimes = this.measurements.filter(m => m.name.startsWith('scan:'));
+    const indexTimes = this.measurements.filter(m => m.name.startsWith('index:'));
+    const searchTimes = this.measurements.filter(m => m.name.startsWith('search:'));
+
+    const avg = arr => arr.reduce((a, b) => a + b, 0) / arr.length;
+    const fmt = ms => ms.toFixed(1) + 'ms';
+
+    console.log(`\nScan:   avg ${fmt(avg(scanTimes.map(m => m.duration)))}  max ${fmt(Math.max(...scanTimes.map(m => m.duration)))}`);
+    console.log(`Index:  avg ${fmt(avg(indexTimes.map(m => m.duration)))}  max ${fmt(Math.max(...indexTimes.map(m => m.duration)))}`);
+    console.log(`Search: avg ${fmt(avg(searchTimes.map(m => m.duration)))}  max ${fmt(Math.max(...searchTimes.map(m => m.duration)))}`);
+
+    console.log('\n' + '-'.repeat(80));
+    console.log('PER-CODEBASE RESULTS');
+    console.log('-'.repeat(80));
+
+    for (const r of results) {
+      if (r.skipped) {
+        console.log(`\n${r.label}: skipped (not found)`);
+        continue;
+      }
+      const scanT = scanTimes.find(m => m.name === `scan:${r.label}`)?.duration || 0;
+      const indexT = indexTimes.find(m => m.name === `index:${r.label}`)?.duration || 0;
+      console.log(`\n${r.label} (${r.chunks} chunks): scan ${fmt(scanT)} | index ${fmt(indexT)}`);
+      for (const q of r.queries) {
+        const top = q.results[0];
+        const topStr = top ? `${top.file_path}:${top.line_start} (${(top.score * 100).toFixed(0)}%)` : 'no results';
+        console.log(`  "${q.query}" → ${q.count} results in ${fmt(q.time)} | top: ${topStr}`);
+      }
+    }
+  }
+}
+
+async function profileCodebase(codebasePath, profiler) {
+  const rootPath = codebasePath.replace('~', process.env.HOME);
+  const label = codebasePath.split('/').pop();
+
+  if (!existsSync(rootPath)) {
+    console.log(`  ${label}: not found, skipping`);
+    return { label, skipped: true };
+  }
+
+  process.stdout.write(`▶ ${label}... `);
+
+  profiler.start(`scan:${label}`);
+  const ignorePatterns = loadIgnorePatterns(rootPath);
+  const chunks = scanRepository(rootPath, ignorePatterns);
+  profiler.end(`scan:${label}`, chunks.length);
+
+  if (chunks.length === 0) {
+    console.log('0 chunks');
+    return { label, chunks: 0, queries: [] };
+  }
+
+  profiler.start(`index:${label}`);
+  const indexData = buildTextIndex(chunks);
+  profiler.end(`index:${label}`, chunks.length);
+
+  const queryResults = [];
+  for (const query of TEST_QUERIES) {
+    const t0 = performance.now();
+    const results = searchText(query, chunks, indexData);
+    const elapsed = performance.now() - t0;
+    profiler.measurements.push({ name: `search:${label}:${query}`, duration: elapsed });
+    queryResults.push({ query, count: results.length, time: elapsed, results: results.slice(0, 1) });
+  }
+
+  console.log(`${chunks.length} chunks, ${queryResults.map(q => q.time.toFixed(0) + 'ms').join('/')}`);
+  return { label, chunks: chunks.length, queries: queryResults };
+}
+
+async function main() {
+  console.log('Profiling search across codebases...\n');
+
+  const profiler = new Profiler();
+  const results = [];
+
+  for (const codebase of CODEBASES) {
+    results.push(await profileCodebase(codebase, profiler));
+  }
+
+  profiler.report(results);
+}
+
+main().catch(err => {
+  console.error('Profile error:', err.message);
+  process.exit(1);
+});

package/src/ignore-parser.js

@@ -197,6 +197,8 @@ const IGNORED_DIRECTORIES = new Set([
   'node_modules', 'bower_components', 'jspm_packages', 'web_modules',
   // Version control
   '.git', '.svn', '.hg', '.bzr',
+  // Tool config (AI assistants, editors)
+  '.claude', '.cursor', '.aider',
   // IDE
   '.vscode', '.idea', '.vs', '.atom',
   // Build outputs (unambiguous names only)

package/src/search-worker.js

@@ -1,6 +1,6 @@
 import { parentPort } from 'worker_threads';
 import { resolve, relative } from 'path';
-import { existsSync, readFileSync, readdirSync, statSync } from 'fs';
+import { existsSync, readFileSync, statSync } from 'fs';
 import { loadIgnorePatterns } from './ignore-parser.js';
 import { scanRepository } from './scanner.js';
 import { buildTextIndex, searchText } from './text-search.js';
@@ -21,26 +21,22 @@ function findEnclosingContext(content, lineStart) {
 }
 
 function getFileTotalLines(absoluteFilePath) {
+  if (fileLineCountCache.has(absoluteFilePath)) {
+    return fileLineCountCache.get(absoluteFilePath);
+  }
   try {
     const content = readFileSync(absoluteFilePath, 'utf8');
-    return content.split('\n').length;
+    const count = content.split('\n').length;
+    fileLineCountCache.set(absoluteFilePath, count);
+    return count;
   } catch {
     return null;
   }
 }
 
 let indexCache = new Map();
-
-function getWorkspaceFolders(workspacePath) {
-  try {
-    const entries = readdirSync(workspacePath, { withFileTypes: true });
-    return entries
-      .filter(e => e.isDirectory() && !e.name.startsWith('.'))
-      .map(e => resolve(workspacePath, e.name));
-  } catch {
-    return [];
-  }
-}
+// Cache file line counts to avoid repeated disk reads on every search
+const fileLineCountCache = new Map();
 
 async function initializeIndex(repositoryPath) {
   const absolutePath = resolve(repositoryPath);
@@ -87,7 +83,17 @@ async function performSearch(repositoryPath, query) {
       return { error: indexData.error, results: [] };
     }
 
-    const results = searchText(query, indexData.chunks, indexData.indexData);
+    const rawResults = searchText(query, indexData.chunks, indexData.indexData);
+
+    // Deduplicate: keep best-scoring chunk per file, then take top results
+    const bestPerFile = new Map();
+    for (const r of rawResults) {
+      const existing = bestPerFile.get(r.file_path);
+      if (!existing || r.score > existing.score) {
+        bestPerFile.set(r.file_path, r);
+      }
+    }
+    const results = Array.from(bestPerFile.values()).sort((a, b) => b.score - a.score);
 
     return {
       query,
@@ -115,53 +121,6 @@ async function performSearch(repositoryPath, query) {
   }
 }
 
-async function performSearchAll(workspacePaths, query, limit = 10) {
-  const allResults = [];
-
-  for (const repoPath of workspacePaths) {
-    const absolutePath = resolve(repoPath);
-    if (!existsSync(absolutePath)) continue;
-
-    const indexData = await initializeIndex(absolutePath);
-    if (indexData.error || !indexData.chunks) continue;
-
-    const results = searchText(query, indexData.chunks, indexData.indexData);
-    const repoName = absolutePath.split('/').pop();
-
-    const seenFiles = new Set();
-    for (const r of results) {
-      if (!seenFiles.has(r.file_path)) {
-        seenFiles.add(r.file_path);
-        allResults.push({ ...r, repoName, repoPath: absolutePath });
-      }
-      if (seenFiles.size >= limit) break;
-    }
-  }
-
-  allResults.sort((a, b) => b.score - a.score);
-  const top = allResults.slice(0, limit);
-
-  return {
-    query,
-    resultsCount: top.length,
-    results: top.map((r, idx) => {
-      const absoluteFilePath = resolve(r.repoPath, r.file_path);
-      const totalLines = getFileTotalLines(absoluteFilePath);
-      const enclosingContext = findEnclosingContext(r.content, r.line_start);
-      return {
-        rank: idx + 1,
-        absolutePath: absoluteFilePath,
-        relativePath: `${r.repoName}/${r.file_path}`,
-        lines: `${r.line_start}-${r.line_end}`,
-        totalLines,
-        enclosingContext,
-        score: (r.score * 100).toFixed(1),
-        snippet: r.content.split('\n').slice(0, 30).join('\n'),
-      };
-    }),
-  };
-}
-
 if (parentPort) {
   parentPort.on('message', async (msg) => {
     try {
@@ -170,26 +129,6 @@ if (parentPort) {
         return;
       }
 
-      if (msg.type === 'index-all') {
-        const folders = msg.workspacePaths || getWorkspaceFolders(msg.workspacePath || '');
-        let indexed = 0;
-        for (const folder of folders) {
-          if (existsSync(folder)) {
-            await initializeIndex(folder);
-            indexed++;
-          }
-        }
-        parentPort.postMessage({ id: msg.id, result: { indexed, message: `Indexed ${indexed} repositories` } });
-        return;
-      }
-
-      if (msg.type === 'search-all') {
-        const folders = msg.workspacePaths || getWorkspaceFolders(msg.workspacePath || '');
-        const result = await performSearchAll(folders, msg.query, msg.limit || 10);
-        parentPort.postMessage({ id: msg.id, result });
-        return;
-      }
-
       if (msg.type === 'search') {
         const result = await performSearch(msg.repositoryPath || process.cwd(), msg.query);
         parentPort.postMessage({ id: msg.id, result });

package/src/text-search.js

@@ -1,35 +1,72 @@
 export function buildTextIndex(chunks) {
   const index = new Map();
-  const chunkMetadata = [];
-
-  chunks.forEach((chunk, idx) => {
-    const tokens = tokenize(chunk.content);
-    const fileNameTokens = tokenize(chunk.file_path);
-    const symbols = extractSymbols(chunk.content);
-    const frequency = new Map();
-
-    tokens.forEach(token => {
-      frequency.set(token, (frequency.get(token) || 0) + 1);
-      if (!index.has(token)) {
-        index.set(token, new Set());
-      }
-      index.get(token).add(idx);
-    });
+  const chunkMetadata = new Array(chunks.length);
+
+  for (let idx = 0; idx < chunks.length; idx++) {
+    const chunk = chunks[idx];
+    const frequency = tokenizeToFrequency(chunk.content, index, idx);
+    const fileNameTokens = new Set(tokenize(chunk.file_path));
+    const symbols = new Set(extractSymbols(chunk.content));
 
     chunkMetadata[idx] = {
-      tokens,
       fileNameTokens,
       symbols,
       frequency,
       isCode: isCodeFile(chunk.file_path),
+      contentLower: chunk.content.toLowerCase(),
     };
-  });
+  }
+
+  // Precompute IDF for each token: log((N+1)/(df+1))
+  const N = chunks.length;
+  const idf = new Map();
+  for (const [token, docSet] of index) {
+    idf.set(token, Math.log((N + 1) / (docSet.size + 1)) + 1);
+  }
+
+  return { index, chunkMetadata, idf };
+}
+
+function tokenizeToFrequency(text, index, chunkIdx) {
+  const frequency = new Map();
+
+  for (const word of text.split(/\s+/)) {
+    if (word.length === 0) continue;
+
+    const hasUpperCase = word !== word.toLowerCase();
+    if (hasUpperCase) {
+      const camelTokens = word.match(/[A-Z]?[a-z]+|[A-Z]+(?=[A-Z][a-z]|\d|\W|$)|[0-9]+/g);
+      if (camelTokens) {
+        for (const t of camelTokens) {
+          if (t.length > 1) addToken(t.toLowerCase(), frequency, index, chunkIdx);
+        }
+      }
+    }
+
+    const cleaned = word.replace(/[^\w]/g, '').toLowerCase();
+    if (cleaned.length > 1) {
+      addToken(cleaned, frequency, index, chunkIdx);
+      if (word.includes('-') || word.includes('_') || word.includes('.')) {
+        for (const part of word.split(/[-_.]/)) {
+          const partCleaned = part.replace(/[^\w]/g, '').toLowerCase();
+          if (partCleaned.length > 1 && partCleaned !== cleaned) addToken(partCleaned, frequency, index, chunkIdx);
+        }
+      }
+    }
+  }
 
-  return { index, chunkMetadata };
+  return frequency;
+}
+
+function addToken(token, frequency, index, chunkIdx) {
+  frequency.set(token, (frequency.get(token) || 0) + 1);
+  let docSet = index.get(token);
+  if (!docSet) { docSet = new Set(); index.set(token, docSet); }
+  docSet.add(chunkIdx);
 }
 
 export function searchText(query, chunks, indexData) {
-  const { index, chunkMetadata } = indexData;
+  const { index, chunkMetadata, idf } = indexData;
   const queryTokens = tokenize(query);
   const querySymbols = extractSymbols(query);
   const chunkScores = new Map();
@@ -47,36 +84,53 @@ export function searchText(query, chunks, indexData) {
     }
   });
 
-  for (const idx of candidates) {
+  const queryLower = query.toLowerCase();
+
+  let scoringCandidates = candidates;
+  if (candidates.size > 500) {
+    const ranked = Array.from(candidates).sort((a, b) => {
+      let aSum = 0, bSum = 0;
+      for (const token of queryTokens) {
+        if (index.has(token)) {
+          if (index.get(token).has(a)) aSum += idf.get(token) || 1;
+          if (index.get(token).has(b)) bSum += idf.get(token) || 1;
+        }
+      }
+      return bSum - aSum;
+    });
+    scoringCandidates = new Set(ranked.slice(0, 500));
+  }
+
+  for (const idx of scoringCandidates) {
     const chunk = chunks[idx];
     const meta = chunkMetadata[idx];
     let score = 0;
 
-    // Exact phrase match - highest priority (saves embedding cost)
-    if (chunk.content.toLowerCase().includes(query.toLowerCase())) {
+    if (queryTokens.length > 1 && meta.contentLower.includes(queryLower)) {
       score += 30;
     }
 
     // Symbol match in content - function/class named after query terms
     querySymbols.forEach(symbol => {
-      if (meta.symbols.includes(symbol)) score += 10;
+      if (meta.symbols.has(symbol)) score += 10;
     });
 
     // Filename token match - strong signal that this file is about the query topic
     let fileNameMatches = 0;
     queryTokens.forEach(token => {
-      if (meta.fileNameTokens.includes(token)) fileNameMatches++;
+      if (meta.fileNameTokens.has(token)) fileNameMatches++;
     });
     if (fileNameMatches > 0) {
       score += fileNameMatches * 10;
     }
 
-    // Token frequency scoring
+    // TF-IDF scoring: reward rare tokens that appear in this chunk
     queryTokens.forEach(token => {
       if (index.has(token) && index.get(token).has(idx)) {
-        const freq = meta.frequency.get(token) || 1;
+        const tf = Math.min(meta.frequency.get(token) || 1, 5);
+        const tokenIdf = idf ? (idf.get(token) || 1) : 1;
         const lengthBoost = token.length > 4 ? 1.5 : 1;
-        score += lengthBoost * Math.min(freq, 5);
+        score += lengthBoost * tf * tokenIdf;
       }
     });