codebasesearch 0.1.23 → 0.1.25

package/.prd

@@ -1,78 +1 @@
-{
-  "project": "code-search",
-  "created": "2026-03-12",
-  "objective": "Profile and improve code-search speed and result quality",
-  "items": [
-    {
-      "id": "fix-dedup-buildtextindex",
-      "subject": "Remove duplicate buildTextIndex from search.js",
-      "status": "pending",
-      "description": "search.js has a private copy of buildTextIndex and tokenize/extractSymbols/isCodeFile that duplicates text-search.js. Import the exported buildTextIndex from text-search.js instead.",
-      "category": "refactor",
-      "effort": "small",
-      "blocking": ["fix-score-normalization", "fix-hybrid-weights"],
-      "blockedBy": []
-    },
-    {
-      "id": "fix-chunk-size",
-      "subject": "Reduce chunk size from 300 to 60 lines for better semantic granularity",
-      "status": "pending",
-      "description": "scanner.js uses 300-line chunks. Embeddings work best on 50-100 line chunks. Reduce to 60-line chunks with 15-line overlap for better vector search quality.",
-      "category": "feature",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": []
-    },
-    {
-      "id": "fix-score-normalization",
-      "subject": "Fix text search score normalization so top result is always 1.0",
-      "status": "pending",
-      "description": "Text scores divide raw by 100 but scores can exceed 100. Use dynamic max-score scaling. Lower hasGoodTextResults threshold from 0.5 to 0.3.",
-      "category": "bug",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": ["fix-dedup-buildtextindex"]
-    },
-    {
-      "id": "fix-hybrid-weights",
-      "subject": "Boost text-only exact-match results in hybrid merge",
-      "status": "pending",
-      "description": "Text-only results are capped at 20% weight. Give high-scoring text-only results a floor finalScore of 0.4.",
-      "category": "feature",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": ["fix-dedup-buildtextindex"]
-    },
-    {
-      "id": "fix-vector-cache-key",
-      "subject": "Strengthen vector search cache key to 20 dimensions",
-      "status": "pending",
-      "description": "Cache key uses only first 5 embedding dims. Use 20 dims for near-zero collision rate.",
-      "category": "bug",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": []
-    },
-    {
-      "id": "remove-dead-meanpooling",
-      "subject": "Remove dead meanPooling function from embeddings.js",
-      "status": "pending",
-      "description": "meanPooling is defined but never called. Remove dead code.",
-      "category": "refactor",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": []
-    },
-    {
-      "id": "verify-and-commit",
-      "subject": "Verify improvements and commit all changes",
-      "status": "pending",
-      "description": "Run end-to-end search logic test inline. Commit and push all changes.",
-      "category": "infra",
-      "effort": "small",
-      "blocking": [],
-      "blockedBy": ["fix-dedup-buildtextindex", "fix-chunk-size", "fix-score-normalization", "fix-hybrid-weights", "fix-vector-cache-key", "remove-dead-meanpooling"]
-    }
-  ],
-  "completed": []
-}
+{}

package/mcp.js

@@ -1,6 +1,5 @@
 #!/usr/bin/env node
 
-// MUST patch sharp before any other imports
 import fs from 'fs';
 import path from 'path';
 import { fileURLToPath } from 'url';
@@ -26,147 +25,76 @@ import { join } from 'path';
 import { existsSync, readFileSync, appendFileSync, writeFileSync } from 'fs';
 import { supervisor } from './src/supervisor.js';
 
-async function ensureIgnoreEntry(rootPath) {
+function ensureIgnoreEntry(rootPath) {
   const gitignorePath = join(rootPath, '.gitignore');
   const entry = '.code-search/';
-
   try {
     if (existsSync(gitignorePath)) {
       const content = readFileSync(gitignorePath, 'utf8');
-      if (!content.includes(entry)) {
-        appendFileSync(gitignorePath, `\n${entry}`);
-      }
+      if (!content.includes(entry)) appendFileSync(gitignorePath, `\n${entry}`);
     } else {
       writeFileSync(gitignorePath, `${entry}\n`);
     }
-  } catch (e) {
-    // Ignore write errors
-  }
+  } catch (e) {}
 }
 
-const server = new Server(
-  {
-    name: 'code-search-mcp',
-    version: '0.1.0',
-  },
-  {
-    capabilities: {
-      tools: {},
-    },
-  }
-);
+function formatResults(result, query) {
+  if (result.resultsCount === 0) return `No results found for: "${query}"`;
+  const plural = result.resultsCount !== 1 ? 's' : '';
+  const header = `Found ${result.resultsCount} result${plural} for: "${query}"\n\n`;
+  const body = result.results.map((r) => {
+    const pathPart = r.relativePath || r.absolutePath;
+    const lineCount = r.totalLines ? ` [${r.totalLines}L]` : '';
+    const ctx = r.enclosingContext ? ` (in: ${r.enclosingContext})` : '';
+    const rHeader = `${r.rank}. ${pathPart}${lineCount}:${r.lines}${ctx} (score: ${r.score}%)`;
+    const rBody = r.snippet.split('\n').map((line) => `   ${line}`).join('\n');
+    return `${rHeader}\n${rBody}`;
+  }).join('\n\n');
+  return header + body;
+}
+
+function errResponse(msg) {
+  return { content: [{ type: 'text', text: msg }], isError: true };
+}
 
-server.setRequestHandler(ListToolsRequestSchema, async () => {
-  return {
-    tools: [
-      {
-        name: 'search',
-        description:
-          'Search through a code repository using semantic search with Jina embeddings. Automatically indexes the repository before searching.',
-        inputSchema: {
-          type: 'object',
-          properties: {
-            repository_path: {
-              type: 'string',
-              description:
-                'Absolute or relative path to the repository to search in (defaults to current directory)',
-            },
-            query: {
-              type: 'string',
-              description:
-                'Natural language search query (e.g., "authentication middleware", "database connection")',
-            },
-          },
-          required: ['query'],
+function okResponse(text) {
+  return { content: [{ type: 'text', text }] };
+}
+
+const server = new Server({ name: 'code-search-mcp', version: '0.1.0' }, { capabilities: { tools: {} } });
+
+server.setRequestHandler(ListToolsRequestSchema, async () => ({
+  tools: [
+    {
+      name: 'search',
+      description: 'Search through a code repository. Automatically indexes before searching.',
+      inputSchema: {
+        type: 'object',
+        properties: {
+          repository_path: { type: 'string', description: 'Path to repository (defaults to current directory)' },
+          query: { type: 'string', description: 'Natural language search query' },
         },
+        required: ['query'],
       },
-    ],
-  };
-});
+    },
+  ],
+}));
 
 server.setRequestHandler(CallToolRequestSchema, async (request) => {
   const { name, arguments: args } = request.params;
-
-  if (name !== 'search') {
-    return {
-      content: [
-        {
-          type: 'text',
-          text: `Unknown tool: ${name}`,
-        },
-      ],
-      isError: true,
-    };
-  }
-
   const query = args?.query;
-  const repositoryPath = args?.repository_path || cwd();
-
-  if (!query || typeof query !== 'string') {
-    return {
-      content: [
-        {
-          type: 'text',
-          text: 'Error: query is required and must be a string',
-        },
-      ],
-      isError: true,
-    };
-  }
 
-  try {
-    await ensureIgnoreEntry(repositoryPath);
-    const result = await supervisor.sendRequest({
-      type: 'search',
-      query,
-      repositoryPath,
-    });
-
-    if (result.error) {
-      return {
-        content: [
-          {
-            type: 'text',
-            text: `Error: ${result.error}`,
-          },
-        ],
-        isError: true,
-      };
-    }
+  if (name !== 'search') return errResponse(`Unknown tool: ${name}`);
+  if (!query || typeof query !== 'string') return errResponse('Error: query is required and must be a string');
 
-    const repoPath = typeof repositoryPath === 'string' ? repositoryPath : null;
-    const text =
-      result.resultsCount === 0
-        ? `No results found for query: "${query}"`
-        : `Found ${result.resultsCount} result${result.resultsCount !== 1 ? 's' : ''} for query: "${query}"\n\n${result.results
-            .map((r) => {
-              const pathPart = r.relativePath || r.absolutePath;
-              const lineCount = r.totalLines ? ` [${r.totalLines}L]` : '';
-              const ctx = r.enclosingContext ? ` (in: ${r.enclosingContext})` : '';
-              const header = `${r.rank}. ${pathPart}${lineCount}:${r.lines}${ctx} (score: ${r.score}%)`;
-              const body = r.snippet.split('\n').map((line) => `   ${line}`).join('\n');
-              return `${header}\n${body}`;
-            })
-            .join('\n\n')}`;
-
-    return {
-      content: [
-        {
-          type: 'text',
-          text,
-        },
-      ],
-    };
+  try {
+    const repositoryPath = args?.repository_path || cwd();
+    ensureIgnoreEntry(repositoryPath);
+    const result = await supervisor.sendRequest({ type: 'search', query, repositoryPath });
+    if (result.error) return errResponse(`Error: ${result.error}`);
+    return okResponse(formatResults(result, query));
   } catch (error) {
-    return {
-      content: [
-        {
-          type: 'text',
-          text: `Error: ${error.message}`,
-        },
-      ],
-      isError: true,
-    };
+    return errResponse(`Error: ${error.message}`);
   }
 });
 
@@ -176,25 +104,13 @@ export async function startMcpServer() {
 }
 
 const isMain = process.argv[1] && (
-  process.argv[1] === fileURLToPath(import.meta.url) || 
-  process.argv[1].endsWith('mcp.js') || 
+  process.argv[1] === fileURLToPath(import.meta.url) ||
+  process.argv[1].endsWith('mcp.js') ||
   process.argv[1].endsWith('code-search-mcp')
 );
 
-if (isMain) {
-  main().catch((error) => {
-    console.error('Server error:', error);
-  });
-}
-
-process.on('uncaughtException', (error) => {
-  console.error('Uncaught exception:', error);
-});
+process.on('uncaughtException', (error) => console.error('Uncaught exception:', error));
+process.on('unhandledRejection', (reason) => console.error('Unhandled rejection:', reason));
 
-process.on('unhandledRejection', (reason) => {
-  console.error('Unhandled rejection:', reason);
-});
-
-async function main() {
-  await startMcpServer();
-}
+async function main() { await startMcpServer(); }
+if (isMain) main().catch((error) => console.error('Server error:', error));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codebasesearch",
-  "version": "0.1.23",
+  "version": "0.1.25",
   "description": "Ultra-simple code search tool with Jina embeddings, LanceDB, and MCP protocol support",
   "type": "module",
   "bin": {

package/profile-results.txt

@@ -0,0 +1,248 @@
+Performance profiling across codebases...
+This will take several minutes.
+
+
+▶ Profiling xbot...
+  Scanned: 7 chunks in 8ms
+Vector store initialized
+Loading embeddings model (this may take a moment on first run)...
+dtype not specified for "model". Using the default dtype (fp32) for this device (cpu).
+Indexed 7 chunks
+  Embeddings: 7/7 chunks in 1530ms
+Searching for: "function class method"
+  Text search: 7 results in 88ms
+
+▶ Profiling fsbrowse...
+  Scanned: 6 chunks in 7ms
+Vector store initialized
+Indexed 6 chunks
+  Embeddings: 6/6 chunks in 1126ms
+Searching for: "function class method"
+  Text search: 6 results in 26ms
+
+▶ Profiling agentauth...
+  Scanned: 11 chunks in 2ms
+Vector store initialized
+Indexed 11 chunks
+  Embeddings: 11/11 chunks in 1957ms
+Searching for: "function class method"
+  Text search: 10 results in 40ms
+
+▶ Profiling webtalk...
+  Scanned: 40 chunks in 28ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 8 chunks
+  Embeddings: 40/40 chunks in 6680ms
+Searching for: "function class method"
+  Text search: 10 results in 80ms
+
+▶ Profiling plugforge...
+  Scanned: 66 chunks in 11ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 2 chunks
+  Embeddings: 66/66 chunks in 10923ms
+Searching for: "function class method"
+  Text search: 10 results in 83ms
+
+▶ Profiling agentgui...
+  Scanned: 83 chunks in 10ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 19 chunks
+  Embeddings: 83/83 chunks in 16535ms
+Searching for: "function class method"
+  Text search: 10 results in 146ms
+
+▶ Profiling seqos...
+  Scanned: 76 chunks in 9ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 12 chunks
+  Embeddings: 76/76 chunks in 13730ms
+Searching for: "function class method"
+  Text search: 10 results in 113ms
+
+▶ Profiling docmcp...
+  Scanned: 74 chunks in 53ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 10 chunks
+  Embeddings: 74/74 chunks in 13526ms
+Searching for: "function class method"
+  Text search: 10 results in 95ms
+
+▶ Profiling zellous...
+  Scanned: 77 chunks in 54ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 13 chunks
+  Embeddings: 77/77 chunks in 13463ms
+Searching for: "function class method"
+  Text search: 10 results in 120ms
+
+▶ Profiling teatree...
+  Scanned: 93 chunks in 10ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 29 chunks
+  Embeddings: 93/93 chunks in 15931ms
+Searching for: "function class method"
+  Text search: 10 results in 82ms
+
+▶ Profiling moonlanding...
+  Scanned: 323 chunks in 35ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 3 chunks
+  Embeddings: 323/323 chunks in 69519ms
+Searching for: "function class method"
+  Text search: 10 results in 672ms
+
+▶ Profiling pp...
+  Scanned: 1041 chunks in 461ms
+Vector store initialized
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 32 chunks
+Indexed 17 chunks
+  Embeddings: 1041/1041 chunks in 286137ms
+Searching for: "function class method"
+  Text search: 10 results in 1312ms
+
+================================================================================
+PERFORMANCE PROFILE
+================================================================================
+
+scan:
+  Total: 687.90ms
+  Avg:   57.32ms
+  Min:   2.05ms
+  Max:   461.34ms
+  Count: 12
+  Per-chunk: 8.189ms/chunk
+
+embeddings:
+  Total: 451057.24ms
+  Avg:   37588.10ms
+  Min:   1125.83ms
+  Max:   286136.65ms
+  Count: 12
+  Per-chunk: 5369.729ms/chunk
+
+search-text:
+  Total: 2856.57ms
+  Avg:   238.05ms
+  Min:   26.30ms
+  Max:   1311.63ms
+  Count: 12
+  Per-chunk: 34.007ms/chunk
+
+--------------------------------------------------------------------------------
+CODEBASE TIMING
+--------------------------------------------------------------------------------
+
+xbot (7 chunks): 1625ms
+  scan: 8ms (0.5%)
+  embeddings: 1530ms (94.1%)
+  search-text: 88ms (5.4%)
+
+fsbrowse (6 chunks): 1159ms
+  scan: 7ms (0.6%)
+  embeddings: 1126ms (97.1%)
+  search-text: 26ms (2.3%)
+
+agentauth (11 chunks): 1999ms
+  scan: 2ms (0.1%)
+  embeddings: 1957ms (97.9%)
+  search-text: 40ms (2.0%)
+
+webtalk (40 chunks): 6788ms
+  scan: 28ms (0.4%)
+  embeddings: 6680ms (98.4%)
+  search-text: 80ms (1.2%)
+
+plugforge (66 chunks): 11018ms
+  scan: 11ms (0.1%)
+  embeddings: 10923ms (99.1%)
+  search-text: 83ms (0.8%)
+
+agentgui (83 chunks): 16691ms
+  scan: 10ms (0.1%)
+  embeddings: 16535ms (99.1%)
+  search-text: 146ms (0.9%)
+
+seqos (76 chunks): 13852ms
+  scan: 9ms (0.1%)
+  embeddings: 13730ms (99.1%)
+  search-text: 113ms (0.8%)
+
+docmcp (74 chunks): 13674ms
+  scan: 53ms (0.4%)
+  embeddings: 13526ms (98.9%)
+  search-text: 95ms (0.7%)
+
+zellous (77 chunks): 13637ms
+  scan: 54ms (0.4%)
+  embeddings: 13463ms (98.7%)
+  search-text: 120ms (0.9%)
+
+teatree (93 chunks): 16023ms
+  scan: 10ms (0.1%)
+  embeddings: 15931ms (99.4%)
+  search-text: 82ms (0.5%)
+
+moonlanding (323 chunks): 70226ms
+  scan: 35ms (0.0%)
+  embeddings: 69519ms (99.0%)
+  search-text: 672ms (1.0%)
+
+pp (1041 chunks): 287910ms
+  scan: 461ms (0.2%)
+  embeddings: 286137ms (99.4%)
+  search-text: 1312ms (0.5%)

package/profile.js

@@ -0,0 +1,152 @@
+#!/usr/bin/env node
+
+import { existsSync } from 'fs';
+import { loadIgnorePatterns } from './src/ignore-parser.js';
+import { scanRepository } from './src/scanner.js';
+import { buildTextIndex, searchText } from './src/text-search.js';
+
+const CODEBASES = [
+  '~/workspace/agentauth',
+  '~/workspace/agentgui',
+  '~/workspace/docmcp',
+  '~/workspace/friday-staging',
+  '~/workspace/fsbrowse',
+  '~/workspace/gmweb',
+  '~/workspace/hookie',
+  '~/workspace/invoic',
+  '~/workspace/mcp-thorns',
+  '~/workspace/models',
+  '~/workspace/moonlanding',
+  '~/workspace/myworkreview-staging',
+  '~/workspace/opencode-source',
+  '~/workspace/plugforge',
+  '~/workspace/pp',
+  '~/workspace/proxypilot-setup',
+  '~/workspace/seqos',
+  '~/workspace/sttttsmodels',
+  '~/workspace/teatree',
+  '~/workspace/webtalk',
+  '~/workspace/webtalk-repo',
+  '~/workspace/xbot',
+  '~/workspace/zellous',
+];
+
+// Realistic queries that represent actual usage
+const TEST_QUERIES = [
+  'authentication',
+  'database connection',
+  'error handling',
+  'HTTP request',
+  'user session',
+];
+
+class Profiler {
+  constructor() {
+    this.marks = {};
+    this.measurements = [];
+  }
+
+  start(name) {
+    this.marks[name] = performance.now();
+  }
+
+  end(name, extra = null) {
+    if (!this.marks[name]) return 0;
+    const duration = performance.now() - this.marks[name];
+    this.measurements.push({ name, duration, extra });
+    delete this.marks[name];
+    return duration;
+  }
+
+  report(results) {
+    console.log('\n' + '='.repeat(80));
+    console.log('PERFORMANCE SUMMARY');
+    console.log('='.repeat(80));
+
+    const scanTimes = this.measurements.filter(m => m.name.startsWith('scan:'));
+    const indexTimes = this.measurements.filter(m => m.name.startsWith('index:'));
+    const searchTimes = this.measurements.filter(m => m.name.startsWith('search:'));
+
+    const avg = arr => arr.reduce((a, b) => a + b, 0) / arr.length;
+    const fmt = ms => ms.toFixed(1) + 'ms';
+
+    console.log(`\nScan:   avg ${fmt(avg(scanTimes.map(m => m.duration)))}  max ${fmt(Math.max(...scanTimes.map(m => m.duration)))}`);
+    console.log(`Index:  avg ${fmt(avg(indexTimes.map(m => m.duration)))}  max ${fmt(Math.max(...indexTimes.map(m => m.duration)))}`);
+    console.log(`Search: avg ${fmt(avg(searchTimes.map(m => m.duration)))}  max ${fmt(Math.max(...searchTimes.map(m => m.duration)))}`);
+
+    console.log('\n' + '-'.repeat(80));
+    console.log('PER-CODEBASE RESULTS');
+    console.log('-'.repeat(80));
+
+    for (const r of results) {
+      if (r.skipped) {
+        console.log(`\n${r.label}: skipped (not found)`);
+        continue;
+      }
+      const scanT = scanTimes.find(m => m.name === `scan:${r.label}`)?.duration || 0;
+      const indexT = indexTimes.find(m => m.name === `index:${r.label}`)?.duration || 0;
+      console.log(`\n${r.label} (${r.chunks} chunks): scan ${fmt(scanT)} | index ${fmt(indexT)}`);
+      for (const q of r.queries) {
+        const top = q.results[0];
+        const topStr = top ? `${top.file_path}:${top.line_start} (${(top.score * 100).toFixed(0)}%)` : 'no results';
+        console.log(`  "${q.query}" → ${q.count} results in ${fmt(q.time)} | top: ${topStr}`);
+      }
+    }
+  }
+}
+
+async function profileCodebase(codebasePath, profiler) {
+  const rootPath = codebasePath.replace('~', process.env.HOME);
+  const label = codebasePath.split('/').pop();
+
+  if (!existsSync(rootPath)) {
+    console.log(`  ${label}: not found, skipping`);
+    return { label, skipped: true };
+  }
+
+  process.stdout.write(`▶ ${label}... `);
+
+  profiler.start(`scan:${label}`);
+  const ignorePatterns = loadIgnorePatterns(rootPath);
+  const chunks = scanRepository(rootPath, ignorePatterns);
+  profiler.end(`scan:${label}`, chunks.length);
+
+  if (chunks.length === 0) {
+    console.log('0 chunks');
+    return { label, chunks: 0, queries: [] };
+  }
+
+  profiler.start(`index:${label}`);
+  const indexData = buildTextIndex(chunks);
+  profiler.end(`index:${label}`, chunks.length);
+
+  const queryResults = [];
+  for (const query of TEST_QUERIES) {
+    const t0 = performance.now();
+    const results = searchText(query, chunks, indexData);
+    const elapsed = performance.now() - t0;
+    profiler.measurements.push({ name: `search:${label}:${query}`, duration: elapsed });
+    queryResults.push({ query, count: results.length, time: elapsed, results: results.slice(0, 1) });
+  }
+
+  console.log(`${chunks.length} chunks, ${queryResults.map(q => q.time.toFixed(0) + 'ms').join('/')}`);
+  return { label, chunks: chunks.length, queries: queryResults };
+}
+
+async function main() {
+  console.log('Profiling search across codebases...\n');
+
+  const profiler = new Profiler();
+  const results = [];
+
+  for (const codebase of CODEBASES) {
+    results.push(await profileCodebase(codebase, profiler));
+  }
+
+  profiler.report(results);
+}
+
+main().catch(err => {
+  console.error('Profile error:', err.message);
+  process.exit(1);
+});

package/src/ignore-parser.js

@@ -197,6 +197,8 @@ const IGNORED_DIRECTORIES = new Set([
   'node_modules', 'bower_components', 'jspm_packages', 'web_modules',
   // Version control
   '.git', '.svn', '.hg', '.bzr',
+  // Tool config (AI assistants, editors)
+  '.claude', '.cursor', '.aider',
   // IDE
   '.vscode', '.idea', '.vs', '.atom',
   // Build outputs (unambiguous names only)

package/src/search-worker.js

@@ -1,6 +1,6 @@
 import { parentPort } from 'worker_threads';
 import { resolve, relative } from 'path';
-import { existsSync, readFileSync } from 'fs';
+import { existsSync, readFileSync, statSync } from 'fs';
 import { loadIgnorePatterns } from './ignore-parser.js';
 import { scanRepository } from './scanner.js';
 import { buildTextIndex, searchText } from './text-search.js';
@@ -21,22 +21,34 @@ function findEnclosingContext(content, lineStart) {
 }
 
 function getFileTotalLines(absoluteFilePath) {
+  if (fileLineCountCache.has(absoluteFilePath)) {
+    return fileLineCountCache.get(absoluteFilePath);
+  }
   try {
     const content = readFileSync(absoluteFilePath, 'utf8');
-    return content.split('\n').length;
+    const count = content.split('\n').length;
+    fileLineCountCache.set(absoluteFilePath, count);
+    return count;
   } catch {
     return null;
   }
 }
 
 let indexCache = new Map();
+// Cache file line counts to avoid repeated disk reads on every search
+const fileLineCountCache = new Map();
 
 async function initializeIndex(repositoryPath) {
   const absolutePath = resolve(repositoryPath);
-  const cacheKey = absolutePath;
 
-  if (indexCache.has(cacheKey)) {
-    return indexCache.get(cacheKey);
+  const cached = indexCache.get(absolutePath);
+  if (cached) {
+    try {
+      const dirStat = statSync(absolutePath);
+      if (dirStat.mtimeMs <= cached.indexedAt) return cached;
+    } catch {
+      return cached;
+    }
   }
 
   try {
@@ -48,8 +60,8 @@ async function initializeIndex(repositoryPath) {
     }
 
     const indexData = buildTextIndex(chunks);
-    const result = { chunks, indexData };
-    indexCache.set(cacheKey, result);
+    const result = { chunks, indexData, indexedAt: Date.now() };
+    indexCache.set(absolutePath, result);
 
     return result;
   } catch (error) {
@@ -71,7 +83,17 @@ async function performSearch(repositoryPath, query) {
       return { error: indexData.error, results: [] };
     }
 
-    const results = searchText(query, indexData.chunks, indexData.indexData);
+    const rawResults = searchText(query, indexData.chunks, indexData.indexData);
+
+    // Deduplicate: keep best-scoring chunk per file, then take top results
+    const bestPerFile = new Map();
+    for (const r of rawResults) {
+      const existing = bestPerFile.get(r.file_path);
+      if (!existing || r.score > existing.score) {
+        bestPerFile.set(r.file_path, r);
+      }
+    }
+    const results = Array.from(bestPerFile.values()).sort((a, b) => b.score - a.score);
 
     return {
       query,

package/src/text-search.js

@@ -1,35 +1,72 @@
 export function buildTextIndex(chunks) {
   const index = new Map();
-  const chunkMetadata = [];
-
-  chunks.forEach((chunk, idx) => {
-    const tokens = tokenize(chunk.content);
-    const fileNameTokens = tokenize(chunk.file_path);
-    const symbols = extractSymbols(chunk.content);
-    const frequency = new Map();
-
-    tokens.forEach(token => {
-      frequency.set(token, (frequency.get(token) || 0) + 1);
-      if (!index.has(token)) {
-        index.set(token, new Set());
-      }
-      index.get(token).add(idx);
-    });
+  const chunkMetadata = new Array(chunks.length);
+
+  for (let idx = 0; idx < chunks.length; idx++) {
+    const chunk = chunks[idx];
+    const frequency = tokenizeToFrequency(chunk.content, index, idx);
+    const fileNameTokens = new Set(tokenize(chunk.file_path));
+    const symbols = new Set(extractSymbols(chunk.content));
 
     chunkMetadata[idx] = {
-      tokens,
       fileNameTokens,
       symbols,
       frequency,
       isCode: isCodeFile(chunk.file_path),
+      contentLower: chunk.content.toLowerCase(),
     };
-  });
+  }
+
+  // Precompute IDF for each token: log((N+1)/(df+1))
+  const N = chunks.length;
+  const idf = new Map();
+  for (const [token, docSet] of index) {
+    idf.set(token, Math.log((N + 1) / (docSet.size + 1)) + 1);
+  }
+
+  return { index, chunkMetadata, idf };
+}
+
+function tokenizeToFrequency(text, index, chunkIdx) {
+  const frequency = new Map();
+
+  for (const word of text.split(/\s+/)) {
+    if (word.length === 0) continue;
+
+    const hasUpperCase = word !== word.toLowerCase();
+    if (hasUpperCase) {
+      const camelTokens = word.match(/[A-Z]?[a-z]+|[A-Z]+(?=[A-Z][a-z]|\d|\W|$)|[0-9]+/g);
+      if (camelTokens) {
+        for (const t of camelTokens) {
+          if (t.length > 1) addToken(t.toLowerCase(), frequency, index, chunkIdx);
+        }
+      }
+    }
+
+    const cleaned = word.replace(/[^\w]/g, '').toLowerCase();
+    if (cleaned.length > 1) {
+      addToken(cleaned, frequency, index, chunkIdx);
+      if (word.includes('-') || word.includes('_') || word.includes('.')) {
+        for (const part of word.split(/[-_.]/)) {
+          const partCleaned = part.replace(/[^\w]/g, '').toLowerCase();
+          if (partCleaned.length > 1 && partCleaned !== cleaned) addToken(partCleaned, frequency, index, chunkIdx);
+        }
+      }
+    }
+  }
 
-  return { index, chunkMetadata };
+  return frequency;
+}
+
+function addToken(token, frequency, index, chunkIdx) {
+  frequency.set(token, (frequency.get(token) || 0) + 1);
+  let docSet = index.get(token);
+  if (!docSet) { docSet = new Set(); index.set(token, docSet); }
+  docSet.add(chunkIdx);
 }
 
 export function searchText(query, chunks, indexData) {
-  const { index, chunkMetadata } = indexData;
+  const { index, chunkMetadata, idf } = indexData;
   const queryTokens = tokenize(query);
   const querySymbols = extractSymbols(query);
   const chunkScores = new Map();
@@ -47,36 +84,53 @@ export function searchText(query, chunks, indexData) {
     }
   });
 
-  for (const idx of candidates) {
+  const queryLower = query.toLowerCase();
+
+  let scoringCandidates = candidates;
+  if (candidates.size > 500) {
+    const ranked = Array.from(candidates).sort((a, b) => {
+      let aSum = 0, bSum = 0;
+      for (const token of queryTokens) {
+        if (index.has(token)) {
+          if (index.get(token).has(a)) aSum += idf.get(token) || 1;
+          if (index.get(token).has(b)) bSum += idf.get(token) || 1;
+        }
+      }
+      return bSum - aSum;
+    });
+    scoringCandidates = new Set(ranked.slice(0, 500));
+  }
+
+  for (const idx of scoringCandidates) {
     const chunk = chunks[idx];
     const meta = chunkMetadata[idx];
     let score = 0;
 
-    // Exact phrase match - highest priority (saves embedding cost)
-    if (chunk.content.toLowerCase().includes(query.toLowerCase())) {
+    if (queryTokens.length > 1 && meta.contentLower.includes(queryLower)) {
       score += 30;
     }
 
     // Symbol match in content - function/class named after query terms
     querySymbols.forEach(symbol => {
-      if (meta.symbols.includes(symbol)) score += 10;
+      if (meta.symbols.has(symbol)) score += 10;
     });
 
     // Filename token match - strong signal that this file is about the query topic
     let fileNameMatches = 0;
     queryTokens.forEach(token => {
-      if (meta.fileNameTokens.includes(token)) fileNameMatches++;
+      if (meta.fileNameTokens.has(token)) fileNameMatches++;
     });
     if (fileNameMatches > 0) {
       score += fileNameMatches * 10;
     }
 
-    // Token frequency scoring
+    // TF-IDF scoring: reward rare tokens that appear in this chunk
     queryTokens.forEach(token => {
       if (index.has(token) && index.get(token).has(idx)) {
-        const freq = meta.frequency.get(token) || 1;
+        const tf = Math.min(meta.frequency.get(token) || 1, 5);
+        const tokenIdf = idf ? (idf.get(token) || 1) : 1;
         const lengthBoost = token.length > 4 ? 1.5 : 1;
-        score += lengthBoost * Math.min(freq, 5);
+        score += lengthBoost * tf * tokenIdf;
       }
     });