codebasesearch 0.1.21 → 0.1.23

package/.prd

@@ -0,0 +1,78 @@
+{
+  "project": "code-search",
+  "created": "2026-03-12",
+  "objective": "Profile and improve code-search speed and result quality",
+  "items": [
+    {
+      "id": "fix-dedup-buildtextindex",
+      "subject": "Remove duplicate buildTextIndex from search.js",
+      "status": "pending",
+      "description": "search.js has a private copy of buildTextIndex and tokenize/extractSymbols/isCodeFile that duplicates text-search.js. Import the exported buildTextIndex from text-search.js instead.",
+      "category": "refactor",
+      "effort": "small",
+      "blocking": ["fix-score-normalization", "fix-hybrid-weights"],
+      "blockedBy": []
+    },
+    {
+      "id": "fix-chunk-size",
+      "subject": "Reduce chunk size from 300 to 60 lines for better semantic granularity",
+      "status": "pending",
+      "description": "scanner.js uses 300-line chunks. Embeddings work best on 50-100 line chunks. Reduce to 60-line chunks with 15-line overlap for better vector search quality.",
+      "category": "feature",
+      "effort": "small",
+      "blocking": [],
+      "blockedBy": []
+    },
+    {
+      "id": "fix-score-normalization",
+      "subject": "Fix text search score normalization so top result is always 1.0",
+      "status": "pending",
+      "description": "Text scores divide raw by 100 but scores can exceed 100. Use dynamic max-score scaling. Lower hasGoodTextResults threshold from 0.5 to 0.3.",
+      "category": "bug",
+      "effort": "small",
+      "blocking": [],
+      "blockedBy": ["fix-dedup-buildtextindex"]
+    },
+    {
+      "id": "fix-hybrid-weights",
+      "subject": "Boost text-only exact-match results in hybrid merge",
+      "status": "pending",
+      "description": "Text-only results are capped at 20% weight. Give high-scoring text-only results a floor finalScore of 0.4.",
+      "category": "feature",
+      "effort": "small",
+      "blocking": [],
+      "blockedBy": ["fix-dedup-buildtextindex"]
+    },
+    {
+      "id": "fix-vector-cache-key",
+      "subject": "Strengthen vector search cache key to 20 dimensions",
+      "status": "pending",
+      "description": "Cache key uses only first 5 embedding dims. Use 20 dims for near-zero collision rate.",
+      "category": "bug",
+      "effort": "small",
+      "blocking": [],
+      "blockedBy": []
+    },
+    {
+      "id": "remove-dead-meanpooling",
+      "subject": "Remove dead meanPooling function from embeddings.js",
+      "status": "pending",
+      "description": "meanPooling is defined but never called. Remove dead code.",
+      "category": "refactor",
+      "effort": "small",
+      "blocking": [],
+      "blockedBy": []
+    },
+    {
+      "id": "verify-and-commit",
+      "subject": "Verify improvements and commit all changes",
+      "status": "pending",
+      "description": "Run end-to-end search logic test inline. Commit and push all changes.",
+      "category": "infra",
+      "effort": "small",
+      "blocking": [],
+      "blockedBy": ["fix-dedup-buildtextindex", "fix-chunk-size", "fix-score-normalization", "fix-hybrid-weights", "fix-vector-cache-key", "remove-dead-meanpooling"]
+    }
+  ],
+  "completed": []
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codebasesearch",
-  "version": "0.1.21",
+  "version": "0.1.23",
   "description": "Ultra-simple code search tool with Jina embeddings, LanceDB, and MCP protocol support",
   "type": "module",
   "bin": {

package/src/cli.js

@@ -75,29 +75,39 @@ export async function run(args) {
     console.log('Generating embeddings and indexing...');
 
     // Generate embeddings in batches and upsert immediately to free memory
-    const batchSize = 32;
-    let processedCount = 0;
-
-    for (let i = 0; i < chunks.length; i += batchSize) {
-      const batchChunks = chunks.slice(i, i + batchSize);
-      const batchTexts = batchChunks.map(c => c.content);
-      const batchEmbeddings = await generateEmbeddings(batchTexts);
-
-      // Create batch with embeddings
-      const batchWithEmbeddings = batchChunks.map((chunk, idx) => ({
-        ...chunk,
-        vector: batchEmbeddings[idx]
-      }));
+    // Optimize batch size based on chunk count (larger batches are more efficient)
+    let batchSize = 32;
+    if (chunks.length > 500) batchSize = 64;
+    if (chunks.length > 1000) batchSize = 96;
 
-      // Upsert immediately to free memory
-      await upsertChunks(batchWithEmbeddings);
-      processedCount += batchWithEmbeddings.length;
+    let processedCount = 0;
+    let embeddingsAvailable = true;
+
+    try {
+      for (let i = 0; i < chunks.length; i += batchSize) {
+        const batchChunks = chunks.slice(i, i + batchSize);
+        const batchTexts = batchChunks.map(c => c.content);
+        const batchEmbeddings = await generateEmbeddings(batchTexts);
+
+        // Create batch with embeddings
+        const batchWithEmbeddings = batchChunks.map((chunk, idx) => ({
+          ...chunk,
+          vector: batchEmbeddings[idx]
+        }));
+
+        // Upsert immediately to free memory
+        await upsertChunks(batchWithEmbeddings);
+        processedCount += batchWithEmbeddings.length;
+      }
+    } catch (embeddingError) {
+      console.warn(`Warning: Embedding generation failed (${embeddingError.message}). Using text-only search.\n`);
+      embeddingsAvailable = false;
     }
 
     console.log('Index created\n');
 
-    // Execute search
-    const results = await executeSearch(query);
+    // Execute search with chunks for hybrid search (text-only if embeddings failed)
+    const results = await executeSearch(query, 10, chunks);
 
     // Format and display results
     const output = formatResults(results);

package/src/embeddings.js

@@ -13,6 +13,7 @@ try {
 
 let modelCache = null;
 let cacheCleared = false;
+let modelLoadTime = 0;
 
 function clearModelCache() {
   const cacheDirs = [
@@ -37,6 +38,7 @@ async function getModel(retryOnError = true) {
     return modelCache;
   }
 
+  const modelStart = performance.now();
   console.error('Loading embeddings model (this may take a moment on first run)...');
 
   const modelLoadPromise = pipeline(
@@ -50,6 +52,7 @@ async function getModel(retryOnError = true) {
 
   try {
     modelCache = await Promise.race([modelLoadPromise, timeoutPromise]);
+    modelLoadTime = performance.now() - modelStart;
   } catch (e) {
     if (retryOnError && !cacheCleared && (e.message.includes('Protobuf') || e.message.includes('parsing'))) {
       console.error('Detected corrupted cache, clearing and retrying...');
@@ -65,37 +68,8 @@ async function getModel(retryOnError = true) {
   return modelCache;
 }
 
-async function meanPooling(modelOutput, attentionMask) {
-  // Get token embeddings from model output
-  const tokenEmbeddings = modelOutput.data;
-  const embeddingDim = modelOutput.dims[modelOutput.dims.length - 1];
-  const batchSize = modelOutput.dims[0];
-  const seqLength = modelOutput.dims[1];
-
-  const pooled = [];
-
-  for (let b = 0; b < batchSize; b++) {
-    let sum = new Array(embeddingDim).fill(0);
-    let count = 0;
-
-    for (let s = 0; s < seqLength; s++) {
-      const tokenIdx = b * seqLength + s;
-      const maskValue = attentionMask[tokenIdx] || 1;
-
-      if (maskValue > 0) {
-        const tokenStart = tokenIdx * embeddingDim;
-        for (let d = 0; d < embeddingDim; d++) {
-          sum[d] += tokenEmbeddings[tokenStart + d] * maskValue;
-        }
-        count += maskValue;
-      }
-    }
-
-    const normalized = sum.map(v => v / Math.max(count, 1e-9));
-    pooled.push(normalized);
-  }
-
-  return pooled;
+export function getModelLoadTime() {
+  return modelLoadTime;
 }
 
 export async function generateEmbeddings(texts) {
@@ -105,11 +79,16 @@ export async function generateEmbeddings(texts) {
     texts = [texts];
   }
 
-  // Generate embeddings for all texts
-  const embeddings = await model(texts, {
-    pooling: 'mean',
-    normalize: true
-  });
+  // Generate embeddings for all texts with timeout per batch
+  const embeddings = await Promise.race([
+    model(texts, {
+      pooling: 'mean',
+      normalize: true
+    }),
+    new Promise((_, reject) =>
+      setTimeout(() => reject(new Error('Embedding generation timeout')), 60000)
+    )
+  ]);
 
   // Convert to regular arrays
   const result = [];

package/src/ignore-parser.js

@@ -106,109 +106,17 @@ function loadDefaultIgnores() {
 
 function getHardcodedIgnores() {
   return new Set([
-    'node_modules', '.git', '.svn', '.hg', 'dist', 'build', 'out',
-    'target', 'vendor', '__pycache__', '.pytest_cache', '.mypy_cache',
-    '.next', '.nuxt', '.cache', '.parcel-cache', '.vite', '.turbo',
-    'coverage', '.nyc_output', '.firebase', '.terraform', '.aws',
-    '.azure', '.gcloud', '.vscode', '.idea', '.vs', 'bin', 'obj',
-    '.gradle', '.mvn', 'Pods', 'DerivedData', '.bundle',
-    '.yarn', '.pnp', 'pnpm-lock.yaml', '.pnpm-store',
-    '.tox', '.eggs', '*.egg-info', '.venv', 'venv', 'env',
-    '.tsc', '.eslintcache', '.stylelintcache', '.parcel-cache',
-    'temp', 'tmp', '.tmp', '.DS_Store', 'Thumbs.db',
-    '.swp', '.swo', '*.swp', '*.swo', '.tern-port',
-    'dist-server', 'out-tsc', '.cache', '.parcel-cache',
-    'typings', '.env', '.env.local', '.env.*.local',
-    // JSON files - PRIMARY PRIORITY for memory reduction
-    '*.json', 'package-lock.json', 'yarn.lock', 'pnpm-lock.yaml',
-    'Gemfile.lock', 'poetry.lock', 'Pipfile.lock',
-    // Lock files
-    '*.lock',
-    // Build outputs
-    'public', 'static', 'site', '_site', '.docusaurus', '.gatsby',
-    // Cache/dependency directories
-    '.rush', '.lerna', '.nx',
-    // IDE/editor configs
-    '.cursor', '.replit', '.sublime-project', '.sublime-workspace',
-    '*.iml', '.project', '.classpath', '.settings', '*.sublime-*',
+    // Lock files / package manager artifacts
+    'package-lock.json', 'yarn.lock', 'pnpm-lock.yaml',
+    'Gemfile.lock', 'poetry.lock', 'Pipfile.lock', 'Cargo.lock',
+    'composer.lock', 'go.sum',
     // OS files
-    '.Spotlight-V100', '.Trashes', 'ehthumbs.db', '.fseventsd',
-    '.TemporaryItems', '.AppleDouble', '.LSOverride', 'desktop.ini',
-    // Large data files
-    '*.db', '*.sqlite', '*.sqlite3', '*.bak', '*.dump',
-    '*.backup', '*.data', '*.orig',
-    // Logs and temp
-    '*.log', 'logs', 'npm-debug.log', 'yarn-error.log',
-    // Test coverage and reports
-    'lcov.info', '.coverage', 'test-results',
-    // Database related
-    'storage', 'fixtures',
-    // LLM/Vector related
-    '.llamaindex', '.chroma', '.vectorstore', '.embeddings',
-    '.langchain', '.autogen', '.semantic-kernel', '.openai-cache',
-    '.anthropic-cache', 'embeddings', 'vector-db', 'faiss-index',
-    'chromadb', 'pinecone-cache', 'weaviate-data',
-    // Compiled output
+    '.DS_Store', 'Thumbs.db', 'desktop.ini',
+    // Editor swap files
+    '.tern-port',
+    // Compiled binary artifacts (files, not dirs)
     '*.min.js', '*.min.css', '*.bundle.js', '*.chunk.js', '*.map',
-    // Generated/build artifacts
-    '.assets', 'out-tsc', 'cmake_build_debug', 'cmake_build_release',
-    // Version managers
-    '.rbenv', '.nvm', '.nvmrc',
-    // Ruby specific
-    '*.gem', '*.rbc', '/pkg', '/spec/reports', '/spec/examples.txt',
-    '/test/tmp', '/test/version_tmp', 'lib/bundler/man', '.ruby-version',
-    // Go specific
-    'go.work',
-    // Rust specific
-    'Cargo.lock', '**/*.rs.bk', '*.pdb',
-    // Java specific
-    '*.class', '*.jar', '*.war', '*.ear', '*.nar', '*.nupkg', '*.snupkg',
-    // C# specific
-    '*.suo', '*.user', '*.userosscache', '*.sln.docstates',
-    'project.lock.json', 'project.fragment.lock.json', 'artifacts',
-    // C/C++ specific
-    '*.o', '*.a', '*.so', '*.exe', '*.obj', '*.dll', '*.dylib',
-    'CMakeFiles', 'CMakeCache.txt', '*.cmake',
-    // Swift/Xcode specific
-    '*.xcodeproj', '*.xcworkspace', '*.moved-aside', '*.pbxuser',
-    '*.mode1v3', '*.mode2v3', '*.perspectivev3',
-    // Scala/SBT specific
-    'lib_managed', 'src_managed', 'project/boot', 'project/plugins/project',
-    '.history', '.lib',
-    // PHP specific
-    'composer.lock', '*.phar',
-    // Docker
-    '.dockerignore', 'docker-compose.override.yml', '.docker',
-    // Documentation build
-    'docs/_build', '.vuepress',
-    // Testing frameworks
-    'jest.config', 'vitest.config', 'pytest.ini', 'tox.ini',
-    '__tests__', '__mocks__', 'spec', 'cypress', 'playwright',
-    // Monorepo workspace patterns (implicit through directory coverage)
-    '.turbo', '.nx',
-    // Python package patterns
-    '*.py[cod]', '*$py.class', '.Python', 'pip-log.txt',
-    'pip-delete-this-directory.txt', '.hypothesis', '.pyre', '.pytype',
-    '*.whl',
-    // Config/metadata that are typically low-value
-    '*.config.js', '*.config.ts', 'webpack.config.js', 'rollup.config.js',
-    'vite.config.js', 'tsconfig.json', 'jsconfig.json', 'babel.config',
-    '.babelrc', '.eslintrc', '.prettierrc', '.stylelintrc', '.editorconfig',
-    '*.local', '*.development', '*.production',
-    // Node specific
-    '.npm', '.node_repl_history', '*.tsbuildinfo', 'yarn-error.log',
-    // Documentation/reference files that don't help with search
-    '*.md', '*.txt', '*.rst', '*.adoc', 'docs', 'documentation', 'wiki',
-    'CHANGELOG', 'HISTORY', 'NEWS', 'UPGRADING', 'FAQ', 'CONTRIBUTING',
-    'SECURITY', 'LICENSE', 'LICENCE', 'COPYRIGHT', 'NOTICE', 'AUTHORS',
-    'THIRDPARTY',
-    // Test and coverage files
-    '*.test', '*.spec', 'test', 'tests', 'htmlcov',
-    // Profiling
-    '*.prof', '*.cpuprofile', '*.heapprofile',
-    // Misc
-    '.tern-port', 'firebase-debug.log', 'firestore-debug.log',
-    'ui-debug.log', '.firebaserc', '.stackdump'
+    '*.tsbuildinfo',
   ]);
 }
 
@@ -283,76 +191,54 @@ export function loadIgnorePatterns(rootPath) {
   return merged;
 }
 
-// Directories to always ignore
+// Directories to always ignore - only clear non-source directories
 const IGNORED_DIRECTORIES = new Set([
-  // Dependencies - NEVER include
+  // Dependencies
   'node_modules', 'bower_components', 'jspm_packages', 'web_modules',
   // Version control
-  '.git', '.svn', '.hg', '.bzr', '.vscode', '.idea', '.vs', '.atom', '.sublime-project',
-  // Build outputs - comprehensive list
-  'dist', 'dist-server', 'dist-ssr', 'dist-client', 'dist-server',
-  'build', 'built', 'Build', 'BUILD',
-  'out', 'output', 'Output', 'OUT', 'release', 'Release', 'RELEASE',
-  'target', 'Target', 'TARGET',
-  'bin', 'Bin', 'BIN', 'obj', 'Obj', 'OBJ',
-  'public', 'static', 'assets', 'www', 'wwwroot',
-  'site', '_site', '.site', '.docusaurus', '.gatsby', '.vuepress',
-  'storybook-static', '.nuxt', 'nuxt', '.next', 'next',
-  'out-tsc', 'tsc', '.tsc',
+  '.git', '.svn', '.hg', '.bzr',
+  // IDE
+  '.vscode', '.idea', '.vs', '.atom',
+  // Build outputs (unambiguous names only)
+  'dist', 'dist-server', 'dist-ssr', 'dist-client',
+  'build', 'built',
+  'out', 'out-tsc',
+  'target',
+  'storybook-static', '.docusaurus', '.gatsby', '.vuepress',
+  '.nuxt', '.next',
+  '.tsc',
   // Cache directories
-  '.cache', 'cache', '.parcel-cache', '.vite', 'vite', '.turbo', 'turbo',
+  '.cache', '.parcel-cache', '.vite', '.turbo',
   '.npm', '.yarn', '.pnp', '.pnpm-store', '.rush', '.lerna', '.nx',
   // Testing
   'coverage', '.nyc_output', '.coverage', 'htmlcov', 'test-results',
-  'test', 'tests', 'Test', 'Tests', 'TEST', 'TESTS',
   '__tests__', '__mocks__', '__snapshots__', '__fixtures__',
-  'cypress', 'playwright', 'e2e', 'integration', 'spec', 'specs',
+  'cypress', 'playwright',
   '.tox', '.eggs', '.hypothesis', '.pyre', '.pytype',
   // Python
-  '__pycache__', '.pytest_cache', '.mypy_cache', '.venv', 'venv', 'env',
-  'env.bak', 'venv.bak', '.Python', 'pip-wheel-metadata', '*.egg-info',
+  '__pycache__', '.pytest_cache', '.mypy_cache', '.venv', 'venv',
   // Java/Gradle/Maven
-  '.gradle', '.mvn', 'gradle', 'mvn', '.settings', '.project', '.classpath',
+  '.gradle', '.mvn',
   // iOS/Android
-  'Pods', 'DerivedData', 'build', '.bundle', 'xcuserdata', '.xcodeproj', '.xcworkspace',
+  'Pods', 'DerivedData', '.bundle', 'xcuserdata',
   // Ruby
-  'vendor', '.bundle', '.ruby-version', 'pkg',
-  // Rust
-  'target', 'Cargo.lock',
-  // Go
-  'vendor', 'Godeps',
-  // PHP
-  'vendor', 'composer',
+  '.bundle', 'pkg',
   // Infrastructure
   '.terraform', '.terragrunt-cache', '.pulumi', '.serverless', '.firebase',
-  '.aws', '.azure', '.gcloud', '.vercel', '.netlify', '.now',
-  // Docker
-  '.docker', 'docker', '.dockerignore',
+  '.aws', '.azure', '.gcloud', '.vercel', '.netlify',
   // Temp files
-  'temp', 'tmp', '.tmp', '.temp', 'tmpfs', 'scratch', '.scratch',
-  // Documentation
-  'docs', 'doc', 'documentation', 'wiki', 'guides', 'examples', 'demo', 'demos',
-  'CHANGELOG', 'HISTORY', 'NEWS', 'LICENSE', 'LICENCE', 'COPYING', 'AUTHORS',
-  // IDE/Editor
-  '.vs', '.vscode', '.idea', '.eclipse', '.settings', '.classpath', '.project',
-  // Logs
-  'logs', 'log', '*.log',
-  // Data/Storage
-  'storage', 'data', 'database', 'db', 'fixtures', 'seeds',
-  'uploads', 'files', 'media', 'resources', 'assets', 'images', 'img',
-  // LLM/AI
+  'temp', 'tmp', '.tmp', '.temp',
+  // LLM/AI artifacts
   '.llamaindex', '.chroma', '.vectorstore', '.embeddings',
   '.langchain', '.autogen', '.semantic-kernel', '.openai-cache',
   '.anthropic-cache', 'embeddings', 'vector-db', 'faiss-index',
   'chromadb', 'pinecone-cache', 'weaviate-data',
-  // Package managers
-  '.yarn', '.pnpm', '.npm', '.bun',
-  // Compiled outputs
-  'typings', 'types', '@types', 'type-definitions',
-  // Misc
-  'public', 'static', 'site', '_site',
-  'cmake_build_debug', 'cmake_build_release', 'CMakeFiles', 'CMakeCache.txt',
-  'out-tsc', 'dist-server', 'server', 'client', 'browser', 'esm', 'cjs', 'umd', 'lib', 'es'
+  // Package manager caches
+  '.pnpm', '.bun',
+  // Static/built asset directories
+  'assets', 'static', 'public', 'wwwroot', 'www',
+  // Misc generated
+  'cmake_build_debug', 'cmake_build_release', 'CMakeFiles',
 ]);
 
 export function isCodeFile(filePath) {
@@ -373,28 +259,34 @@ export function isCodeFile(filePath) {
 export function shouldIgnoreDirectory(dirPath) {
   const normalizedPath = dirPath.replace(/\\/g, '/');
   const pathParts = normalizedPath.split('/');
-  
   for (const part of pathParts) {
     if (IGNORED_DIRECTORIES.has(part)) {
       return true;
     }
   }
-  
   return false;
 }
 
-export function shouldIgnore(filePath, ignorePatterns) {
+export function shouldIgnore(filePath, ignorePatterns, isDirectory = false) {
   const normalizedPath = filePath.replace(/\\/g, '/');
   const pathParts = normalizedPath.split('/');
   const fileName = pathParts[pathParts.length - 1];
-  
-  // Check if any directory in path should be ignored
+
+  if (isDirectory) {
+    if (IGNORED_DIRECTORIES.has(fileName)) return true;
+    for (const pattern of ignorePatterns) {
+      if (!pattern.includes('/') && fileName === pattern) return true;
+    }
+    return false;
+  }
+
+  // For files: check all ancestor directories
   for (const part of pathParts.slice(0, -1)) {
     if (IGNORED_DIRECTORIES.has(part)) {
       return true;
     }
   }
-  
+
   // Check if it's a code file using whitelist
   if (!isCodeFile(filePath)) {
     return true;
@@ -402,22 +294,13 @@ export function shouldIgnore(filePath, ignorePatterns) {
 
   // Check against additional ignore patterns
   for (const pattern of ignorePatterns) {
-    // Handle path patterns (contain /)
     if (pattern.includes('/')) {
-      if (normalizedPath.includes(pattern)) {
-        return true;
-      }
-    }
-    // Handle exact file name patterns
-    else if (fileName === pattern) {
+      if (normalizedPath.includes(pattern)) return true;
+    } else if (fileName === pattern) {
       return true;
-    }
-    // Handle directory name patterns (match any path part)
-    else {
+    } else {
       for (const part of pathParts) {
-        if (part === pattern || part.startsWith(pattern + '/')) {
-          return true;
-        }
+        if (part === pattern) return true;
       }
     }
   }

package/src/scanner.js

@@ -1,6 +1,6 @@
 import { readdirSync, statSync, readFileSync } from 'fs';
 import { join, relative } from 'path';
-import { shouldIgnore, isCodeFile } from './ignore-parser.js';
+import { shouldIgnore, shouldIgnoreDirectory, isCodeFile } from './ignore-parser.js';
 
 function getFileExtension(filePath) {
   const lastDot = filePath.lastIndexOf('.');
@@ -33,14 +33,15 @@ function walkDirectory(dirPath, ignorePatterns, relativePath = '') {
       // Normalize to forward slashes for consistent ignore pattern matching
       const normalizedRelPath = relPath.replace(/\\/g, '/');
 
-      // Check if should ignore
-      if (shouldIgnore(normalizedRelPath, ignorePatterns)) {
-        continue;
-      }
-
       if (entry.isDirectory()) {
+        if (shouldIgnoreDirectory(normalizedRelPath) || shouldIgnore(normalizedRelPath, ignorePatterns, true)) {
+          continue;
+        }
         files.push(...walkDirectory(fullPath, ignorePatterns, relPath));
       } else if (entry.isFile()) {
+        if (shouldIgnore(normalizedRelPath, ignorePatterns, false)) {
+          continue;
+        }
         if (isCodeFile(normalizedRelPath) && !isBinaryFile(entry.name)) {
           try {
             const stat = entry.isSymbolicLink ? null : statSync(fullPath);
@@ -64,7 +65,7 @@ function walkDirectory(dirPath, ignorePatterns, relativePath = '') {
   return files;
 }
 
-function chunkContent(content, chunkSize = 1000, overlapSize = 100) {
+function chunkContent(content, chunkSize = 60, overlapSize = 15) {
   const lines = content.split('\n');
   const chunks = [];
 
@@ -80,7 +81,6 @@ function chunkContent(content, chunkSize = 1000, overlapSize = 100) {
       });
     }
 
-    // Stop if we've reached the end
     if (endIdx === lines.length) {
       break;
     }
@@ -99,7 +99,7 @@ export function scanRepository(rootPath, ignorePatterns) {
       const mtime = file.mtime;
 
       // For small files, treat as single chunk
-      if (content.split('\n').length <= 1000) {
+      if (content.split('\n').length <= 60) {
         chunks.push({
           file_path: file.relativePath,
           chunk_index: 0,

package/src/search.js

@@ -1,20 +1,84 @@
 import { generateSingleEmbedding } from './embeddings.js';
 import { searchSimilar } from './store.js';
+import { buildTextIndex, searchText } from './text-search.js';
 
-export async function executeSearch(query, limit = 10) {
+export async function executeSearch(query, limit = 10, allChunks = null, skipVector = false) {
   if (!query || query.trim().length === 0) {
     throw new Error('Query cannot be empty');
   }
 
   console.error(`Searching for: "${query}"`);
 
-  // Generate embedding for query
-  const queryEmbedding = await generateSingleEmbedding(query);
+  try {
+    let vectorResults = [];
+    let textResults = [];
 
-  // Search vector store
-  const results = await searchSimilar(queryEmbedding, limit);
+    if (allChunks && allChunks.length > 0) {
+      const textIndexData = buildTextIndex(allChunks);
+      textResults = searchText(query, allChunks, textIndexData);
+    }
+
+    const hasGoodTextResults = textResults.length > 0 && textResults[0].score > 0.3;
+    if (!skipVector && !hasGoodTextResults) {
+      try {
+        const queryEmbedding = await generateSingleEmbedding(query);
+        vectorResults = await searchSimilar(queryEmbedding, limit * 2);
+      } catch (e) {
+        console.warn(`Vector search unavailable: ${e.message}`);
+      }
+    }
+
+    if (vectorResults.length > 0 && textResults.length > 0) {
+      return mergeSearchResults(vectorResults, textResults.slice(0, limit * 2), limit);
+    }
+
+    const allResults = vectorResults.length > 0 ? vectorResults : textResults;
+    return allResults.slice(0, limit);
+  } catch (error) {
+    console.error('Search error:', error.message);
+    if (allChunks && allChunks.length > 0) {
+      const textIndexData = buildTextIndex(allChunks);
+      const textResults = searchText(query, allChunks, textIndexData);
+      return textResults.slice(0, limit);
+    }
+    throw error;
+  }
+}
+
+function mergeSearchResults(vectorResults, textResults, limit) {
+  const merged = new Map();
+
+  vectorResults.forEach((result) => {
+    const key = `${result.file_path}:${result.chunk_index}`;
+    merged.set(key, {
+      ...result,
+      vectorScore: result.score || 0,
+      textScore: 0,
+      finalScore: (result.score || 0) * 0.8
+    });
+  });
+
+  textResults.forEach((result) => {
+    const key = `${result.file_path}:${result.chunk_index || 0}`;
+    if (merged.has(key)) {
+      const existing = merged.get(key);
+      existing.textScore = result.score || 0;
+      existing.finalScore = (existing.vectorScore * 0.8) + (result.score * 0.2);
+    } else {
+      const textScore = result.score || 0;
+      const finalScore = Math.max(textScore * 0.2, textScore > 0.7 ? 0.4 : 0);
+      merged.set(key, {
+        ...result,
+        vectorScore: 0,
+        textScore,
+        finalScore
+      });
+    }
+  });
 
-  return results;
+  return Array.from(merged.values())
+    .sort((a, b) => b.finalScore - a.finalScore)
+    .slice(0, limit);
 }
 
 export function formatResults(results) {
@@ -27,15 +91,14 @@ export function formatResults(results) {
 
   for (let i = 0; i < results.length; i++) {
     const result = results[i];
-    const match = i + 1;
+    const scoreValue = result.finalScore !== undefined ? result.finalScore : (result.score || 0);
+    const scorePercent = (scoreValue * 100).toFixed(1);
 
-    lines.push(`${match}. ${result.file_path}:${result.line_start}-${result.line_end} (score: ${(result.score * 100).toFixed(1)}%)`);
+    lines.push(`${i + 1}. ${result.file_path}:${result.line_start}-${result.line_end} (score: ${scorePercent}%)`);
 
-    // Show code snippet (first 3 lines)
     const codeLines = result.content.split('\n').slice(0, 3);
     for (const line of codeLines) {
-      const trimmed = line.slice(0, 80); // Limit line length
-      lines.push(`   > ${trimmed}`);
+      lines.push(`   > ${line.slice(0, 80)}`);
     }
 
     lines.push('');

package/src/store.js

@@ -5,6 +5,7 @@ import { mkdirSync, existsSync } from 'fs';
 let dbConnection = null;
 let tableRef = null;
 let isFirstBatch = true;
+let vectorSearchCache = new Map();
 
 export async function initStore(dbPath) {
   // Ensure directory exists
@@ -121,12 +122,19 @@ export async function searchSimilar(queryEmbedding, limit = 10) {
     // Ensure vector is a proper array/tensor
     const query = Array.isArray(queryEmbedding) ? queryEmbedding : Array.from(queryEmbedding);
 
+    // Check cache using 20-dimension hash for near-zero collision rate
+    const cacheKey = query.slice(0, 20).join(',');
+    const cached = vectorSearchCache.get(cacheKey);
+    if (cached) {
+      return cached.slice(0, limit);
+    }
+
     const results = await tableRef
       .search(query)
       .limit(limit)
       .execute();
 
-    return results.map(result => {
+    const formattedResults = results.map(result => {
       const distance = result._distance !== undefined ? result._distance : (result.distance || 0);
       const score = distance !== null && distance !== undefined ? 1 / (1 + distance) : 0;
       return {
@@ -139,6 +147,15 @@ export async function searchSimilar(queryEmbedding, limit = 10) {
         score: score
       };
     });
+
+    // Cache results (keep max 100 cached searches)
+    if (vectorSearchCache.size > 100) {
+      const firstKey = vectorSearchCache.keys().next().value;
+      vectorSearchCache.delete(firstKey);
+    }
+    vectorSearchCache.set(cacheKey, formattedResults);
+
+    return formattedResults;
   } catch (e) {
     console.error('Search failed:', e.message);
     return [];

package/src/text-search.js

@@ -4,19 +4,21 @@ export function buildTextIndex(chunks) {
 
   chunks.forEach((chunk, idx) => {
     const tokens = tokenize(chunk.content);
+    const fileNameTokens = tokenize(chunk.file_path);
     const symbols = extractSymbols(chunk.content);
     const frequency = new Map();
 
     tokens.forEach(token => {
       frequency.set(token, (frequency.get(token) || 0) + 1);
       if (!index.has(token)) {
-        index.set(token, []);
+        index.set(token, new Set());
       }
-      index.get(token).push(idx);
+      index.get(token).add(idx);
     });
 
     chunkMetadata[idx] = {
       tokens,
+      fileNameTokens,
       symbols,
       frequency,
       isCode: isCodeFile(chunk.file_path),
@@ -32,47 +34,66 @@ export function searchText(query, chunks, indexData) {
   const querySymbols = extractSymbols(query);
   const chunkScores = new Map();
 
-  chunks.forEach((chunk, idx) => {
+  // Use index to find candidate chunks efficiently
+  const candidates = new Set();
+  queryTokens.forEach(token => {
+    if (index.has(token)) {
+      for (const idx of index.get(token)) candidates.add(idx);
+    }
+  });
+  querySymbols.forEach(sym => {
+    if (index.has(sym)) {
+      for (const idx of index.get(sym)) candidates.add(idx);
+    }
+  });
+
+  for (const idx of candidates) {
+    const chunk = chunks[idx];
+    const meta = chunkMetadata[idx];
     let score = 0;
 
+    // Exact phrase match - highest priority (saves embedding cost)
+    if (chunk.content.toLowerCase().includes(query.toLowerCase())) {
+      score += 30;
+    }
+
+    // Symbol match in content - function/class named after query terms
+    querySymbols.forEach(symbol => {
+      if (meta.symbols.includes(symbol)) score += 10;
+    });
+
+    // Filename token match - strong signal that this file is about the query topic
+    let fileNameMatches = 0;
     queryTokens.forEach(token => {
-      if (index.has(token)) {
-        if (index.get(token).includes(idx)) {
-          const freq = chunkMetadata[idx].frequency.get(token) || 1;
-          const boost = token.length > 4 ? 1.5 : 1;
-          score += boost * freq;
-        }
-      }
+      if (meta.fileNameTokens.includes(token)) fileNameMatches++;
     });
+    if (fileNameMatches > 0) {
+      score += fileNameMatches * 10;
+    }
 
-    querySymbols.forEach(symbol => {
-      if (chunkMetadata[idx].symbols.includes(symbol)) {
-        score += 5;
+    // Token frequency scoring
+    queryTokens.forEach(token => {
+      if (index.has(token) && index.get(token).has(idx)) {
+        const freq = meta.frequency.get(token) || 1;
+        const lengthBoost = token.length > 4 ? 1.5 : 1;
+        score += lengthBoost * Math.min(freq, 5);
       }
     });
 
-    const exactMatch = chunk.content.includes(query);
-    if (exactMatch) {
-      score += 10;
-    }
+    // Code file boost
+    if (meta.isCode) score *= 1.2;
 
-    if (chunkMetadata[idx].isCode) {
-      score *= 1.2;
-    }
+    if (score > 0) chunkScores.set(idx, score);
+  }
 
-    if (score > 0) {
-      chunkScores.set(idx, score);
-    }
-  });
+  const entries = Array.from(chunkScores.entries()).sort((a, b) => b[1] - a[1]);
+  const maxScore = entries.length > 0 ? entries[0][1] : 1;
 
-  const results = Array.from(chunkScores.entries())
-    .map(([idx, score]) => ({
-      ...chunks[idx],
-      score: Math.min(score / 100, 1),
-      _rawScore: score,
-    }))
-    .filter(r => r.score > 0)
-    .sort((a, b) => b._rawScore - a._rawScore);
+  const results = entries.map(([idx, score]) => ({
+    ...chunks[idx],
+    score: score / maxScore,
+    _rawScore: score,
+  }));
 
   return results;
 }
@@ -80,20 +101,24 @@ export function searchText(query, chunks, indexData) {
 function tokenize(text) {
   const tokens = new Set();
 
-  text.toLowerCase().split(/\s+/).forEach(word => {
+  text.split(/\s+/).forEach(word => {
     if (word.length === 0) return;
 
-    tokens.add(word.replace(/[^\w]/g, ''));
-
-    const camelCaseTokens = word.match(/[a-z]+|[A-Z][a-z]*|[0-9]+/g) || [];
+    // camelCase/PascalCase split BEFORE lowercasing so uppercase boundaries are visible
+    const camelCaseTokens = word.match(/[A-Z]?[a-z]+|[A-Z]+(?=[A-Z][a-z]|\d|\W|$)|[0-9]+/g) || [];
     camelCaseTokens.forEach(t => {
       if (t.length > 1) tokens.add(t.toLowerCase());
     });
 
-    const snakeCaseTokens = word.split(/[-_]/).filter(t => t.length > 0);
-    snakeCaseTokens.forEach(t => {
-      if (t.length > 1) tokens.add(t.toLowerCase());
+    // snake_case and kebab-case split
+    word.split(/[-_.]/).forEach(t => {
+      const cleaned = t.replace(/[^\w]/g, '').toLowerCase();
+      if (cleaned.length > 1) tokens.add(cleaned);
     });
+
+    // Full word lowercased (stripped of punctuation)
+    const cleaned = word.replace(/[^\w]/g, '').toLowerCase();
+    if (cleaned.length > 1) tokens.add(cleaned);
   });
 
   return Array.from(tokens).filter(t => t.length > 1);