codebasesearch 0.1.10 → 0.1.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codebasesearch",
-  "version": "0.1.10",
+  "version": "0.1.11",
   "description": "Ultra-simple code search tool with Jina embeddings, LanceDB, and MCP protocol support",
   "type": "module",
   "bin": {

package/src/search-worker.js

@@ -20,16 +20,16 @@ async function initializeIndex(repositoryPath) {
     const chunks = scanRepository(absolutePath, ignorePatterns);
 
     if (chunks.length === 0) {
-      return { error: 'No code chunks found', chunks: [], index: null };
+      return { error: 'No code chunks found', chunks: [], indexData: null };
     }
 
-    const index = buildTextIndex(chunks);
-    const indexData = { chunks, index };
-    indexCache.set(cacheKey, indexData);
+    const indexData = buildTextIndex(chunks);
+    const result = { chunks, indexData };
+    indexCache.set(cacheKey, result);
 
-    return indexData;
+    return result;
   } catch (error) {
-    return { error: error.message, chunks: [], index: null };
+    return { error: error.message, chunks: [], indexData: null };
   }
 }
 
@@ -47,7 +47,7 @@ async function performSearch(repositoryPath, query) {
       return { error: indexData.error, results: [] };
     }
 
-    const results = searchText(query, indexData.chunks, indexData.index);
+    const results = searchText(query, indexData.chunks, indexData.indexData);
 
     return {
       query,

package/src/text-search.js

@@ -1,48 +1,129 @@
 export function buildTextIndex(chunks) {
   const index = new Map();
+  const chunkMetadata = [];
 
   chunks.forEach((chunk, idx) => {
     const tokens = tokenize(chunk.content);
+    const symbols = extractSymbols(chunk.content);
+    const frequency = new Map();
+
     tokens.forEach(token => {
+      frequency.set(token, (frequency.get(token) || 0) + 1);
       if (!index.has(token)) {
         index.set(token, []);
       }
       index.get(token).push(idx);
     });
+
+    chunkMetadata[idx] = {
+      tokens,
+      symbols,
+      frequency,
+      isCode: isCodeFile(chunk.file_path),
+    };
   });
 
-  return index;
+  return { index, chunkMetadata };
 }
 
-export function searchText(query, chunks, index) {
+export function searchText(query, chunks, indexData) {
+  const { index, chunkMetadata } = indexData;
   const queryTokens = tokenize(query);
+  const querySymbols = extractSymbols(query);
   const chunkScores = new Map();
 
-  queryTokens.forEach(token => {
-    if (index.has(token)) {
-      index.get(token).forEach(chunkIdx => {
-        if (!chunkScores.has(chunkIdx)) {
-          chunkScores.set(chunkIdx, 0);
+  chunks.forEach((chunk, idx) => {
+    let score = 0;
+
+    queryTokens.forEach(token => {
+      if (index.has(token)) {
+        if (index.get(token).includes(idx)) {
+          const freq = chunkMetadata[idx].frequency.get(token) || 1;
+          const boost = token.length > 4 ? 1.5 : 1;
+          score += boost * freq;
         }
-        chunkScores.set(chunkIdx, chunkScores.get(chunkIdx) + 1);
-      });
+      }
+    });
+
+    querySymbols.forEach(symbol => {
+      if (chunkMetadata[idx].symbols.includes(symbol)) {
+        score += 5;
+      }
+    });
+
+    const exactMatch = chunk.content.includes(query);
+    if (exactMatch) {
+      score += 10;
+    }
+
+    if (chunkMetadata[idx].isCode) {
+      score *= 1.2;
+    }
+
+    if (score > 0) {
+      chunkScores.set(idx, score);
     }
   });
 
   const results = Array.from(chunkScores.entries())
     .map(([idx, score]) => ({
       ...chunks[idx],
-      score: score / queryTokens.length,
-      matchCount: score,
+      score: Math.min(score / 100, 1),
+      _rawScore: score,
     }))
     .filter(r => r.score > 0)
-    .sort((a, b) => b.score - a.score);
+    .sort((a, b) => b._rawScore - a._rawScore);
 
   return results;
 }
 
 function tokenize(text) {
-  return text
-    .toLowerCase()
-    .match(/\b\w+\b/g) || [];
+  const tokens = new Set();
+
+  text.toLowerCase().split(/\s+/).forEach(word => {
+    if (word.length === 0) return;
+
+    tokens.add(word.replace(/[^\w]/g, ''));
+
+    const camelCaseTokens = word.match(/[a-z]+|[A-Z][a-z]*|[0-9]+/g) || [];
+    camelCaseTokens.forEach(t => {
+      if (t.length > 1) tokens.add(t.toLowerCase());
+    });
+
+    const snakeCaseTokens = word.split(/[-_]/).filter(t => t.length > 0);
+    snakeCaseTokens.forEach(t => {
+      if (t.length > 1) tokens.add(t.toLowerCase());
+    });
+  });
+
+  return Array.from(tokens).filter(t => t.length > 1);
+}
+
+function extractSymbols(text) {
+  const symbols = new Set();
+
+  const functionMatches = text.match(/(?:async\s+)?function\s+(\w+)|const\s+(\w+)\s*=\s*(?:async\s*)?\(/g) || [];
+  functionMatches.forEach(match => {
+    const name = match.match(/\w+(?=\s*[=\(])/)?.[0];
+    if (name) symbols.add(name.toLowerCase());
+  });
+
+  const classMatches = text.match(/class\s+(\w+)/g) || [];
+  classMatches.forEach(match => {
+    const name = match.match(/\w+$/)?.[0];
+    if (name) symbols.add(name.toLowerCase());
+  });
+
+  const exportMatches = text.match(/export\s+(?:async\s+)?(?:function|class)\s+(\w+)/g) || [];
+  exportMatches.forEach(match => {
+    const name = match.match(/\w+$/)?.[0];
+    if (name) symbols.add(name.toLowerCase());
+  });
+
+  return Array.from(symbols);
+}
+
+function isCodeFile(filePath) {
+  const codeExtensions = ['.js', '.ts', '.jsx', '.tsx', '.py', '.java', '.go', '.rs', '.rb'];
+  return codeExtensions.some(ext => filePath.endsWith(ext));
 }