codebase-context 1.2.2 → 1.5.1

package/dist/core/search.js

@@ -1,18 +1,72 @@
 /**
  * Hybrid search combining semantic vector search with keyword matching
  */
-import Fuse from "fuse.js";
-import path from "path";
-import { promises as fs } from "fs";
-import { getEmbeddingProvider, } from "../embeddings/index.js";
-import { getStorageProvider } from "../storage/index.js";
-import { analyzerRegistry } from "./analyzer-registry.js";
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import Fuse from 'fuse.js';
+import path from 'path';
+import { promises as fs } from 'fs';
+import { getEmbeddingProvider } from '../embeddings/index.js';
+import { getStorageProvider } from '../storage/index.js';
+import { analyzerRegistry } from './analyzer-registry.js';
+import { IndexCorruptedError } from '../errors/index.js';
+import { isTestingRelatedQuery } from '../preflight/query-scope.js';
+import { assessSearchQuality } from './search-quality.js';
+import { CODEBASE_CONTEXT_DIRNAME, INTELLIGENCE_FILENAME, KEYWORD_INDEX_FILENAME, VECTOR_DB_DIRNAME } from '../constants/codebase-context.js';
 const DEFAULT_SEARCH_OPTIONS = {
     useSemanticSearch: true,
     useKeywordSearch: true,
     semanticWeight: 0.7,
     keywordWeight: 0.3,
+    profile: 'explore',
+    enableQueryExpansion: true,
+    enableLowConfidenceRescue: true,
+    candidateFloor: 30
 };
+const QUERY_EXPANSION_HINTS = [
+    {
+        pattern: /\b(auth|authentication|login|signin|sign-in|session|token|oauth)\b/i,
+        terms: ['auth', 'login', 'token', 'session', 'guard', 'oauth']
+    },
+    {
+        pattern: /\b(route|routes|routing|router|navigate|navigation|redirect|path)\b/i,
+        terms: ['router', 'route', 'navigation', 'redirect', 'path']
+    },
+    {
+        pattern: /\b(config|configuration|configure|setup|register|provider|providers|bootstrap)\b/i,
+        terms: ['config', 'setup', 'register', 'provider', 'bootstrap']
+    },
+    {
+        pattern: /\b(role|roles|permission|permissions|authorization|authorisation|access)\b/i,
+        terms: ['roles', 'permissions', 'access', 'policy', 'guard']
+    },
+    {
+        pattern: /\b(interceptor|middleware|request|response|http)\b/i,
+        terms: ['interceptor', 'middleware', 'http', 'request', 'response']
+    },
+    {
+        pattern: /\b(theme|styles?|styling|palette|color|branding|upload)\b/i,
+        terms: ['theme', 'styles', 'palette', 'color', 'branding', 'upload']
+    }
+];
+const QUERY_STOP_WORDS = new Set([
+    'the',
+    'a',
+    'an',
+    'to',
+    'of',
+    'for',
+    'and',
+    'or',
+    'with',
+    'in',
+    'on',
+    'by',
+    'how',
+    'are',
+    'is',
+    'after',
+    'before'
+]);
 export class CodebaseSearcher {
     rootPath;
     storagePath;
@@ -25,7 +79,7 @@ export class CodebaseSearcher {
     patternIntelligence = null;
     constructor(rootPath) {
         this.rootPath = rootPath;
-        this.storagePath = path.join(rootPath, ".codebase-index");
+        this.storagePath = path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, VECTOR_DB_DIRNAME);
     }
     async initialize() {
         if (this.initialized)
@@ -35,38 +89,41 @@ export class CodebaseSearcher {
             await this.loadPatternIntelligence();
             this.embeddingProvider = await getEmbeddingProvider();
             this.storageProvider = await getStorageProvider({
-                path: this.storagePath,
+                path: this.storagePath
             });
             this.initialized = true;
         }
         catch (error) {
-            console.warn("Partial initialization (keyword search only):", error);
+            if (error instanceof IndexCorruptedError) {
+                throw error; // Propagate to handler for auto-heal
+            }
+            console.warn('Partial initialization (keyword search only):', error);
             this.initialized = true;
         }
     }
     async loadKeywordIndex() {
         try {
-            const indexPath = path.join(this.rootPath, ".codebase-index.json");
-            const content = await fs.readFile(indexPath, "utf-8");
+            const indexPath = path.join(this.rootPath, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME);
+            const content = await fs.readFile(indexPath, 'utf-8');
             this.chunks = JSON.parse(content);
             this.fuseIndex = new Fuse(this.chunks, {
                 keys: [
-                    { name: "content", weight: 0.4 },
-                    { name: "metadata.componentName", weight: 0.25 },
-                    { name: "filePath", weight: 0.15 },
-                    { name: "relativePath", weight: 0.15 },
-                    { name: "componentType", weight: 0.15 },
-                    { name: "layer", weight: 0.1 },
-                    { name: "tags", weight: 0.15 },
+                    { name: 'content', weight: 0.4 },
+                    { name: 'metadata.componentName', weight: 0.25 },
+                    { name: 'filePath', weight: 0.15 },
+                    { name: 'relativePath', weight: 0.15 },
+                    { name: 'componentType', weight: 0.15 },
+                    { name: 'layer', weight: 0.1 },
+                    { name: 'tags', weight: 0.15 }
                 ],
                 includeScore: true,
                 threshold: 0.4,
                 useExtendedSearch: true,
-                ignoreLocation: true,
+                ignoreLocation: true
             });
         }
         catch (error) {
-            console.warn("Keyword index load failed:", error);
+            console.warn('Keyword index load failed:', error);
             this.chunks = [];
             this.fuseIndex = null;
         }
@@ -76,15 +133,15 @@ export class CodebaseSearcher {
      */
     async loadPatternIntelligence() {
         try {
-            const intelligencePath = path.join(this.rootPath, ".codebase-intelligence.json");
-            const content = await fs.readFile(intelligencePath, "utf-8");
+            const intelligencePath = path.join(this.rootPath, CODEBASE_CONTEXT_DIRNAME, INTELLIGENCE_FILENAME);
+            const content = await fs.readFile(intelligencePath, 'utf-8');
             const intelligence = JSON.parse(content);
             const decliningPatterns = new Set();
             const risingPatterns = new Set();
             const patternWarnings = new Map();
             // Extract pattern indicators from intelligence data
             if (intelligence.patterns) {
-                for (const [category, data] of Object.entries(intelligence.patterns)) {
+                for (const [_category, data] of Object.entries(intelligence.patterns)) {
                     const patternData = data;
                     // Track primary pattern
                     if (patternData.primary?.trend === 'Rising') {
@@ -108,7 +165,7 @@ export class CodebaseSearcher {
             console.error(`[search] Loaded pattern intelligence: ${decliningPatterns.size} declining, ${risingPatterns.size} rising patterns`);
         }
         catch (error) {
-            console.warn("Pattern intelligence load failed (will proceed without trend detection):", error);
+            console.warn('Pattern intelligence load failed (will proceed without trend detection):', error);
             this.patternIntelligence = null;
         }
     }
@@ -138,79 +195,129 @@ export class CodebaseSearcher {
         }
         return { trend: 'Stable' };
     }
-    async search(query, limit = 5, filters, options = DEFAULT_SEARCH_OPTIONS) {
-        if (!this.initialized) {
-            await this.initialize();
-        }
-        const { useSemanticSearch, useKeywordSearch, semanticWeight, keywordWeight, } = {
-            ...DEFAULT_SEARCH_OPTIONS,
-            ...options,
-        };
-        const results = new Map();
-        if (useSemanticSearch && this.embeddingProvider && this.storageProvider) {
-            try {
-                const vectorResults = await this.semanticSearch(query, limit * 2, filters);
-                vectorResults.forEach((result) => {
-                    const id = result.chunk.id;
-                    const existing = results.get(id);
-                    if (existing) {
-                        existing.scores.push(result.score * (semanticWeight || 0.7));
-                    }
-                    else {
-                        results.set(id, {
-                            chunk: result.chunk,
-                            scores: [result.score * (semanticWeight || 0.7)],
-                        });
-                    }
-                });
-            }
-            catch (error) {
-                console.warn("Semantic search failed:", error);
+    isTestFile(filePath) {
+        const normalized = filePath.toLowerCase().replace(/\\/g, '/');
+        return (normalized.includes('.spec.') ||
+            normalized.includes('.test.') ||
+            normalized.includes('/e2e/') ||
+            normalized.includes('/__tests__/'));
+    }
+    normalizeQueryTerms(query) {
+        return query
+            .toLowerCase()
+            .split(/[^a-z0-9_]+/)
+            .filter((term) => term.length > 2 && !QUERY_STOP_WORDS.has(term));
+    }
+    buildQueryVariants(query, maxExpansions) {
+        const variants = [{ query, weight: 1 }];
+        if (maxExpansions <= 0)
+            return variants;
+        const normalized = query.toLowerCase();
+        const terms = new Set(this.normalizeQueryTerms(query));
+        for (const hint of QUERY_EXPANSION_HINTS) {
+            if (!hint.pattern.test(query))
+                continue;
+            for (const term of hint.terms) {
+                if (!normalized.includes(term)) {
+                    terms.add(term);
+                }
             }
         }
-        if (useKeywordSearch && this.fuseIndex) {
-            try {
-                const keywordResults = await this.keywordSearch(query, limit * 2, filters);
-                keywordResults.forEach((result) => {
-                    const id = result.chunk.id;
-                    const existing = results.get(id);
-                    if (existing) {
-                        existing.scores.push(result.score * (keywordWeight || 0.3));
-                    }
-                    else {
-                        results.set(id, {
-                            chunk: result.chunk,
-                            scores: [result.score * (keywordWeight || 0.3)],
-                        });
-                    }
-                });
-            }
-            catch (error) {
-                console.warn("Keyword search failed:", error);
+        const addedTerms = Array.from(terms).filter((term) => !normalized.includes(term));
+        if (addedTerms.length === 0)
+            return variants;
+        const firstExpansion = `${query} ${addedTerms.slice(0, 6).join(' ')}`.trim();
+        if (firstExpansion !== query) {
+            variants.push({ query: firstExpansion, weight: 0.35 });
+        }
+        if (maxExpansions > 1 && addedTerms.length > 6) {
+            const secondExpansion = `${query} ${addedTerms.slice(6, 12).join(' ')}`.trim();
+            if (secondExpansion !== query) {
+                variants.push({ query: secondExpansion, weight: 0.25 });
             }
         }
-        const combinedResults = Array.from(results.entries())
-            .map(([id, { chunk, scores }]) => {
+        return variants.slice(0, 1 + maxExpansions);
+    }
+    isCompositionRootFile(filePath) {
+        const normalized = filePath.toLowerCase().replace(/\\/g, '/');
+        const base = path.basename(normalized);
+        if (/^(main|index|bootstrap|startup)\./.test(base))
+            return true;
+        return (normalized.includes('/routes') ||
+            normalized.includes('/routing') ||
+            normalized.includes('/router') ||
+            normalized.includes('/config') ||
+            normalized.includes('/providers'));
+    }
+    queryPathTokenOverlap(filePath, query) {
+        const queryTerms = new Set(this.normalizeQueryTerms(query));
+        if (queryTerms.size === 0)
+            return 0;
+        const pathTerms = this.normalizeQueryTerms(filePath.replace(/\\/g, '/'));
+        return pathTerms.reduce((count, term) => (queryTerms.has(term) ? count + 1 : count), 0);
+    }
+    isLikelyWiringOrFlowQuery(query) {
+        return /\b(route|router|routing|navigate|navigation|redirect|auth|authentication|login|provider|register|config|configuration|interceptor|middleware)\b/i.test(query);
+    }
+    isActionOrHowQuery(query) {
+        return /\b(how|where|configure|configured|setup|register|wire|wiring|navigate|redirect|login|authenticate|copy|upload|handle|create|update|delete)\b/i.test(query);
+    }
+    isDefinitionHeavyResult(chunk) {
+        const normalizedPath = chunk.filePath.toLowerCase().replace(/\\/g, '/');
+        const componentType = (chunk.componentType || '').toLowerCase();
+        if (['type', 'interface', 'enum', 'constant'].includes(componentType))
+            return true;
+        return (normalizedPath.includes('/models/') ||
+            normalizedPath.includes('/interfaces/') ||
+            normalizedPath.includes('/types/') ||
+            normalizedPath.includes('/constants'));
+    }
+    scoreAndSortResults(query, limit, results, profile) {
+        const likelyWiringQuery = this.isLikelyWiringOrFlowQuery(query);
+        const actionQuery = this.isActionOrHowQuery(query);
+        return Array.from(results.entries())
+            .map(([_id, { chunk, scores }]) => {
             // Calculate base combined score
             let combinedScore = scores.reduce((sum, score) => sum + score, 0);
             // Normalize to 0-1 range (scores are already weighted)
             // If both semantic and keyword matched, max possible is ~1.0
             combinedScore = Math.min(1.0, combinedScore);
-            // Boost scores for Angular components with proper detection
-            if (chunk.componentType && chunk.framework === "angular") {
-                combinedScore = Math.min(1.0, combinedScore * 1.3);
+            // Slight boost when analyzer identified a concrete component type
+            if (chunk.componentType && chunk.componentType !== 'unknown') {
+                combinedScore = Math.min(1.0, combinedScore * 1.1);
             }
             // Boost if layer is detected
-            if (chunk.layer && chunk.layer !== "unknown") {
+            if (chunk.layer && chunk.layer !== 'unknown') {
                 combinedScore = Math.min(1.0, combinedScore * 1.1);
             }
+            // Query-aware reranking to reduce noisy matches in practical workflows.
+            if (!isTestingRelatedQuery(query) && this.isTestFile(chunk.filePath)) {
+                combinedScore = combinedScore * 0.75;
+            }
+            if (actionQuery && this.isDefinitionHeavyResult(chunk)) {
+                combinedScore = combinedScore * 0.82;
+            }
+            if (actionQuery &&
+                ['service', 'component', 'interceptor', 'guard', 'module', 'resolver'].includes((chunk.componentType || '').toLowerCase())) {
+                combinedScore = Math.min(1.0, combinedScore * 1.06);
+            }
+            // Light intent-aware boost for likely wiring/configuration queries.
+            if (likelyWiringQuery && profile !== 'explore') {
+                if (this.isCompositionRootFile(chunk.filePath)) {
+                    combinedScore = Math.min(1.0, combinedScore * 1.12);
+                }
+            }
+            const pathOverlap = this.queryPathTokenOverlap(chunk.filePath, query);
+            if (pathOverlap >= 2) {
+                combinedScore = Math.min(1.0, combinedScore * 1.08);
+            }
             // v1.2: Detect pattern trend and apply momentum boost
             const { trend, warning } = this.detectChunkTrend(chunk);
             if (trend === 'Rising') {
                 combinedScore = Math.min(1.0, combinedScore * 1.15); // +15% for modern patterns
             }
             else if (trend === 'Declining') {
-                combinedScore = combinedScore * 0.90; // -10% for legacy patterns
+                combinedScore = combinedScore * 0.9; // -10% for legacy patterns
             }
             const summary = this.generateSummary(chunk);
             const snippet = this.generateSnippet(chunk.content);
@@ -229,29 +336,122 @@ export class CodebaseSearcher {
                 metadata: chunk.metadata,
                 // v1.2: Pattern momentum awareness
                 trend,
-                patternWarning: warning,
+                patternWarning: warning
             };
         })
             .sort((a, b) => b.score - a.score)
             .slice(0, limit);
-        return combinedResults;
+    }
+    pickBetterResultSet(query, primary, rescue) {
+        const primaryQuality = assessSearchQuality(query, primary);
+        const rescueQuality = assessSearchQuality(query, rescue);
+        if (rescueQuality.status === 'ok' &&
+            primaryQuality.status === 'low_confidence' &&
+            rescueQuality.confidence >= primaryQuality.confidence) {
+            return rescue;
+        }
+        if (rescueQuality.confidence >= primaryQuality.confidence + 0.05) {
+            return rescue;
+        }
+        return primary;
+    }
+    async collectHybridMatches(queryVariants, candidateLimit, filters, useSemanticSearch, useKeywordSearch, semanticWeight, keywordWeight) {
+        const results = new Map();
+        if (useSemanticSearch && this.embeddingProvider && this.storageProvider) {
+            try {
+                for (const variant of queryVariants) {
+                    const vectorResults = await this.semanticSearch(variant.query, candidateLimit, filters);
+                    vectorResults.forEach((result) => {
+                        const id = result.chunk.id;
+                        const weightedScore = result.score * semanticWeight * variant.weight;
+                        const existing = results.get(id);
+                        if (existing) {
+                            existing.scores.push(weightedScore);
+                        }
+                        else {
+                            results.set(id, {
+                                chunk: result.chunk,
+                                scores: [weightedScore]
+                            });
+                        }
+                    });
+                }
+            }
+            catch (error) {
+                if (error instanceof IndexCorruptedError) {
+                    throw error; // Propagate to handler for auto-heal
+                }
+                console.warn('Semantic search failed:', error);
+            }
+        }
+        if (useKeywordSearch && this.fuseIndex) {
+            try {
+                for (const variant of queryVariants) {
+                    const keywordResults = await this.keywordSearch(variant.query, candidateLimit, filters);
+                    keywordResults.forEach((result) => {
+                        const id = result.chunk.id;
+                        const weightedScore = result.score * keywordWeight * variant.weight;
+                        const existing = results.get(id);
+                        if (existing) {
+                            existing.scores.push(weightedScore);
+                        }
+                        else {
+                            results.set(id, {
+                                chunk: result.chunk,
+                                scores: [weightedScore]
+                            });
+                        }
+                    });
+                }
+            }
+            catch (error) {
+                console.warn('Keyword search failed:', error);
+            }
+        }
+        return results;
+    }
+    async search(query, limit = 5, filters, options = DEFAULT_SEARCH_OPTIONS) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        const { useSemanticSearch, useKeywordSearch, semanticWeight, keywordWeight, profile, enableQueryExpansion, enableLowConfidenceRescue, candidateFloor } = {
+            ...DEFAULT_SEARCH_OPTIONS,
+            ...options
+        };
+        const candidateLimit = Math.max(limit * 2, candidateFloor || 30);
+        const primaryVariants = this.buildQueryVariants(query, enableQueryExpansion ? 1 : 0);
+        const primaryMatches = await this.collectHybridMatches(primaryVariants, candidateLimit, filters, Boolean(useSemanticSearch), Boolean(useKeywordSearch), semanticWeight || 0.7, keywordWeight || 0.3);
+        const primaryResults = this.scoreAndSortResults(query, limit, primaryMatches, (profile || 'explore'));
+        if (!enableLowConfidenceRescue) {
+            return primaryResults;
+        }
+        const primaryQuality = assessSearchQuality(query, primaryResults);
+        if (primaryQuality.status !== 'low_confidence') {
+            return primaryResults;
+        }
+        const rescueVariants = this.buildQueryVariants(query, 2).slice(1);
+        if (rescueVariants.length === 0) {
+            return primaryResults;
+        }
+        const rescueMatches = await this.collectHybridMatches(rescueVariants.map((variant, index) => ({
+            query: variant.query,
+            weight: index === 0 ? 1 : 0.8
+        })), candidateLimit, filters, Boolean(useSemanticSearch), Boolean(useKeywordSearch), semanticWeight || 0.7, keywordWeight || 0.3);
+        const rescueResults = this.scoreAndSortResults(query, limit, rescueMatches, (profile || 'explore'));
+        return this.pickBetterResultSet(query, primaryResults, rescueResults);
     }
     generateSummary(chunk) {
-        const analyzer = chunk.framework
-            ? analyzerRegistry.get(chunk.framework)
-            : null;
+        const analyzer = chunk.framework ? analyzerRegistry.get(chunk.framework) : null;
         if (analyzer && analyzer.summarize) {
             try {
                 const summary = analyzer.summarize(chunk);
                 // Only use analyzer summary if it's meaningful (not the generic fallback)
-                if (summary &&
-                    !summary.startsWith("Code in ") &&
-                    !summary.includes(": lines ")) {
+                if (summary && !summary.startsWith('Code in ') && !summary.includes(': lines ')) {
                     return summary;
                 }
             }
             catch (error) {
-                console.warn("Analyzer summary failed:", error);
+                console.warn('Analyzer summary failed:', error);
             }
         }
         // Enhanced generic summary
@@ -273,21 +473,21 @@ export class CodebaseSearcher {
         // Last resort: describe the file type
         const ext = path.extname(fileName).slice(1);
         const langMap = {
-            ts: "TypeScript",
-            js: "JavaScript",
-            html: "HTML template",
-            scss: "SCSS styles",
-            css: "CSS styles",
-            json: "JSON config",
+            ts: 'TypeScript',
+            js: 'JavaScript',
+            html: 'HTML template',
+            scss: 'SCSS styles',
+            css: 'CSS styles',
+            json: 'JSON config'
         };
         return `${langMap[ext] || ext.toUpperCase()} in ${fileName}.`;
     }
-    generateSnippet(content, maxLines = 100) {
-        const lines = content.split("\n");
+    generateSnippet(content, maxLines = 20) {
+        const lines = content.split('\n');
         if (lines.length <= maxLines) {
             return content;
         }
-        const snippet = lines.slice(0, maxLines).join("\n");
+        const snippet = lines.slice(0, maxLines).join('\n');
         const remaining = lines.length - maxLines;
         return `${snippet}\n\n... [${remaining} more lines]`;
     }
@@ -299,7 +499,7 @@ export class CodebaseSearcher {
         const results = await this.storageProvider.search(queryVector, limit, filters);
         return results.map((r) => ({
             chunk: r.chunk,
-            score: r.score,
+            score: r.score
         }));
     }
     async keywordSearch(query, limit, filters) {
@@ -310,8 +510,7 @@ export class CodebaseSearcher {
         if (filters) {
             fuseResults = fuseResults.filter((r) => {
                 const chunk = r.item;
-                if (filters.componentType &&
-                    chunk.componentType !== filters.componentType) {
+                if (filters.componentType && chunk.componentType !== filters.componentType) {
                     return false;
                 }
                 if (filters.layer && chunk.layer !== filters.layer) {
@@ -339,14 +538,14 @@ export class CodebaseSearcher {
             const queryLower = query.toLowerCase();
             const fileName = path.basename(chunk.filePath).toLowerCase();
             const relativePathLower = chunk.relativePath.toLowerCase();
-            const componentName = chunk.metadata?.componentName?.toLowerCase() || "";
+            const componentName = chunk.metadata?.componentName?.toLowerCase() || '';
             // Exact class name match
             if (componentName && queryLower === componentName) {
                 score = Math.min(1.0, score + 0.3);
             }
             // Exact file name match
             if (fileName === queryLower ||
-                fileName.replace(/\.ts$/, "") === queryLower.replace(/\.ts$/, "")) {
+                fileName.replace(/\.ts$/, '') === queryLower.replace(/\.ts$/, '')) {
                 score = Math.min(1.0, score + 0.2);
             }
             // File path contains query
@@ -356,7 +555,7 @@ export class CodebaseSearcher {
             }
             return {
                 chunk,
-                score,
+                score
             };
         });
     }
@@ -371,9 +570,9 @@ export class CodebaseSearcher {
         const queryLower = query.toLowerCase();
         const matchingTags = (chunk.tags || []).filter((tag) => queryLower.includes(tag.toLowerCase()));
         if (matchingTags.length > 0) {
-            reasons.push(`tags: ${matchingTags.join(", ")}`);
+            reasons.push(`tags: ${matchingTags.join(', ')}`);
         }
-        return reasons.length > 0 ? reasons.join("; ") : "content match";
+        return reasons.length > 0 ? reasons.join('; ') : 'content match';
     }
     async getChunkCount() {
         if (this.storageProvider) {