codebase-context 1.2.1 → 1.5.0

package/dist/core/indexer.js

@@ -2,16 +2,18 @@
  * Core Indexer - Orchestrates codebase indexing
  * Scans files, delegates to analyzers, creates embeddings, stores in vector DB
  */
-import { promises as fs } from "fs";
-import path from "path";
-import { glob } from "glob";
-import ignore from "ignore";
-import { analyzerRegistry } from "./analyzer-registry.js";
-import { isCodeFile, isBinaryFile } from "../utils/language-detection.js";
-import { getEmbeddingProvider, } from "../embeddings/index.js";
-import { getStorageProvider, } from "../storage/index.js";
-import { LibraryUsageTracker, PatternDetector, ImportGraph, InternalFileGraph } from "../utils/usage-tracker.js";
-import { getFileCommitDates } from "../utils/git-dates.js";
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { promises as fs } from 'fs';
+import path from 'path';
+import { glob } from 'glob';
+import ignore from 'ignore';
+import { analyzerRegistry } from './analyzer-registry.js';
+import { isCodeFile, isBinaryFile } from '../utils/language-detection.js';
+import { getEmbeddingProvider } from '../embeddings/index.js';
+import { getStorageProvider } from '../storage/index.js';
+import { LibraryUsageTracker, PatternDetector, ImportGraph, InternalFileGraph } from '../utils/usage-tracker.js';
+import { getFileCommitDates } from '../utils/git-dates.js';
+import { CODEBASE_CONTEXT_DIRNAME, INTELLIGENCE_FILENAME, KEYWORD_INDEX_FILENAME, VECTOR_DB_DIRNAME } from '../constants/codebase-context.js';
 export class CodebaseIndexer {
     rootPath;
     config;
@@ -22,13 +24,13 @@ export class CodebaseIndexer {
         this.config = this.mergeConfig(options.config);
         this.onProgressCallback = options.onProgress;
         this.progress = {
-            phase: "initializing",
+            phase: 'initializing',
             percentage: 0,
             filesProcessed: 0,
             totalFiles: 0,
             chunksCreated: 0,
             errors: [],
-            startedAt: new Date(),
+            startedAt: new Date()
         };
     }
     mergeConfig(userConfig) {
@@ -37,44 +39,38 @@ export class CodebaseIndexer {
                 angular: { enabled: true, priority: 100 },
                 react: { enabled: false, priority: 90 },
                 vue: { enabled: false, priority: 90 },
-                generic: { enabled: true, priority: 10 },
+                generic: { enabled: true, priority: 10 }
             },
-            include: ["**/*.{ts,tsx,js,jsx,html,css,scss,sass,less}"],
-            exclude: [
-                "node_modules/**",
-                "dist/**",
-                "build/**",
-                ".git/**",
-                "coverage/**",
-            ],
+            include: ['**/*.{ts,tsx,js,jsx,html,css,scss,sass,less}'],
+            exclude: ['node_modules/**', 'dist/**', 'build/**', '.git/**', 'coverage/**'],
             respectGitignore: true,
             parsing: {
                 maxFileSize: 1048576, // 1MB
                 chunkSize: 100,
                 chunkOverlap: 10,
                 parseTests: true,
-                parseNodeModules: false,
+                parseNodeModules: false
             },
             styleGuides: {
                 autoDetect: true,
-                paths: ["STYLE_GUIDE.md", "docs/style-guide.md", "ARCHITECTURE.md"],
-                parseMarkdown: true,
+                paths: ['STYLE_GUIDE.md', 'docs/style-guide.md', 'ARCHITECTURE.md'],
+                parseMarkdown: true
             },
             documentation: {
                 autoDetect: true,
                 includeReadmes: true,
-                includeChangelogs: false,
+                includeChangelogs: false
             },
             embedding: {
-                provider: "transformers",
-                model: "Xenova/bge-base-en-v1.5",
-                batchSize: 100,
+                provider: 'transformers',
+                model: 'Xenova/bge-small-en-v1.5',
+                batchSize: 100
             },
             skipEmbedding: false,
             storage: {
-                provider: "lancedb",
-                path: "./codebase-index",
-            },
+                provider: 'lancedb',
+                path: './codebase-index'
+            }
         };
         return {
             ...defaultConfig,
@@ -84,10 +80,10 @@ export class CodebaseIndexer {
             styleGuides: { ...defaultConfig.styleGuides, ...userConfig?.styleGuides },
             documentation: {
                 ...defaultConfig.documentation,
-                ...userConfig?.documentation,
+                ...userConfig?.documentation
             },
             embedding: { ...defaultConfig.embedding, ...userConfig?.embedding },
-            storage: { ...defaultConfig.storage, ...userConfig?.storage },
+            storage: { ...defaultConfig.storage, ...userConfig?.storage }
         };
     }
     async index() {
@@ -110,14 +106,14 @@ export class CodebaseIndexer {
                 shared: 0,
                 feature: 0,
                 infrastructure: 0,
-                unknown: 0,
+                unknown: 0
             },
             errors: [],
-            startedAt: new Date(),
+            startedAt: new Date()
         };
         try {
             // Phase 1: Scanning
-            this.updateProgress("scanning", 0);
+            this.updateProgress('scanning', 0);
             let files = await this.scanFiles();
             // Memory safety: limit total files to prevent heap exhaustion
             const MAX_FILES = 10000;
@@ -130,7 +126,7 @@ export class CodebaseIndexer {
             this.progress.totalFiles = files.length;
             console.error(`Found ${files.length} files to index`);
             // Phase 2: Analyzing & Parsing
-            this.updateProgress("analyzing", 0);
+            this.updateProgress('analyzing', 0);
             const allChunks = [];
             const libraryTracker = new LibraryUsageTracker();
             const patternDetector = new PatternDetector();
@@ -145,13 +141,13 @@ export class CodebaseIndexer {
                 this.progress.percentage = Math.round(((i + 1) / files.length) * 100);
                 try {
                     // Normalize line endings to \n for consistent cross-platform output
-                    const rawContent = await fs.readFile(file, "utf-8");
-                    const content = rawContent.replace(/\r\n/g, "\n");
+                    const rawContent = await fs.readFile(file, 'utf-8');
+                    const content = rawContent.replace(/\r\n/g, '\n');
                     const result = await analyzerRegistry.analyzeFile(file, content);
                     if (result) {
                         allChunks.push(...result.chunks);
                         stats.indexedFiles++;
-                        stats.totalLines += content.split("\n").length;
+                        stats.totalLines += content.split('\n').length;
                         // Track library usage AND import graph from imports
                         for (const imp of result.imports) {
                             libraryTracker.track(imp.source, file);
@@ -176,9 +172,9 @@ export class CodebaseIndexer {
                         }
                         // Track exports for unused export detection
                         if (result.exports && result.exports.length > 0) {
-                            const fileExports = result.exports.map(exp => ({
+                            const fileExports = result.exports.map((exp) => ({
                                 name: exp.name,
-                                type: exp.isDefault ? 'default' : exp.type || 'other',
+                                type: exp.isDefault ? 'default' : exp.type || 'other'
                             }));
                             internalFileGraph.trackExports(file, fileExports);
                         }
@@ -214,12 +210,12 @@ export class CodebaseIndexer {
                         // Track file for Golden File scoring (framework-agnostic based on patterns)
                         const detectedPatterns = result.metadata?.detectedPatterns || [];
                         const hasPattern = (category, name) => detectedPatterns.some((p) => p.category === category && p.name === name);
-                        const patternScore = ((hasPattern('dependencyInjection', 'inject() function') ? 1 : 0) +
+                        const patternScore = (hasPattern('dependencyInjection', 'inject() function') ? 1 : 0) +
                             (hasPattern('stateManagement', 'Signals') ? 1 : 0) +
                             (hasPattern('reactivity', 'Computed') ? 1 : 0) +
                             (hasPattern('reactivity', 'Effect') ? 1 : 0) +
                             (hasPattern('componentStyle', 'Standalone') ? 1 : 0) +
-                            (hasPattern('componentInputs', 'Signal-based inputs') ? 1 : 0));
+                            (hasPattern('componentInputs', 'Signal-based inputs') ? 1 : 0);
                         if (patternScore >= 3) {
                             patternDetector.trackGoldenFile(relPath, patternScore, {
                                 inject: hasPattern('dependencyInjection', 'inject() function'),
@@ -227,7 +223,7 @@ export class CodebaseIndexer {
                                 computed: hasPattern('reactivity', 'Computed'),
                                 effect: hasPattern('reactivity', 'Effect'),
                                 standalone: hasPattern('componentStyle', 'Standalone'),
-                                signalInputs: hasPattern('componentInputs', 'Signal-based inputs'),
+                                signalInputs: hasPattern('componentInputs', 'Signal-based inputs')
                             });
                         }
                         // Update component statistics
@@ -250,8 +246,8 @@ export class CodebaseIndexer {
                     stats.errors.push({
                         filePath: file,
                         error: error instanceof Error ? error.message : String(error),
-                        phase: "analyzing",
-                        timestamp: new Date(),
+                        phase: 'analyzing',
+                        timestamp: new Date()
                     });
                 }
                 if (this.onProgressCallback) {
@@ -261,8 +257,7 @@ export class CodebaseIndexer {
             stats.totalChunks = allChunks.length;
             stats.avgChunkSize =
                 allChunks.length > 0
-                    ? Math.round(allChunks.reduce((sum, c) => sum + c.content.length, 0) /
-                        allChunks.length)
+                    ? Math.round(allChunks.reduce((sum, c) => sum + c.content.length, 0) / allChunks.length)
                     : 0;
             // Memory safety: limit chunks to prevent embedding memory issues
             const MAX_CHUNKS = 5000;
@@ -272,9 +267,9 @@ export class CodebaseIndexer {
                 chunksToEmbed = allChunks.slice(0, MAX_CHUNKS);
             }
             // Phase 3: Embedding
-            let chunksWithEmbeddings = [];
+            const chunksWithEmbeddings = [];
             if (!this.config.skipEmbedding) {
-                this.updateProgress("embedding", 50);
+                this.updateProgress('embedding', 50);
                 console.error(`Creating embeddings for ${chunksToEmbed.length} chunks...`);
                 // Initialize embedding provider
                 const embeddingProvider = await getEmbeddingProvider(this.config.embedding);
@@ -291,56 +286,57 @@ export class CodebaseIndexer {
                         if (chunk.componentType) {
                             parts.unshift(`Type: ${chunk.componentType}`);
                         }
-                        return parts.join("\n");
+                        return parts.join('\n');
                     });
                     const embeddings = await embeddingProvider.embedBatch(texts);
                     for (let j = 0; j < batch.length; j++) {
                         chunksWithEmbeddings.push({
                             ...batch[j],
-                            embedding: embeddings[j],
+                            embedding: embeddings[j]
                         });
                     }
                     // Update progress
                     const embeddingProgress = 50 + Math.round((i / chunksToEmbed.length) * 25);
-                    this.updateProgress("embedding", embeddingProgress);
-                    if ((i + batchSize) % 100 === 0 ||
-                        i + batchSize >= chunksToEmbed.length) {
+                    this.updateProgress('embedding', embeddingProgress);
+                    if ((i + batchSize) % 100 === 0 || i + batchSize >= chunksToEmbed.length) {
                         console.error(`Embedded ${Math.min(i + batchSize, chunksToEmbed.length)}/${chunksToEmbed.length} chunks`);
                     }
                 }
             }
             else {
-                console.error("Skipping embedding generation (skipEmbedding=true)");
+                console.error('Skipping embedding generation (skipEmbedding=true)');
             }
             // Phase 4: Storing
-            this.updateProgress("storing", 75);
+            this.updateProgress('storing', 75);
+            const contextDir = path.join(this.rootPath, CODEBASE_CONTEXT_DIRNAME);
+            await fs.mkdir(contextDir, { recursive: true });
             if (!this.config.skipEmbedding) {
                 console.error(`Storing ${chunksToEmbed.length} chunks...`);
                 // Store in LanceDB for vector search
-                const storagePath = path.join(this.rootPath, ".codebase-index");
+                const storagePath = path.join(contextDir, VECTOR_DB_DIRNAME);
                 const storageProvider = await getStorageProvider({ path: storagePath });
                 await storageProvider.clear(); // Clear existing index
                 await storageProvider.store(chunksWithEmbeddings);
             }
             // Also save JSON for keyword search (Fuse.js) - use chunksToEmbed for consistency
-            const indexPath = path.join(this.rootPath, ".codebase-index.json");
+            const indexPath = path.join(contextDir, KEYWORD_INDEX_FILENAME);
             // Write without pretty-printing to save memory
             await fs.writeFile(indexPath, JSON.stringify(chunksToEmbed));
             // Save library usage and pattern stats
-            const intelligencePath = path.join(this.rootPath, ".codebase-intelligence.json");
+            const intelligencePath = path.join(contextDir, INTELLIGENCE_FILENAME);
             const libraryStats = libraryTracker.getStats();
             // Extract tsconfig paths for AI to understand import aliases
             let tsconfigPaths;
             try {
-                const tsconfigPath = path.join(this.rootPath, "tsconfig.json");
-                const tsconfigContent = await fs.readFile(tsconfigPath, "utf-8");
+                const tsconfigPath = path.join(this.rootPath, 'tsconfig.json');
+                const tsconfigContent = await fs.readFile(tsconfigPath, 'utf-8');
                 const tsconfig = JSON.parse(tsconfigContent);
                 if (tsconfig.compilerOptions?.paths) {
                     tsconfigPaths = tsconfig.compilerOptions.paths;
                     console.error(`Found ${Object.keys(tsconfigPaths).length} path aliases in tsconfig.json`);
                 }
             }
-            catch (error) {
+            catch (_error) {
                 // No tsconfig.json or no paths defined
             }
             const intelligence = {
@@ -352,15 +348,15 @@ export class CodebaseIndexer {
                 tsconfigPaths,
                 importGraph: {
                     usages: importGraph.getAllUsages(),
-                    topUsed: importGraph.getTopUsed(30),
+                    topUsed: importGraph.getTopUsed(30)
                 },
                 // Internal file graph for circular dependency and unused export detection
                 internalFileGraph: internalFileGraph.toJSON(),
-                generatedAt: new Date().toISOString(),
+                generatedAt: new Date().toISOString()
             };
             await fs.writeFile(intelligencePath, JSON.stringify(intelligence, null, 2));
             // Phase 5: Complete
-            this.updateProgress("complete", 100);
+            this.updateProgress('complete', 100);
             stats.duration = Date.now() - startTime;
             stats.completedAt = new Date();
             console.error(`Indexing complete in ${stats.duration}ms`);
@@ -368,12 +364,12 @@ export class CodebaseIndexer {
             return stats;
         }
         catch (error) {
-            this.progress.phase = "error";
+            this.progress.phase = 'error';
             stats.errors.push({
                 filePath: this.rootPath,
                 error: error instanceof Error ? error.message : String(error),
                 phase: this.progress.phase,
-                timestamp: new Date(),
+                timestamp: new Date()
             });
             throw error;
         }
@@ -384,23 +380,23 @@ export class CodebaseIndexer {
         let ig = null;
         if (this.config.respectGitignore) {
             try {
-                const gitignorePath = path.join(this.rootPath, ".gitignore");
-                const gitignoreContent = await fs.readFile(gitignorePath, "utf-8");
+                const gitignorePath = path.join(this.rootPath, '.gitignore');
+                const gitignoreContent = await fs.readFile(gitignorePath, 'utf-8');
                 ig = ignore.default().add(gitignoreContent);
             }
-            catch (error) {
+            catch (_error) {
                 // No .gitignore or couldn't read it
             }
         }
         // Scan with glob
-        const includePatterns = this.config.include || ["**/*"];
+        const includePatterns = this.config.include || ['**/*'];
         const excludePatterns = this.config.exclude || [];
         for (const pattern of includePatterns) {
             const matches = await glob(pattern, {
                 cwd: this.rootPath,
                 absolute: true,
                 ignore: excludePatterns,
-                nodir: true,
+                nodir: true
             });
             for (const file of matches) {
                 const relativePath = path.relative(this.rootPath, file);
@@ -420,7 +416,7 @@ export class CodebaseIndexer {
                         continue;
                     }
                 }
-                catch (error) {
+                catch (_error) {
                     continue;
                 }
                 files.push(file);
@@ -436,76 +432,147 @@ export class CodebaseIndexer {
         }
     }
     async detectMetadata() {
-        // Try to use the most specific analyzer for metadata detection
-        const primaryAnalyzer = analyzerRegistry.getAll()[0]; // Highest priority
-        let metadata;
-        if (primaryAnalyzer) {
-            metadata = await primaryAnalyzer.detectCodebaseMetadata(this.rootPath);
-        }
-        else {
-            // Fallback metadata
-            metadata = {
-                name: path.basename(this.rootPath),
-                rootPath: this.rootPath,
-                languages: [],
-                dependencies: [],
-                architecture: {
-                    type: "mixed",
-                    layers: {
-                        presentation: 0,
-                        business: 0,
-                        data: 0,
-                        state: 0,
-                        core: 0,
-                        shared: 0,
-                        feature: 0,
-                        infrastructure: 0,
-                        unknown: 0,
-                    },
-                    patterns: [],
-                },
-                styleGuides: [],
-                documentation: [],
-                projectStructure: {
-                    type: "single-app",
+        // Get all registered analyzers (sorted by priority, highest first)
+        const analyzers = analyzerRegistry.getAll();
+        // Start with base metadata template
+        let metadata = {
+            name: path.basename(this.rootPath),
+            rootPath: this.rootPath,
+            languages: [],
+            dependencies: [],
+            architecture: {
+                type: 'mixed',
+                layers: {
+                    presentation: 0,
+                    business: 0,
+                    data: 0,
+                    state: 0,
+                    core: 0,
+                    shared: 0,
+                    feature: 0,
+                    infrastructure: 0,
+                    unknown: 0
                 },
-                statistics: {
-                    totalFiles: 0,
-                    totalLines: 0,
-                    totalComponents: 0,
-                    componentsByType: {},
-                    componentsByLayer: {
-                        presentation: 0,
-                        business: 0,
-                        data: 0,
-                        state: 0,
-                        core: 0,
-                        shared: 0,
-                        feature: 0,
-                        infrastructure: 0,
-                        unknown: 0,
-                    },
-                },
-                customMetadata: {},
-            };
+                patterns: []
+            },
+            styleGuides: [],
+            documentation: [],
+            projectStructure: {
+                type: 'single-app'
+            },
+            statistics: {
+                totalFiles: 0,
+                totalLines: 0,
+                totalComponents: 0,
+                componentsByType: {},
+                componentsByLayer: {
+                    presentation: 0,
+                    business: 0,
+                    data: 0,
+                    state: 0,
+                    core: 0,
+                    shared: 0,
+                    feature: 0,
+                    infrastructure: 0,
+                    unknown: 0
+                }
+            },
+            customMetadata: {}
+        };
+        // Loop through all analyzers (highest priority first) and merge their metadata
+        // Higher priority analyzers' values win on conflicts
+        for (const analyzer of analyzers) {
+            try {
+                const analyzerMeta = await analyzer.detectCodebaseMetadata(this.rootPath);
+                metadata = this.mergeMetadata(metadata, analyzerMeta);
+            }
+            catch (error) {
+                // Analyzer failed, continue with next
+                console.warn(`Analyzer ${analyzer.name} failed to detect metadata:`, error);
+            }
         }
         // Load intelligence data if available
         try {
-            const intelligencePath = path.join(this.rootPath, ".codebase-intelligence.json");
-            const intelligenceContent = await fs.readFile(intelligencePath, "utf-8");
+            const intelligencePath = path.join(this.rootPath, CODEBASE_CONTEXT_DIRNAME, INTELLIGENCE_FILENAME);
+            const intelligenceContent = await fs.readFile(intelligencePath, 'utf-8');
             const intelligence = JSON.parse(intelligenceContent);
             metadata.customMetadata = {
                 ...metadata.customMetadata,
                 libraryUsage: intelligence.libraryUsage,
                 patterns: intelligence.patterns,
-                intelligenceGeneratedAt: intelligence.generatedAt,
+                intelligenceGeneratedAt: intelligence.generatedAt
             };
         }
-        catch (error) {
+        catch (_error) {
             // Intelligence file doesn't exist yet (indexing not run)
         }
         return metadata;
     }
+    /**
+     * Merge two CodebaseMetadata objects.
+     * The 'incoming' metadata takes precedence for non-empty values.
+     */
+    mergeMetadata(base, incoming) {
+        return {
+            name: incoming.name || base.name,
+            rootPath: incoming.rootPath || base.rootPath,
+            languages: [...new Set([...base.languages, ...incoming.languages])], // Merge and deduplicate
+            dependencies: this.mergeDependencies(base.dependencies, incoming.dependencies),
+            framework: incoming.framework || base.framework, // Framework from higher priority analyzer wins
+            architecture: {
+                type: incoming.architecture?.type || base.architecture.type,
+                layers: this.mergeLayers(base.architecture.layers, incoming.architecture?.layers),
+                patterns: [
+                    ...new Set([
+                        ...(base.architecture.patterns || []),
+                        ...(incoming.architecture?.patterns || [])
+                    ])
+                ] // Merge and deduplicate
+            },
+            styleGuides: [...new Set([...base.styleGuides, ...incoming.styleGuides])], // Merge and deduplicate
+            documentation: [...new Set([...base.documentation, ...incoming.documentation])], // Merge and deduplicate
+            projectStructure: incoming.projectStructure?.type !== 'single-app'
+                ? incoming.projectStructure
+                : base.projectStructure,
+            statistics: this.mergeStatistics(base.statistics, incoming.statistics),
+            customMetadata: { ...base.customMetadata, ...incoming.customMetadata }
+        };
+    }
+    mergeDependencies(base, incoming) {
+        const seen = new Set(base.map((d) => d.name));
+        const result = [...base];
+        for (const dep of incoming) {
+            if (!seen.has(dep.name)) {
+                result.push(dep);
+                seen.add(dep.name);
+            }
+        }
+        return result;
+    }
+    mergeLayers(base, incoming) {
+        if (!incoming)
+            return base;
+        return {
+            presentation: Math.max(base.presentation || 0, incoming.presentation || 0),
+            business: Math.max(base.business || 0, incoming.business || 0),
+            data: Math.max(base.data || 0, incoming.data || 0),
+            state: Math.max(base.state || 0, incoming.state || 0),
+            core: Math.max(base.core || 0, incoming.core || 0),
+            shared: Math.max(base.shared || 0, incoming.shared || 0),
+            feature: Math.max(base.feature || 0, incoming.feature || 0),
+            infrastructure: Math.max(base.infrastructure || 0, incoming.infrastructure || 0),
+            unknown: Math.max(base.unknown || 0, incoming.unknown || 0)
+        };
+    }
+    mergeStatistics(base, incoming) {
+        return {
+            totalFiles: Math.max(base.totalFiles || 0, incoming.totalFiles || 0),
+            totalLines: Math.max(base.totalLines || 0, incoming.totalLines || 0),
+            totalComponents: Math.max(base.totalComponents || 0, incoming.totalComponents || 0),
+            componentsByType: { ...base.componentsByType, ...incoming.componentsByType },
+            componentsByLayer: this.mergeLayers(base.componentsByLayer, incoming.componentsByLayer)
+        };
+    }
     /**
      * Get regex pattern for extracting code snippets based on pattern category and name
      * This maps abstract pattern names to actual code patterns
@@ -514,24 +581,24 @@ export class CodebaseIndexer {
         const patterns = {
             dependencyInjection: {
                 'inject() function': /\binject\s*[<(]/,
-                'Constructor injection': /constructor\s*\(/,
+                'Constructor injection': /constructor\s*\(/
             },
             stateManagement: {
-                'RxJS': /BehaviorSubject|ReplaySubject|Subject|Observable/,
-                'Signals': /\bsignal\s*[<(]/,
+                RxJS: /BehaviorSubject|ReplaySubject|Subject|Observable/,
+                Signals: /\bsignal\s*[<(]/
             },
             reactivity: {
-                'Effect': /\beffect\s*\(/,
-                'Computed': /\bcomputed\s*[<(]/,
+                Effect: /\beffect\s*\(/,
+                Computed: /\bcomputed\s*[<(]/
             },
             componentStyle: {
-                'Standalone': /standalone\s*:\s*true/,
-                'NgModule-based': /@(?:Component|Directive|Pipe)\s*\(/,
+                Standalone: /standalone\s*:\s*true/,
+                'NgModule-based': /@(?:Component|Directive|Pipe)\s*\(/
             },
             componentInputs: {
                 'Signal-based inputs': /\binput\s*[<(]/,
-                'Decorator-based @Input': /@Input\(\)/,
-            },
+                'Decorator-based @Input': /@Input\(\)/
+            }
         };
         return patterns[category]?.[name] || null;
     }