attocode 0.2.3 → 0.2.5

package/dist/src/integrations/context/codebase-context.js

@@ -0,0 +1,685 @@
+/**
+ * Codebase Context Integration
+ *
+ * Provides intelligent code selection for context management.
+ * Analyzes repository structure and selects relevant code based on:
+ * - Task relevance (semantic matching)
+ * - Architectural importance (entry points, core modules)
+ * - Token budget constraints
+ *
+ * Key features:
+ * - Budget-aware context building (select code that fits)
+ * - Importance scoring (imports, exports, modification frequency)
+ * - Incremental expansion (start minimal, expand as needed)
+ * - Semantic relevance matching (keyword + structure analysis)
+ *
+ * Implementation split (Phase 3d):
+ * - code-analyzer.ts  — AST + dependency analysis helpers
+ * - code-selector.ts  — Token-budget selection + ranking
+ * - codebase-context.ts — Main manager class + types + factory functions (this file)
+ */
+import * as path from 'path';
+// Analyzer functions (file discovery, processing, dependency analysis)
+import { discoverFiles, processFile, extractDependencies, adjustImportanceByConnectivity, matchesPatterns, } from './code-analyzer.js';
+import { clearASTCache, fullReparse, djb2Hash } from './codebase-ast.js';
+// Selector functions (selection, search, ranking, LSP-enhanced context)
+import { selectRelevantCode as selectRelevantCodeImpl, getEnhancedContext as getEnhancedContextImpl, expandContext as expandContextImpl, searchChunks, searchRanked as searchRankedImpl, } from './code-selector.js';
+// Re-export extracted modules for direct consumer access
+export { discoverFiles, processFile, extractDependencies, adjustImportanceByConnectivity, matchesPatterns, matchesGlob, normalizePath, determineChunkType, calculateBaseImportance, extractSymbols, extractSymbolDetails, extractDependencyNames, } from './code-analyzer.js';
+export { selectRelevantCode as selectRelevantCodeFn, getEnhancedContext as getEnhancedContextFn, expandContext as expandContextFn, searchChunks, searchRanked as searchRankedFn, calculateRelevance, calculateSearchScore, sortByStrategy, tokenizeQuery, fuzzyMatch, levenshteinDistance, } from './code-selector.js';
+// =============================================================================
+// CONSTANTS
+// =============================================================================
+const DEFAULT_CONFIG = {
+    root: process.cwd(),
+    includePatterns: ['**/*.ts', '**/*.tsx', '**/*.js', '**/*.jsx', '**/*.py', '**/*.go', '**/*.rs'],
+    excludePatterns: [
+        '**/node_modules/**',
+        '**/dist/**',
+        '**/build/**',
+        '**/.git/**',
+        '**/coverage/**',
+        '**/__pycache__/**',
+        '**/target/**',
+        '**/vendor/**',
+    ],
+    maxFileSize: 100 * 1024, // 100KB
+    tokensPerChar: 0.25, // Approximate: 4 chars per token
+    entryPointPatterns: [
+        '**/index.ts', '**/index.tsx', '**/index.js',
+        '**/main.ts', '**/main.py', '**/main.go',
+        '**/app.ts', '**/app.tsx', '**/app.py',
+        '**/server.ts', '**/server.js',
+        '**/cli.ts', '**/cli.js',
+    ],
+    coreModulePatterns: [
+        '**/src/**', '**/lib/**', '**/core/**',
+        '**/services/**', '**/modules/**',
+    ],
+    analyzeDependencies: true,
+    cacheResults: true,
+    cacheTTL: 5 * 60 * 1000, // 5 minutes
+};
+// =============================================================================
+// CODEBASE CONTEXT MANAGER
+// =============================================================================
+/**
+ * Manages intelligent code selection for context building.
+ *
+ * @example
+ * ```typescript
+ * const codebase = createCodebaseContext({ root: './my-project' });
+ *
+ * // Analyze the repository
+ * const repoMap = await codebase.analyze();
+ *
+ * // Select relevant code for a task within budget
+ * const selection = await codebase.selectRelevantCode({
+ *   task: 'implement user authentication',
+ *   maxTokens: 8000,
+ *   strategy: 'relevance_first',
+ * });
+ *
+ * // Use selected code in context
+ * const context = selection.chunks.map(c => c.content).join('\n\n');
+ * ```
+ */
+export class CodebaseContextManager {
+    config;
+    repoMap = null;
+    cache = new Map();
+    listeners = [];
+    lspManager = null;
+    /** Optional trace collector for emitting codebase analysis events. */
+    traceCollector;
+    /** Optional store for persisting codebase analysis across sessions. */
+    persistenceStore = null;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_CONFIG, ...config };
+    }
+    /**
+     * Set the persistence store for warm startup across sessions.
+     * Accepts any object implementing saveCodebaseAnalysis/loadCodebaseAnalysis (e.g. SQLiteStore).
+     */
+    setPersistenceStore(store) {
+        this.persistenceStore = store;
+    }
+    /**
+     * Set the LSP manager for enhanced context selection.
+     * When set, context selection can use LSP for more accurate relevance scoring.
+     */
+    setLSPManager(lsp) {
+        this.lspManager = lsp;
+    }
+    /**
+     * Get the current LSP manager, if any.
+     */
+    getLSPManager() {
+        return this.lspManager;
+    }
+    /**
+     * Check if LSP is available and active.
+     */
+    hasActiveLSP() {
+        if (!this.lspManager)
+            return false;
+        const servers = this.lspManager.getActiveServers();
+        return servers.length > 0;
+    }
+    // ===========================================================================
+    // ANALYSIS
+    // ===========================================================================
+    /**
+     * Analyze a repository and build its structure map.
+     */
+    async analyze(root) {
+        const repoRoot = root ?? this.config.root;
+        this.emit({ type: 'analysis.started', root: repoRoot });
+        // Check cache
+        if (this.config.cacheResults) {
+            const cached = this.cache.get(repoRoot);
+            if (cached && cached.expires > Date.now()) {
+                this.emit({ type: 'cache.hit', root: repoRoot });
+                this.repoMap = cached.map;
+                return cached.map;
+            }
+            this.emit({ type: 'cache.miss', root: repoRoot });
+        }
+        try {
+            const chunks = new Map();
+            const entryPoints = [];
+            const coreModules = [];
+            const dependencyGraph = new Map();
+            const reverseDependencyGraph = new Map();
+            const analyzerDeps = this;
+            // Discover files
+            const files = await discoverFiles(analyzerDeps, repoRoot);
+            // Warm startup: try loading persisted analysis and only reprocess changed files
+            const savedAnalysis = this.persistenceStore
+                ? this.persistenceStore.loadCodebaseAnalysis(repoRoot)
+                : null;
+            const filesToProcess = savedAnalysis
+                ? await this.diffAgainstSaved(repoRoot, files, savedAnalysis, chunks)
+                : files;
+            // Process each file (only changed/new files when warm-starting)
+            for (const filePath of filesToProcess) {
+                const chunk = await processFile(analyzerDeps, repoRoot, filePath);
+                if (chunk) {
+                    chunks.set(chunk.id, chunk);
+                    // Track entry points
+                    if (chunk.type === 'entry_point') {
+                        entryPoints.push(filePath);
+                    }
+                    // Build dependency graph
+                    if (this.config.analyzeDependencies) {
+                        const fullPath = path.join(repoRoot, filePath);
+                        const deps = extractDependencies(chunk.content, fullPath);
+                        dependencyGraph.set(filePath, deps);
+                        // Build reverse graph
+                        for (const dep of deps) {
+                            if (!reverseDependencyGraph.has(dep)) {
+                                reverseDependencyGraph.set(dep, new Set());
+                            }
+                            reverseDependencyGraph.get(dep).add(filePath);
+                        }
+                    }
+                }
+            }
+            // Restore dependency graph edges for unchanged files from saved analysis
+            if (savedAnalysis && this.config.analyzeDependencies) {
+                for (const [source, targets] of savedAnalysis.deps) {
+                    if (!filesToProcess.includes(source) && chunks.has(source)) {
+                        dependencyGraph.set(source, targets);
+                        for (const dep of targets) {
+                            if (!reverseDependencyGraph.has(dep)) {
+                                reverseDependencyGraph.set(dep, new Set());
+                            }
+                            reverseDependencyGraph.get(dep).add(source);
+                        }
+                    }
+                }
+            }
+            // Ensure entry points include restored unchanged files
+            if (savedAnalysis) {
+                for (const [filePath, chunk] of chunks) {
+                    if (chunk.type === 'entry_point' && !entryPoints.includes(filePath)) {
+                        entryPoints.push(filePath);
+                    }
+                }
+            }
+            // Calculate total tokens
+            let totalTokens = 0;
+            for (const chunk of chunks.values()) {
+                totalTokens += chunk.tokenCount;
+            }
+            // Adjust importance based on dependency graph
+            adjustImportanceByConnectivity(chunks, reverseDependencyGraph);
+            // Identify core modules
+            const moduleDirs = new Set();
+            for (const filePath of files) {
+                const dir = path.dirname(filePath);
+                if (matchesPatterns(analyzerDeps, dir, this.config.coreModulePatterns)) {
+                    moduleDirs.add(dir);
+                }
+            }
+            coreModules.push(...moduleDirs);
+            const repoMap = {
+                root: repoRoot,
+                chunks,
+                entryPoints,
+                coreModules,
+                dependencyGraph,
+                reverseDependencyGraph,
+                totalTokens,
+                analyzedAt: new Date(),
+            };
+            // Cache result
+            if (this.config.cacheResults) {
+                this.cache.set(repoRoot, {
+                    map: repoMap,
+                    expires: Date.now() + this.config.cacheTTL,
+                });
+            }
+            this.repoMap = repoMap;
+            // Persist analysis to SQLite for warm startup next session
+            if (this.persistenceStore) {
+                try {
+                    this.persistenceStore.saveCodebaseAnalysis(repoRoot, chunks.values(), dependencyGraph);
+                }
+                catch { /* persistence is non-blocking */ }
+            }
+            this.emit({
+                type: 'analysis.completed',
+                totalFiles: chunks.size,
+                totalTokens,
+            });
+            // Emit trace event with full codebase map data
+            this.traceCollector?.record({
+                type: 'codebase.map',
+                data: {
+                    totalFiles: chunks.size,
+                    totalTokens,
+                    entryPoints,
+                    coreModules,
+                    dependencyEdges: Array.from(dependencyGraph.entries()).map(([file, imports]) => ({
+                        file,
+                        imports: Array.from(imports),
+                    })),
+                    topChunks: Array.from(chunks.values())
+                        .sort((a, b) => b.importance - a.importance)
+                        .slice(0, 50)
+                        .map(chunk => ({
+                        filePath: chunk.filePath,
+                        tokenCount: chunk.tokenCount,
+                        importance: chunk.importance,
+                        type: chunk.type,
+                        symbols: chunk.symbolDetails,
+                        dependencies: chunk.dependencies,
+                    })),
+                    files: Array.from(chunks.values()).map((chunk) => ({
+                        filePath: chunk.filePath,
+                        directory: path.dirname(chunk.filePath) === '.' ? '' : path.dirname(chunk.filePath),
+                        fileName: path.basename(chunk.filePath),
+                        tokenCount: chunk.tokenCount,
+                        importance: chunk.importance,
+                        type: chunk.type,
+                        symbols: chunk.symbolDetails,
+                        inDegree: reverseDependencyGraph.get(chunk.filePath)?.size ?? 0,
+                        outDegree: dependencyGraph.get(chunk.filePath)?.size ?? 0,
+                    })),
+                },
+            });
+            return repoMap;
+        }
+        catch (error) {
+            this.emit({ type: 'analysis.error', error: error });
+            throw error;
+        }
+    }
+    // ===========================================================================
+    // SELECTION (delegates to code-selector.ts)
+    // ===========================================================================
+    /**
+     * Select relevant code chunks for a task within a token budget.
+     */
+    async selectRelevantCode(options) {
+        return selectRelevantCodeImpl(this.asSelectorDeps(), options);
+    }
+    // ===========================================================================
+    // LSP-ENHANCED CONTEXT SELECTION (delegates to code-selector.ts)
+    // ===========================================================================
+    /**
+     * Select relevant code with LSP-enhanced relevance scoring.
+     *
+     * When an LSP manager is set and active, this method uses LSP to:
+     * - Find files that reference the editing file (who uses this code?)
+     * - Find files that the editing file references (what does this depend on?)
+     * - Boost priority of related files in the selection
+     *
+     * @example
+     * ```typescript
+     * const codebase = createCodebaseContext({ root: './project' });
+     * codebase.setLSPManager(lspManager);
+     *
+     * const result = await codebase.getEnhancedContext({
+     *   task: 'fix bug in authentication',
+     *   maxTokens: 10000,
+     *   editingFile: 'src/auth/login.ts',
+     *   editingPosition: { line: 25, character: 10 },
+     * });
+     *
+     * // Result includes LSP-boosted files
+     * console.log('LSP boosted:', result.lspBoostedFiles);
+     * ```
+     */
+    async getEnhancedContext(options) {
+        return getEnhancedContextImpl(this.asSelectorDeps(), options);
+    }
+    // ===========================================================================
+    // INCREMENTAL EXPANSION (delegates to code-selector.ts)
+    // ===========================================================================
+    /**
+     * Expand context by adding related files to current selection.
+     */
+    async expandContext(currentChunks, options) {
+        return expandContextImpl(this.asSelectorDeps(), currentChunks, options);
+    }
+    // ===========================================================================
+    // UTILITIES
+    // ===========================================================================
+    /**
+     * Get a chunk by file path.
+     */
+    getChunk(filePath) {
+        return this.repoMap?.chunks.get(filePath);
+    }
+    /**
+     * Get all chunks.
+     */
+    getAllChunks() {
+        return this.repoMap ? Array.from(this.repoMap.chunks.values()) : [];
+    }
+    /**
+     * Get entry points.
+     */
+    getEntryPoints() {
+        if (!this.repoMap)
+            return [];
+        return this.repoMap.entryPoints
+            .map((ep) => this.repoMap.chunks.get(ep))
+            .filter((c) => c !== undefined);
+    }
+    /**
+     * Estimate tokens for a string.
+     */
+    estimateTokens(content) {
+        return Math.ceil(content.length * this.config.tokensPerChar);
+    }
+    /**
+     * Get the current repo map.
+     */
+    getRepoMap() {
+        return this.repoMap;
+    }
+    /**
+     * Clear cached data (including AST parse cache).
+     */
+    clearCache() {
+        this.cache.clear();
+        this.repoMap = null;
+        clearASTCache();
+    }
+    // ===========================================================================
+    // WARM STARTUP HELPERS (Phase 3)
+    // ===========================================================================
+    /**
+     * Diff discovered files against saved analysis.
+     * Reuses saved chunks for unchanged files, returns only files that need reprocessing.
+     */
+    async diffAgainstSaved(repoRoot, files, saved, chunks) {
+        const { default: fs } = await import('fs/promises');
+        const changedFiles = [];
+        for (const relativePath of files) {
+            const savedChunk = saved.chunks.get(relativePath);
+            if (savedChunk) {
+                // Check if file content has changed by reading + hashing
+                try {
+                    const fullPath = path.join(repoRoot, relativePath);
+                    const content = await fs.readFile(fullPath, 'utf-8');
+                    const currentHash = djb2Hash(content).toString(36);
+                    if (currentHash === savedChunk.contentHash) {
+                        // Unchanged — restore chunk from saved data
+                        const symbolDetails = JSON.parse(savedChunk.symbolsJson || '[]');
+                        const dependencies = JSON.parse(savedChunk.dependenciesJson || '[]');
+                        chunks.set(relativePath, {
+                            id: relativePath,
+                            filePath: relativePath,
+                            content,
+                            tokenCount: savedChunk.tokenCount,
+                            importance: savedChunk.importance,
+                            type: savedChunk.chunkType,
+                            symbols: symbolDetails.map((s) => s.name),
+                            symbolDetails,
+                            dependencies,
+                            lastModified: new Date(savedChunk.analyzedAt),
+                        });
+                        continue;
+                    }
+                }
+                catch {
+                    // File read failed — treat as changed
+                }
+            }
+            changedFiles.push(relativePath);
+        }
+        return changedFiles;
+    }
+    // ===========================================================================
+    // INCREMENTAL FILE UPDATES (Phase 2)
+    // ===========================================================================
+    /**
+     * Update a single file's chunk in-place after an edit.
+     * Reparses the file, patches the chunk's symbols/deps, and updates dependency graph edges.
+     * Avoids full re-analyze for single-file edits.
+     */
+    async updateFile(filePath, newContent) {
+        if (!this.repoMap)
+            return;
+        const relativePath = path.relative(this.config.root, filePath);
+        const parsed = fullReparse(filePath, newContent);
+        const existingChunk = this.repoMap.chunks.get(relativePath);
+        if (existingChunk) {
+            // Patch in-place
+            existingChunk.content = newContent;
+            existingChunk.tokenCount = Math.ceil(newContent.length * this.config.tokensPerChar);
+            existingChunk.lastModified = new Date();
+            if (parsed) {
+                existingChunk.symbols = parsed.symbols.map(s => s.name);
+                existingChunk.symbolDetails = parsed.symbols.map(s => ({
+                    name: s.name, kind: s.kind, exported: s.exported, line: s.line,
+                }));
+                existingChunk.dependencies = parsed.dependencies
+                    .flatMap(d => d.names.filter(n => !n.startsWith('* as ')));
+            }
+            // Update dependency graph edges for this file
+            this.updateDependencyEdges(relativePath, newContent);
+            // Recalculate total tokens
+            let totalTokens = 0;
+            for (const chunk of this.repoMap.chunks.values()) {
+                totalTokens += chunk.tokenCount;
+            }
+            this.repoMap.totalTokens = totalTokens;
+        }
+        // New files (not in chunks) require a full re-analyze
+    }
+    updateDependencyEdges(filePath, content) {
+        if (!this.repoMap)
+            return;
+        const { dependencyGraph, reverseDependencyGraph } = this.repoMap;
+        // Remove old forward edges
+        const oldDeps = dependencyGraph.get(filePath);
+        if (oldDeps) {
+            for (const dep of oldDeps) {
+                reverseDependencyGraph.get(dep)?.delete(filePath);
+            }
+        }
+        // Build new forward edges (use absolute path for AST cache lookup)
+        const fullPath = path.join(this.config.root, filePath);
+        const newDeps = extractDependencies(content, fullPath);
+        dependencyGraph.set(filePath, newDeps);
+        // Build new reverse edges
+        for (const dep of newDeps) {
+            if (!reverseDependencyGraph.has(dep)) {
+                reverseDependencyGraph.set(dep, new Set());
+            }
+            reverseDependencyGraph.get(dep).add(filePath);
+        }
+    }
+    /**
+     * Subscribe to events.
+     */
+    on(listener) {
+        this.listeners.push(listener);
+        return () => {
+            const idx = this.listeners.indexOf(listener);
+            if (idx >= 0)
+                this.listeners.splice(idx, 1);
+        };
+    }
+    emit(event) {
+        for (const listener of this.listeners) {
+            try {
+                listener(event);
+            }
+            catch {
+                // Ignore listener errors
+            }
+        }
+    }
+    // ===========================================================================
+    // ENHANCED SEARCH (delegates to code-selector.ts)
+    // ===========================================================================
+    /**
+     * Search for code chunks matching a query.
+     *
+     * @example
+     * ```typescript
+     * const results = codebase.search('authentication', {
+     *   includeSymbols: true,
+     *   includePaths: true,
+     * });
+     * ```
+     */
+    search(query, options = {}) {
+        return searchChunks(this.repoMap, query, options);
+    }
+    /**
+     * Search and rank results by relevance.
+     *
+     * @example
+     * ```typescript
+     * const ranked = codebase.searchRanked('user auth', { limit: 10 });
+     * ranked.forEach(r => console.log(`${r.chunk.filePath}: ${r.score}`));
+     * ```
+     */
+    searchRanked(query, options = {}) {
+        return searchRankedImpl(this.repoMap, query, options);
+    }
+    // ===========================================================================
+    // INTERNAL HELPERS
+    // ===========================================================================
+    /**
+     * Build the CodeSelectorDeps facade for delegating to code-selector functions.
+     */
+    asSelectorDeps() {
+        // eslint-disable-next-line @typescript-eslint/no-this-alias
+        const self = this;
+        return {
+            get repoMap() {
+                return self.repoMap;
+            },
+            get lspManager() {
+                return self.lspManager;
+            },
+            hasActiveLSP: () => self.hasActiveLSP(),
+            analyze: () => self.analyze(),
+            emit: (event) => self.emit(event),
+        };
+    }
+}
+// =============================================================================
+// FACTORY FUNCTIONS
+// =============================================================================
+/**
+ * Create a codebase context manager.
+ *
+ * @example
+ * ```typescript
+ * const codebase = createCodebaseContext({
+ *   root: './my-project',
+ *   includePatterns: ['src/**\/*.ts'],
+ *   excludePatterns: ['node_modules/**'],
+ * });
+ *
+ * // Analyze repo
+ * await codebase.analyze();
+ *
+ * // Select code for task
+ * const selection = await codebase.selectRelevantCode({
+ *   task: 'fix authentication bug',
+ *   maxTokens: 10000,
+ *   strategy: 'relevance_first',
+ * });
+ *
+ * console.log(`Selected ${selection.chunks.length} files`);
+ * console.log(`Using ${selection.totalTokens} tokens`);
+ * ```
+ */
+export function createCodebaseContext(config = {}) {
+    return new CodebaseContextManager(config);
+}
+/**
+ * Build context string from selected chunks.
+ */
+export function buildContextFromChunks(chunks, options = {}) {
+    const { includeFilePaths = true, includeSeparators = true, maxTotalTokens, } = options;
+    const parts = [];
+    let totalTokens = 0;
+    for (const chunk of chunks) {
+        if (maxTotalTokens && totalTokens + chunk.tokenCount > maxTotalTokens) {
+            break;
+        }
+        if (includeSeparators && parts.length > 0) {
+            parts.push('\n---\n');
+        }
+        if (includeFilePaths) {
+            parts.push(`// File: ${chunk.filePath}\n`);
+        }
+        parts.push(chunk.content);
+        totalTokens += chunk.tokenCount;
+    }
+    return parts.join('');
+}
+/**
+ * Create a summary of the repo structure.
+ */
+export function summarizeRepoStructure(repoMap) {
+    const lines = [
+        `Repository: ${repoMap.root}`,
+        `Total files: ${repoMap.chunks.size}`,
+        `Total tokens: ${repoMap.totalTokens.toLocaleString()}`,
+        ``,
+        `Entry points:`,
+    ];
+    for (const ep of repoMap.entryPoints.slice(0, 5)) {
+        lines.push(`  - ${ep}`);
+    }
+    lines.push(``, `Core modules:`);
+    for (const mod of repoMap.coreModules.slice(0, 5)) {
+        lines.push(`  - ${mod}`);
+    }
+    // Type breakdown
+    const byType = new Map();
+    for (const chunk of repoMap.chunks.values()) {
+        byType.set(chunk.type, (byType.get(chunk.type) ?? 0) + 1);
+    }
+    lines.push(``, `File types:`);
+    for (const [type, count] of byType) {
+        lines.push(`  - ${type}: ${count}`);
+    }
+    return lines.join('\n');
+}
+/**
+ * Generate an aider-style lightweight repo map: file tree with key symbols.
+ * Used as fallback when task-specific selection returns nothing.
+ */
+export function generateLightweightRepoMap(repoMap, maxTokens = 10000) {
+    const lines = ['## Repository Map\n'];
+    const byDir = new Map();
+    for (const chunk of repoMap.chunks.values()) {
+        const dir = path.dirname(chunk.filePath);
+        if (!byDir.has(dir))
+            byDir.set(dir, []);
+        byDir.get(dir).push(chunk);
+    }
+    let tokenEstimate = 0;
+    for (const [dir, chunks] of [...byDir].sort((a, b) => a[0].localeCompare(b[0]))) {
+        const dirLine = `${dir}/`;
+        lines.push(dirLine);
+        tokenEstimate += dirLine.length * 0.25;
+        for (const chunk of chunks.sort((a, b) => b.importance - a.importance)) {
+            const symbols = chunk.symbols.slice(0, 5).join(', ');
+            const fileLine = `  ${path.basename(chunk.filePath)}${symbols ? ` (${symbols})` : ''}`;
+            lines.push(fileLine);
+            tokenEstimate += fileLine.length * 0.25;
+            if (tokenEstimate > maxTokens)
+                break;
+        }
+        if (tokenEstimate > maxTokens) {
+            lines.push('  ... (truncated)');
+            break;
+        }
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=codebase-context.js.map