lance-context 0.1.0 → 1.1.0

package/dist/search/indexer.js

@@ -1,17 +1,70 @@
 import * as lancedb from '@lancedb/lancedb';
+import * as crypto from 'crypto';
 import * as fs from 'fs/promises';
 import * as path from 'path';
-const CHUNK_SIZE = 100; // lines per chunk
-const CHUNK_OVERLAP = 20; // overlap between chunks
+import { ASTChunker } from './ast-chunker.js';
+import { TreeSitterChunker } from './tree-sitter-chunker.js';
+import { loadConfig, getDefaultPatterns, getDefaultExcludePatterns, getChunkingConfig, getSearchConfig, } from '../config.js';
+import { minimatch } from 'minimatch';
+import { mapInBatches } from '../utils/concurrency.js';
+import { kMeansClustering, calculateSilhouetteScore, } from './clustering.js';
+/** Default concurrency for parallel file processing */
+const FILE_PROCESSING_CONCURRENCY = 10;
 /**
- * Code indexer using LanceDB for vector storage
+ * Compute a checksum for index integrity validation.
+ * Based on sorted file list and chunk count.
  */
+function computeIndexChecksum(files, chunkCount) {
+    const sortedFiles = [...files].sort();
+    const data = JSON.stringify({ files: sortedFiles, chunkCount });
+    return crypto.createHash('sha256').update(data).digest('hex').slice(0, 16);
+}
+/**
+ * Sanitize a file path for use in LanceDB filter expressions.
+ * Prevents SQL injection by only allowing safe path characters.
+ */
+function sanitizePathForFilter(filepath) {
+    // Only allow safe file path characters: alphanumeric, /, ., -, _, space
+    // This is more restrictive than escaping and prevents injection attacks
+    if (!/^[\w\s./-]+$/.test(filepath)) {
+        // If path contains unusual characters, escape single quotes and backslashes
+        return filepath.replace(/\\/g, '\\\\').replace(/'/g, "''");
+    }
+    return filepath.replace(/'/g, "''");
+}
+/**
+ * Code indexer that uses LanceDB for vector storage and semantic search.
+ *
+ * Provides functionality to:
+ * - Index a codebase by chunking files and generating embeddings
+ * - Perform hybrid semantic + keyword search
+ * - Support incremental indexing (only re-index changed files)
+ *
+ * @example
+ * ```typescript
+ * const backend = await createEmbeddingBackend();
+ * const indexer = new CodeIndexer('/path/to/project', backend);
+ * await indexer.initialize();
+ *
+ * // Index the codebase
+ * await indexer.indexCodebase();
+ *
+ * // Search for code
+ * const results = await indexer.search('authentication middleware');
+ * ```
+ */
+/** Maximum number of query embeddings to cache */
+const QUERY_CACHE_MAX_SIZE = 100;
 export class CodeIndexer {
     db = null;
     table = null;
+    metadataTable = null;
     embeddingBackend;
     indexPath;
     projectPath;
+    config = null;
+    /** LRU cache for query embeddings to avoid recomputing identical queries */
+    queryEmbeddingCache = new Map();
     constructor(projectPath, embeddingBackend) {
         this.projectPath = projectPath;
         this.embeddingBackend = embeddingBackend;
@@ -19,6 +72,118 @@ export class CodeIndexer {
     }
     async initialize() {
         this.db = await lancedb.connect(this.indexPath);
+        this.config = await loadConfig(this.projectPath);
+        console.error(`[lance-context] Loaded config with ${this.config.patterns?.length} patterns`);
+    }
+    get metadataPath() {
+        return path.join(this.indexPath, 'index-metadata.json');
+    }
+    /**
+     * Save index metadata to disk
+     */
+    async saveIndexMetadata(fileCount, chunkCount, indexedFiles) {
+        // Convert to relative paths for checksum
+        const relativePaths = indexedFiles.map((f) => path.isAbsolute(f) ? path.relative(this.projectPath, f) : f);
+        const metadata = {
+            lastUpdated: new Date().toISOString(),
+            fileCount,
+            chunkCount,
+            embeddingBackend: this.embeddingBackend.name,
+            embeddingModel: this.embeddingBackend.getModel(),
+            embeddingDimensions: this.embeddingBackend.getDimensions(),
+            version: '1.0.0',
+            checksum: computeIndexChecksum(relativePaths, chunkCount),
+        };
+        await fs.writeFile(this.metadataPath, JSON.stringify(metadata, null, 2));
+    }
+    /**
+     * Load index metadata from disk
+     */
+    async loadIndexMetadata() {
+        try {
+            const content = await fs.readFile(this.metadataPath, 'utf-8');
+            return JSON.parse(content);
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Get the modification time of a file
+     */
+    async getFileMtime(filepath) {
+        const stats = await fs.stat(filepath);
+        return stats.mtimeMs;
+    }
+    /**
+     * Get stored metadata for all indexed files
+     */
+    async getStoredMetadata() {
+        const tableNames = await this.db.tableNames();
+        if (!tableNames.includes('file_metadata')) {
+            return new Map();
+        }
+        this.metadataTable = await this.db.openTable('file_metadata');
+        const rows = await this.metadataTable.query().toArray();
+        const metadata = new Map();
+        for (const row of rows) {
+            metadata.set(row.filepath, row.mtime);
+        }
+        return metadata;
+    }
+    /**
+     * Detect which files have been added, modified, or deleted
+     */
+    async detectFileChanges(currentFiles) {
+        const storedMetadata = await this.getStoredMetadata();
+        const changes = {
+            added: [],
+            modified: [],
+            deleted: [],
+            unchanged: [],
+        };
+        const currentFilesSet = new Set();
+        for (const filepath of currentFiles) {
+            const relativePath = path.relative(this.projectPath, filepath);
+            currentFilesSet.add(relativePath);
+            const currentMtime = await this.getFileMtime(filepath);
+            const storedMtime = storedMetadata.get(relativePath);
+            if (storedMtime === undefined) {
+                changes.added.push(filepath);
+            }
+            else if (currentMtime > storedMtime) {
+                changes.modified.push(filepath);
+            }
+            else {
+                changes.unchanged.push(filepath);
+            }
+        }
+        // Find deleted files
+        for (const [relativePath] of storedMetadata) {
+            if (!currentFilesSet.has(relativePath)) {
+                changes.deleted.push(relativePath);
+            }
+        }
+        return changes;
+    }
+    /**
+     * Save metadata for indexed files
+     */
+    async saveFileMetadata(files) {
+        const metadata = [];
+        for (const filepath of files) {
+            const relativePath = path.relative(this.projectPath, filepath);
+            const mtime = await this.getFileMtime(filepath);
+            metadata.push({ filepath: relativePath, mtime });
+        }
+        // Drop and recreate metadata table
+        const tableNames = await this.db.tableNames();
+        if (tableNames.includes('file_metadata')) {
+            await this.db.dropTable('file_metadata');
+        }
+        if (metadata.length > 0) {
+            this.metadataTable = await this.db.createTable('file_metadata', metadata);
+        }
     }
     async getStatus() {
         const tableNames = await this.db?.tableNames();
@@ -30,54 +195,199 @@ export class CodeIndexer {
                 chunkCount: 0,
                 lastUpdated: null,
                 indexPath: this.indexPath,
+                embeddingBackend: this.embeddingBackend.name,
+                embeddingModel: this.embeddingBackend.getModel(),
             };
         }
         this.table = await this.db.openTable('code_chunks');
         const count = await this.table.countRows();
+        // Load persisted metadata
+        const metadata = await this.loadIndexMetadata();
+        // Validate index integrity
+        const corruptionCheck = await this.validateIndexIntegrity(metadata, count);
+        // Check for backend mismatch
+        const backendMismatch = this.checkBackendMismatch(metadata);
         return {
             indexed: true,
-            fileCount: 0, // Would need to query distinct files
+            fileCount: metadata?.fileCount ?? 0,
             chunkCount: count,
-            lastUpdated: new Date().toISOString(),
+            lastUpdated: metadata?.lastUpdated ?? null,
             indexPath: this.indexPath,
+            embeddingBackend: metadata?.embeddingBackend ?? this.embeddingBackend.name,
+            embeddingModel: metadata?.embeddingModel ?? this.embeddingBackend.getModel(),
+            corrupted: corruptionCheck.corrupted,
+            corruptionReason: corruptionCheck.reason,
+            backendMismatch: backendMismatch.mismatch,
+            backendMismatchReason: backendMismatch.reason,
         };
     }
-    async indexCodebase(patterns = ['**/*.ts', '**/*.tsx', '**/*.js', '**/*.jsx', '**/*.py', '**/*.go', '**/*.rs'], excludePatterns = ['**/node_modules/**', '**/dist/**', '**/.git/**', '**/build/**']) {
+    /**
+     * Check if the current embedding backend differs from the one used to create the index.
+     * Returns mismatch status and reason if mismatched.
+     */
+    checkBackendMismatch(metadata) {
+        if (!metadata) {
+            return { mismatch: false };
+        }
+        const currentBackend = this.embeddingBackend.name;
+        const currentModel = this.embeddingBackend.getModel();
+        const currentDimensions = this.embeddingBackend.getDimensions();
+        // Check dimension mismatch (critical - will cause search failures)
+        if (metadata.embeddingDimensions && metadata.embeddingDimensions !== currentDimensions) {
+            return {
+                mismatch: true,
+                reason: `Embedding dimension mismatch: index has ${metadata.embeddingDimensions}-dim vectors, ` +
+                    `current backend (${currentBackend}) produces ${currentDimensions}-dim vectors. Reindex required.`,
+            };
+        }
+        // Check model mismatch (different models produce incompatible embeddings)
+        if (metadata.embeddingModel && metadata.embeddingModel !== currentModel) {
+            return {
+                mismatch: true,
+                reason: `Embedding model mismatch: index uses '${metadata.embeddingModel}', ` +
+                    `current backend uses '${currentModel}'. Reindex required.`,
+            };
+        }
+        // Check backend mismatch (even same dimensions may have different embedding spaces)
+        if (metadata.embeddingBackend && metadata.embeddingBackend !== currentBackend) {
+            return {
+                mismatch: true,
+                reason: `Embedding backend changed from '${metadata.embeddingBackend}' to '${currentBackend}'. Reindex required.`,
+            };
+        }
+        return { mismatch: false };
+    }
+    /**
+     * Validate index integrity by checking metadata consistency.
+     * Returns corruption status and reason if corrupted.
+     */
+    async validateIndexIntegrity(metadata, actualChunkCount) {
+        // No metadata file - possible incomplete indexing
+        if (!metadata) {
+            return {
+                corrupted: true,
+                reason: 'Missing index metadata file. Index may be incomplete. Run clear_index followed by index_codebase to rebuild.',
+            };
+        }
+        // Check if chunk count matches
+        if (metadata.chunkCount !== actualChunkCount) {
+            return {
+                corrupted: true,
+                reason: `Chunk count mismatch: metadata says ${metadata.chunkCount}, index has ${actualChunkCount}. Run clear_index followed by index_codebase to rebuild.`,
+            };
+        }
+        // Validate checksum if present
+        if (metadata.checksum) {
+            const storedFiles = await this.getStoredMetadata();
+            const fileList = Array.from(storedFiles.keys());
+            const computedChecksum = computeIndexChecksum(fileList, actualChunkCount);
+            if (computedChecksum !== metadata.checksum) {
+                return {
+                    corrupted: true,
+                    reason: `Checksum mismatch: file metadata does not match index. Run clear_index followed by index_codebase to rebuild.`,
+                };
+            }
+        }
+        return { corrupted: false };
+    }
+    async indexCodebase(patterns, excludePatterns, forceReindex = false, onProgress, autoRepair = false) {
         const { glob } = await import('glob');
+        const report = (progress) => {
+            console.error(`[lance-context] ${progress.message}`);
+            onProgress?.(progress);
+        };
+        // Check for corruption if autoRepair is enabled
+        if (autoRepair) {
+            const status = await this.getStatus();
+            if (status.corrupted) {
+                console.error(`[lance-context] Index corruption detected: ${status.corruptionReason}`);
+                console.error('[lance-context] Auto-repair enabled, clearing and rebuilding index...');
+                await this.clearIndex();
+                // Recursively call with forceReindex but without autoRepair to avoid loops
+                const result = await this.indexCodebase(patterns, excludePatterns, true, onProgress, false);
+                return { ...result, repaired: true };
+            }
+        }
+        // Use provided patterns or fall back to config/defaults
+        const effectivePatterns = patterns || this.config?.patterns || getDefaultPatterns();
+        const effectiveExcludePatterns = excludePatterns || this.config?.excludePatterns || getDefaultExcludePatterns();
+        report({ phase: 'scanning', current: 0, total: 0, message: 'Scanning for files...' });
         // Find all matching files
         const files = [];
-        for (const pattern of patterns) {
+        for (const pattern of effectivePatterns) {
             const matches = await glob(pattern, {
                 cwd: this.projectPath,
-                ignore: excludePatterns,
+                ignore: effectiveExcludePatterns,
                 absolute: true,
             });
             files.push(...matches);
         }
-        console.error(`[lance-context] Found ${files.length} files to index`);
-        // Process files into chunks
-        const allChunks = [];
-        for (const filePath of files) {
-            const chunks = await this.chunkFile(filePath);
-            allChunks.push(...chunks);
+        report({
+            phase: 'scanning',
+            current: files.length,
+            total: files.length,
+            message: `Found ${files.length} files to index`,
+        });
+        // Check if we can do incremental indexing
+        const tableNames = await this.db.tableNames();
+        const hasExistingIndex = tableNames.includes('code_chunks');
+        // Check for embedding dimension or model mismatch
+        let embeddingMismatch = false;
+        if (hasExistingIndex && !forceReindex) {
+            const metadata = await this.loadIndexMetadata();
+            const currentDimensions = this.embeddingBackend.getDimensions();
+            const currentModel = this.embeddingBackend.getModel();
+            // Check dimension mismatch
+            if (metadata?.embeddingDimensions && metadata.embeddingDimensions !== currentDimensions) {
+                console.error(`[lance-context] Embedding dimension mismatch: index has ${metadata.embeddingDimensions}, ` +
+                    `current backend (${this.embeddingBackend.name}) uses ${currentDimensions}. Forcing full reindex.`);
+                embeddingMismatch = true;
+            }
+            // Check model mismatch (even if dimensions match, different models produce incompatible embeddings)
+            if (metadata?.embeddingModel && metadata.embeddingModel !== currentModel) {
+                console.error(`[lance-context] Embedding model mismatch: index uses '${metadata.embeddingModel}', ` +
+                    `current backend uses '${currentModel}'. Forcing full reindex.`);
+                embeddingMismatch = true;
+            }
+        }
+        const canDoIncremental = hasExistingIndex && !forceReindex && !embeddingMismatch;
+        if (canDoIncremental) {
+            return this.indexIncremental(files, onProgress);
         }
-        console.error(`[lance-context] Created ${allChunks.length} chunks`);
-        // Generate embeddings in batches
-        const batchSize = 32;
-        for (let i = 0; i < allChunks.length; i += batchSize) {
-            const batch = allChunks.slice(i, i + batchSize);
-            const texts = batch.map((c) => c.content);
-            const embeddings = await this.embeddingBackend.embedBatch(texts);
-            batch.forEach((chunk, idx) => {
-                chunk.embedding = embeddings[idx];
+        // Full reindex
+        return this.indexFull(files, onProgress);
+    }
+    /**
+     * Perform a full reindex of all files
+     */
+    async indexFull(files, onProgress) {
+        const report = (progress) => {
+            console.error(`[lance-context] ${progress.message}`);
+            onProgress?.(progress);
+        };
+        // Process files into chunks (parallelized for I/O efficiency)
+        report({ phase: 'chunking', current: 0, total: files.length, message: 'Chunking files...' });
+        const chunkResults = await mapInBatches(files, async (filepath) => this.chunkFile(filepath), FILE_PROCESSING_CONCURRENCY, (completed, total) => {
+            report({
+                phase: 'chunking',
+                current: completed,
+                total,
+                message: `Chunked ${completed}/${total} files`,
             });
-            console.error(`[lance-context] Embedded ${i + batch.length}/${allChunks.length} chunks`);
-        }
+        });
+        const allChunks = chunkResults.flat();
+        report({
+            phase: 'chunking',
+            current: files.length,
+            total: files.length,
+            message: `Created ${allChunks.length} chunks`,
+        });
+        // Generate embeddings in batches
+        await this.embedChunks(allChunks, onProgress);
         // Store in LanceDB
-        const dimensions = this.embeddingBackend.getDimensions();
         const data = allChunks.map((chunk) => ({
             id: chunk.id,
-            filePath: chunk.filePath,
+            filepath: chunk.filepath,
             content: chunk.content,
             startLine: chunk.startLine,
             endLine: chunk.endLine,
@@ -90,26 +400,215 @@ export class CodeIndexer {
             await this.db.dropTable('code_chunks');
         }
         this.table = await this.db.createTable('code_chunks', data);
+        // Save file metadata for future incremental indexing
+        await this.saveFileMetadata(files);
+        // Save index metadata with checksum
+        await this.saveIndexMetadata(files.length, allChunks.length, files);
         return {
             filesIndexed: files.length,
             chunksCreated: allChunks.length,
+            incremental: false,
         };
     }
-    async chunkFile(filePath) {
-        const content = await fs.readFile(filePath, 'utf-8');
-        const lines = content.split('\n');
-        const ext = path.extname(filePath).slice(1);
+    /**
+     * Perform incremental indexing - only process changed files
+     */
+    async indexIncremental(files, onProgress) {
+        const report = (progress) => {
+            console.error(`[lance-context] ${progress.message}`);
+            onProgress?.(progress);
+        };
+        const changes = await this.detectFileChanges(files);
+        const filesToProcess = [...changes.added, ...changes.modified];
+        const hasChanges = filesToProcess.length > 0 || changes.deleted.length > 0;
+        if (!hasChanges) {
+            report({
+                phase: 'complete',
+                current: 0,
+                total: 0,
+                message: 'No changes detected, index is up to date',
+            });
+            this.table = await this.db.openTable('code_chunks');
+            const count = await this.table.countRows();
+            return {
+                filesIndexed: 0,
+                chunksCreated: count,
+                incremental: true,
+            };
+        }
+        report({
+            phase: 'scanning',
+            current: 0,
+            total: filesToProcess.length,
+            message: `Incremental update: ${changes.added.length} added, ${changes.modified.length} modified, ${changes.deleted.length} deleted`,
+        });
+        // Open the existing table
+        this.table = await this.db.openTable('code_chunks');
+        // Delete chunks from modified and deleted files
+        const filesToRemove = [
+            ...changes.modified.map((f) => path.relative(this.projectPath, f)),
+            ...changes.deleted,
+        ];
+        if (filesToRemove.length > 0) {
+            for (const relativePath of filesToRemove) {
+                const sanitizedPath = sanitizePathForFilter(relativePath);
+                await this.table.delete(`filepath = '${sanitizedPath}'`);
+            }
+            report({
+                phase: 'chunking',
+                current: 0,
+                total: filesToProcess.length,
+                message: `Removed chunks from ${filesToRemove.length} files`,
+            });
+        }
+        // Process new and modified files (parallelized for I/O efficiency)
+        if (filesToProcess.length > 0) {
+            const chunkResults = await mapInBatches(filesToProcess, async (filepath) => this.chunkFile(filepath), FILE_PROCESSING_CONCURRENCY, (completed, total) => {
+                report({
+                    phase: 'chunking',
+                    current: completed,
+                    total,
+                    message: `Chunked ${completed}/${total} files`,
+                });
+            });
+            const newChunks = chunkResults.flat();
+            report({
+                phase: 'chunking',
+                current: filesToProcess.length,
+                total: filesToProcess.length,
+                message: `Created ${newChunks.length} new chunks`,
+            });
+            // Generate embeddings
+            await this.embedChunks(newChunks, onProgress);
+            // Add new chunks to the table
+            const data = newChunks.map((chunk) => ({
+                id: chunk.id,
+                filepath: chunk.filepath,
+                content: chunk.content,
+                startLine: chunk.startLine,
+                endLine: chunk.endLine,
+                language: chunk.language,
+                vector: chunk.embedding,
+            }));
+            if (data.length > 0) {
+                await this.table.add(data);
+            }
+        }
+        // Update file metadata
+        const allCurrentFiles = [...changes.unchanged, ...changes.added, ...changes.modified];
+        await this.saveFileMetadata(allCurrentFiles);
+        const totalChunks = await this.table.countRows();
+        // Save index metadata with checksum
+        await this.saveIndexMetadata(allCurrentFiles.length, totalChunks, allCurrentFiles);
+        return {
+            filesIndexed: filesToProcess.length,
+            chunksCreated: totalChunks,
+            incremental: true,
+        };
+    }
+    /**
+     * Generate embeddings for chunks in batches
+     */
+    async embedChunks(chunks, onProgress) {
+        const report = (progress) => {
+            console.error(`[lance-context] ${progress.message}`);
+            onProgress?.(progress);
+        };
+        const batchSize = 32;
+        for (let i = 0; i < chunks.length; i += batchSize) {
+            const batch = chunks.slice(i, i + batchSize);
+            const texts = batch.map((c) => c.content);
+            const embeddings = await this.embeddingBackend.embedBatch(texts);
+            batch.forEach((chunk, idx) => {
+                chunk.embedding = embeddings[idx];
+            });
+            report({
+                phase: 'embedding',
+                current: i + batch.length,
+                total: chunks.length,
+                message: `Embedded ${i + batch.length}/${chunks.length} chunks`,
+            });
+        }
+    }
+    async chunkFile(filepath) {
+        const ext = path.extname(filepath).slice(1);
         const language = this.getLanguage(ext);
-        const relativePath = path.relative(this.projectPath, filePath);
+        const relativePath = path.relative(this.projectPath, filepath);
+        // Try AST-aware chunking for TypeScript/JavaScript
+        if (ASTChunker.canParse(filepath)) {
+            try {
+                return await this.chunkFileWithAST(filepath, relativePath, language);
+            }
+            catch {
+                // Fall back to line-based chunking if AST parsing fails
+                console.error(`[lance-context] AST parsing failed for ${relativePath}, falling back to line-based chunking`);
+            }
+        }
+        // Try tree-sitter chunking for other languages (Python, Go, Rust, Java, Kotlin)
+        if (TreeSitterChunker.canParse(filepath)) {
+            try {
+                return await this.chunkFileWithTreeSitter(filepath, relativePath, language);
+            }
+            catch (error) {
+                // Fall back to line-based chunking if tree-sitter parsing fails
+                console.error(`[lance-context] Tree-sitter parsing failed for ${relativePath}, falling back to line-based chunking:`, error);
+            }
+        }
+        // Line-based chunking for unsupported languages or as fallback
+        return this.chunkFileByLines(filepath, relativePath, language);
+    }
+    /**
+     * Chunk a file using AST-aware parsing
+     */
+    async chunkFileWithAST(filepath, relativePath, language) {
+        const astChunker = new ASTChunker();
+        const astChunks = await astChunker.chunkFile(filepath);
+        return astChunks.map((chunk) => ({
+            id: `${relativePath}:${chunk.startLine}-${chunk.endLine}${chunk.name ? `:${chunk.name}` : ''}`,
+            filepath: relativePath,
+            content: chunk.content,
+            startLine: chunk.startLine,
+            endLine: chunk.endLine,
+            language,
+            symbolType: chunk.type,
+            symbolName: chunk.name,
+        }));
+    }
+    /**
+     * Chunk a file using tree-sitter AST parsing (Python, Go, Rust, Java, Kotlin)
+     */
+    async chunkFileWithTreeSitter(filepath, relativePath, language) {
+        const treeSitterChunker = new TreeSitterChunker();
+        const treeSitterChunks = await treeSitterChunker.chunkFile(filepath);
+        return treeSitterChunks.map((chunk) => ({
+            id: `${relativePath}:${chunk.startLine}-${chunk.endLine}${chunk.name ? `:${chunk.name}` : ''}`,
+            filepath: relativePath,
+            content: chunk.content,
+            startLine: chunk.startLine,
+            endLine: chunk.endLine,
+            language,
+            symbolType: chunk.type,
+            symbolName: chunk.name,
+        }));
+    }
+    /**
+     * Chunk a file using line-based splitting (fallback)
+     */
+    async chunkFileByLines(filepath, relativePath, language) {
+        const content = await fs.readFile(filepath, 'utf-8');
+        const lines = content.split('\n');
+        const chunkingConfig = getChunkingConfig(this.config);
+        const chunkSize = chunkingConfig.maxLines;
+        const chunkOverlap = chunkingConfig.overlap;
         const chunks = [];
-        for (let i = 0; i < lines.length; i += CHUNK_SIZE - CHUNK_OVERLAP) {
-            const chunkLines = lines.slice(i, i + CHUNK_SIZE);
+        for (let i = 0; i < lines.length; i += chunkSize - chunkOverlap) {
+            const chunkLines = lines.slice(i, i + chunkSize);
             const chunkContent = chunkLines.join('\n');
             if (chunkContent.trim().length === 0)
                 continue;
             chunks.push({
                 id: `${relativePath}:${i + 1}-${i + chunkLines.length}`,
-                filePath: relativePath,
+                filepath: relativePath,
                 content: chunkContent,
                 startLine: i + 1,
                 endLine: i + chunkLines.length,
@@ -144,30 +643,427 @@ export class CodeIndexer {
         };
         return langMap[ext] || ext;
     }
-    async search(query, limit = 10) {
+    /**
+     * Get query embedding from cache or compute it.
+     * Uses LRU eviction when cache is full.
+     */
+    async getQueryEmbedding(query) {
+        // Check cache first
+        const cached = this.queryEmbeddingCache.get(query);
+        if (cached) {
+            // Move to end for LRU (delete and re-insert)
+            this.queryEmbeddingCache.delete(query);
+            this.queryEmbeddingCache.set(query, cached);
+            return cached;
+        }
+        // Compute embedding
+        const embedding = await this.embeddingBackend.embed(query);
+        // Evict oldest entry if cache is full (first entry in Map)
+        if (this.queryEmbeddingCache.size >= QUERY_CACHE_MAX_SIZE) {
+            const oldestKey = this.queryEmbeddingCache.keys().next().value;
+            if (oldestKey !== undefined) {
+                this.queryEmbeddingCache.delete(oldestKey);
+            }
+        }
+        // Store in cache
+        this.queryEmbeddingCache.set(query, embedding);
+        return embedding;
+    }
+    /**
+     * Check if a filepath matches a glob pattern.
+     * Supports negation patterns starting with '!'.
+     */
+    matchesPathPattern(filepath, pattern) {
+        // Handle negation pattern
+        if (pattern.startsWith('!')) {
+            return !minimatch(filepath, pattern.slice(1));
+        }
+        return minimatch(filepath, pattern);
+    }
+    async search(queryOrOptions, limit) {
+        // Normalize arguments
+        const options = typeof queryOrOptions === 'string'
+            ? { query: queryOrOptions, limit: limit ?? 10 }
+            : queryOrOptions;
+        const { query, limit: resultLimit = 10, pathPattern, languages } = options;
         if (!this.table) {
             const status = await this.getStatus();
             if (!status.indexed) {
                 throw new Error('Codebase not indexed. Run index_codebase first.');
             }
         }
-        const queryEmbedding = await this.embeddingBackend.embed(query);
-        const results = await this.table.search(queryEmbedding).limit(limit).toArray();
-        return results.map((r) => ({
-            id: r.id,
-            filePath: r.filePath,
-            content: r.content,
-            startLine: r.startLine,
-            endLine: r.endLine,
-            language: r.language,
+        const queryEmbedding = await this.getQueryEmbedding(query);
+        const searchConfig = getSearchConfig(this.config);
+        // Fetch more results than needed for re-ranking and filtering
+        // If we have filters, fetch even more to account for filtered-out results
+        const hasFilters = pathPattern !== undefined || (languages && languages.length > 0);
+        const fetchMultiplier = hasFilters ? 5 : 3;
+        const fetchLimit = Math.min(resultLimit * fetchMultiplier, hasFilters ? 100 : 50);
+        const results = await this.table.search(queryEmbedding).limit(fetchLimit).toArray();
+        // Apply filters
+        let filteredResults = results;
+        if (pathPattern) {
+            filteredResults = filteredResults.filter((r) => this.matchesPathPattern(r.filepath, pathPattern));
+        }
+        if (languages && languages.length > 0) {
+            const normalizedLanguages = languages.map((l) => l.toLowerCase());
+            filteredResults = filteredResults.filter((r) => normalizedLanguages.includes(r.language.toLowerCase()));
+        }
+        // Hybrid scoring: combine semantic similarity with keyword matching
+        const scoredResults = filteredResults.map((r, index) => {
+            // Semantic score: inverse of rank (higher is better)
+            const semanticScore = 1 - index / fetchLimit;
+            // Keyword score: based on query term matches
+            const keywordScore = this.calculateKeywordScore(query, r.content, r.filepath);
+            // Combined score using configurable weights
+            const combinedScore = searchConfig.semanticWeight * semanticScore + searchConfig.keywordWeight * keywordScore;
+            return { result: r, score: combinedScore };
+        });
+        // Sort by combined score and take top results
+        scoredResults.sort((a, b) => b.score - a.score);
+        return scoredResults.slice(0, resultLimit).map((sr) => ({
+            id: sr.result.id,
+            filepath: sr.result.filepath,
+            content: sr.result.content,
+            startLine: sr.result.startLine,
+            endLine: sr.result.endLine,
+            language: sr.result.language,
+            symbolType: sr.result.symbolType,
+            symbolName: sr.result.symbolName,
         }));
     }
+    /**
+     * Calculate keyword match score for hybrid search
+     */
+    calculateKeywordScore(query, content, filepath) {
+        const queryTerms = query
+            .toLowerCase()
+            .split(/\s+/)
+            .filter((t) => t.length > 2);
+        if (queryTerms.length === 0)
+            return 0;
+        const contentLower = content.toLowerCase();
+        const filepathLower = filepath.toLowerCase();
+        let matchCount = 0;
+        let exactMatchBonus = 0;
+        for (const term of queryTerms) {
+            // Check content matches
+            if (contentLower.includes(term)) {
+                matchCount++;
+                // Bonus for exact word match (not just substring)
+                const wordBoundaryRegex = new RegExp(`\\b${term}\\b`, 'i');
+                if (wordBoundaryRegex.test(content)) {
+                    exactMatchBonus += 0.5;
+                }
+            }
+            // Bonus for filename/path match
+            if (filepathLower.includes(term)) {
+                matchCount += 0.5;
+            }
+        }
+        // Normalize score to 0-1 range
+        const baseScore = matchCount / queryTerms.length;
+        const bonusScore = Math.min(exactMatchBonus / queryTerms.length, 0.5);
+        return Math.min(baseScore + bonusScore, 1);
+    }
+    /**
+     * Find code chunks semantically similar to a given code snippet or file location.
+     * This is useful for finding duplicate logic, similar implementations, or related code.
+     */
+    async searchSimilar(options) {
+        const { filepath, startLine, endLine, code, limit = 10, threshold = 0, excludeSelf = true, } = options;
+        // Validate input first - need either code or filepath
+        if (!code && !filepath) {
+            throw new Error('Either code or filepath must be provided');
+        }
+        if (!this.table) {
+            const status = await this.getStatus();
+            if (!status.indexed) {
+                throw new Error('Codebase not indexed. Run index_codebase first.');
+            }
+        }
+        // Get the source code to find similar chunks for
+        let sourceCode;
+        let sourceId = null;
+        if (code) {
+            sourceCode = code;
+        }
+        else {
+            // Read from file
+            const fullPath = path.join(this.projectPath, filepath);
+            const fileContent = await fs.readFile(fullPath, 'utf-8');
+            const lines = fileContent.split('\n');
+            const start = startLine ? startLine - 1 : 0;
+            const end = endLine ? endLine : lines.length;
+            sourceCode = lines.slice(start, end).join('\n');
+            // Build source ID for exclusion
+            if (startLine && endLine) {
+                sourceId = `${filepath}:${startLine}-${endLine}`;
+            }
+        }
+        if (!sourceCode.trim()) {
+            throw new Error('Source code is empty');
+        }
+        // Embed the source code
+        const sourceEmbedding = await this.embeddingBackend.embed(sourceCode);
+        // Search for similar chunks - fetch extra to account for filtering
+        const fetchLimit = Math.min((limit + 5) * 2, 100);
+        const results = await this.table.search(sourceEmbedding).limit(fetchLimit).toArray();
+        // LanceDB returns results sorted by distance (ascending)
+        // Convert distance to similarity score (1 - normalized_distance)
+        const maxDistance = results.length > 0 ? Math.max(...results.map((r) => r._distance || 0)) : 1;
+        const scoredResults = [];
+        for (const r of results) {
+            // Skip self if requested
+            if (excludeSelf && sourceId && r.id === sourceId) {
+                continue;
+            }
+            // Also skip if content is identical (for code-based search)
+            if (excludeSelf && code && r.content.trim() === code.trim()) {
+                continue;
+            }
+            // Convert distance to similarity (0 = far, 1 = identical)
+            const distance = r._distance || 0;
+            const similarity = maxDistance > 0 ? 1 - distance / maxDistance : 1;
+            // Apply threshold filter
+            if (similarity < threshold) {
+                continue;
+            }
+            scoredResults.push({
+                id: r.id,
+                filepath: r.filepath,
+                content: r.content,
+                startLine: r.startLine,
+                endLine: r.endLine,
+                language: r.language,
+                similarity,
+                symbolType: r.symbolType,
+                symbolName: r.symbolName,
+            });
+            if (scoredResults.length >= limit) {
+                break;
+            }
+        }
+        return scoredResults;
+    }
     async clearIndex() {
         const tableNames = await this.db.tableNames();
         if (tableNames.includes('code_chunks')) {
             await this.db.dropTable('code_chunks');
         }
         this.table = null;
+        // Clear query embedding cache to prevent stale embeddings
+        this.queryEmbeddingCache.clear();
+        // Clear clustering metadata
+        await this.clearClusteringMetadata();
+    }
+    get clusteringMetadataPath() {
+        return path.join(this.indexPath, 'clustering-metadata.json');
+    }
+    /**
+     * Clear clustering metadata file
+     */
+    async clearClusteringMetadata() {
+        try {
+            await fs.unlink(this.clusteringMetadataPath);
+        }
+        catch {
+            // Ignore errors if file doesn't exist
+        }
+    }
+    /**
+     * Save clustering result to metadata file
+     */
+    async saveClusteringMetadata(result) {
+        await fs.mkdir(this.indexPath, { recursive: true });
+        const data = {
+            clusterCount: result.clusterCount,
+            clusters: result.clusters,
+            // Convert Map to object for JSON serialization
+            assignments: Object.fromEntries(result.assignments),
+            generatedAt: new Date().toISOString(),
+        };
+        await fs.writeFile(this.clusteringMetadataPath, JSON.stringify(data, null, 2));
+    }
+    /**
+     * Load clustering result from metadata file
+     */
+    async loadClusteringMetadata() {
+        try {
+            const content = await fs.readFile(this.clusteringMetadataPath, 'utf-8');
+            const data = JSON.parse(content);
+            return {
+                clusterCount: data.clusterCount,
+                clusters: data.clusters,
+                // Convert object back to Map
+                assignments: new Map(Object.entries(data.assignments).map(([k, v]) => [k, v])),
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Cluster the indexed codebase into semantic concept areas.
+     * Uses k-means clustering on embeddings to discover related code groups.
+     */
+    async clusterConcepts(options = {}) {
+        if (!this.table) {
+            const status = await this.getStatus();
+            if (!status.indexed) {
+                throw new Error('Codebase not indexed. Run index_codebase first.');
+            }
+            this.table = await this.db.openTable('code_chunks');
+        }
+        // Fetch all chunks with embeddings
+        const rows = await this.table.query().toArray();
+        const chunks = rows.map((row) => ({
+            id: row.id,
+            content: row.content,
+            filepath: row.filepath,
+            embedding: row.vector,
+            symbolName: row.symbolName,
+            symbolType: row.symbolType,
+        }));
+        // Perform clustering
+        const result = kMeansClustering(chunks, options);
+        // Save to metadata file
+        await this.saveClusteringMetadata(result);
+        return result;
+    }
+    /**
+     * List all discovered concept clusters.
+     * Returns cached clustering result if available, otherwise clusters first.
+     */
+    async listConcepts(forceRecluster = false) {
+        if (!forceRecluster) {
+            const cached = await this.loadClusteringMetadata();
+            if (cached) {
+                return cached.clusters;
+            }
+        }
+        const result = await this.clusterConcepts();
+        return result.clusters;
+    }
+    /**
+     * Search for code within a specific concept cluster.
+     * Returns chunks that belong to the specified cluster, optionally filtered by query.
+     */
+    async searchByConcept(conceptId, query, limit = 10) {
+        const clustering = await this.loadClusteringMetadata();
+        if (!clustering) {
+            throw new Error('No clustering data available. Run clusterConcepts first.');
+        }
+        // Get chunk IDs in this cluster
+        const chunkIds = new Set();
+        for (const [chunkId, clusterId] of clustering.assignments) {
+            if (clusterId === conceptId) {
+                chunkIds.add(chunkId);
+            }
+        }
+        if (chunkIds.size === 0) {
+            return [];
+        }
+        if (!this.table) {
+            this.table = await this.db.openTable('code_chunks');
+        }
+        // If query provided, use semantic search and filter to cluster
+        if (query) {
+            const queryEmbedding = await this.getQueryEmbedding(query);
+            const results = await this.table
+                .search(queryEmbedding)
+                .limit(limit * 3)
+                .toArray();
+            return results
+                .filter((r) => chunkIds.has(r.id))
+                .slice(0, limit)
+                .map((r) => ({
+                id: r.id,
+                filepath: r.filepath,
+                content: r.content,
+                startLine: r.startLine,
+                endLine: r.endLine,
+                language: r.language,
+                symbolType: r.symbolType,
+                symbolName: r.symbolName,
+            }));
+        }
+        // Without query, return representative chunks from the cluster
+        const cluster = clustering.clusters.find((c) => c.id === conceptId);
+        if (!cluster) {
+            return [];
+        }
+        const results = [];
+        for (const chunkId of cluster.representativeChunks.slice(0, limit)) {
+            // Fetch chunk by ID - LanceDB doesn't have direct ID lookup, so we filter
+            const rows = await this.table.query().where(`id = '${chunkId}'`).limit(1).toArray();
+            if (rows.length > 0) {
+                const r = rows[0];
+                results.push({
+                    id: r.id,
+                    filepath: r.filepath,
+                    content: r.content,
+                    startLine: r.startLine,
+                    endLine: r.endLine,
+                    language: r.language,
+                    symbolType: r.symbolType,
+                    symbolName: r.symbolName,
+                });
+            }
+        }
+        return results;
+    }
+    /**
+     * Generate a comprehensive summary of the codebase including concept areas.
+     */
+    async summarizeCodebase(clusteringOptions) {
+        const status = await this.getStatus();
+        if (!status.indexed) {
+            throw new Error('Codebase not indexed. Run index_codebase first.');
+        }
+        if (!this.table) {
+            this.table = await this.db.openTable('code_chunks');
+        }
+        // Gather language statistics
+        const rows = await this.table.query().toArray();
+        const languageStats = new Map();
+        for (const row of rows) {
+            const lang = row.language;
+            if (!languageStats.has(lang)) {
+                languageStats.set(lang, { fileCount: new Set(), chunkCount: 0 });
+            }
+            const stats = languageStats.get(lang);
+            stats.fileCount.add(row.filepath);
+            stats.chunkCount++;
+        }
+        const languages = Array.from(languageStats.entries())
+            .map(([language, stats]) => ({
+            language,
+            fileCount: stats.fileCount.size,
+            chunkCount: stats.chunkCount,
+        }))
+            .sort((a, b) => b.chunkCount - a.chunkCount);
+        // Perform clustering
+        const chunks = rows.map((row) => ({
+            id: row.id,
+            content: row.content,
+            filepath: row.filepath,
+            embedding: row.vector,
+            symbolName: row.symbolName,
+            symbolType: row.symbolType,
+        }));
+        const clusteringResult = kMeansClustering(chunks, clusteringOptions);
+        await this.saveClusteringMetadata(clusteringResult);
+        // Calculate clustering quality
+        const silhouetteScore = calculateSilhouetteScore(chunks, clusteringResult.assignments, clusteringResult.clusters);
+        return {
+            totalFiles: status.fileCount,
+            totalChunks: status.chunkCount,
+            languages,
+            concepts: clusteringResult.clusters,
+            clusteringQuality: silhouetteScore,
+            generatedAt: new Date().toISOString(),
+        };
     }
 }
 //# sourceMappingURL=indexer.js.map