@mcampa/ai-context-core 0.0.1-beta.05e8984

package/dist/context.js

@@ -0,0 +1,1177 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Context = void 0;
+const splitter_1 = require("./splitter");
+const embedding_1 = require("./embedding");
+const env_manager_1 = require("./utils/env-manager");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const crypto = __importStar(require("crypto"));
+const synchronizer_1 = require("./sync/synchronizer");
+const DEFAULT_SUPPORTED_EXTENSIONS = [
+    // Programming languages
+    ".ts",
+    ".tsx",
+    ".js",
+    ".jsx",
+    ".py",
+    ".java",
+    ".cpp",
+    ".c",
+    ".h",
+    ".hpp",
+    ".cs",
+    ".go",
+    ".rs",
+    ".php",
+    ".rb",
+    ".swift",
+    ".kt",
+    ".scala",
+    ".m",
+    ".mm",
+    // Text and markup files
+    ".md",
+    ".markdown",
+    ".ipynb",
+    // '.txt',  '.json', '.yaml', '.yml', '.xml', '.html', '.htm',
+    // '.css', '.scss', '.less', '.sql', '.sh', '.bash', '.env'
+];
+const DEFAULT_IGNORE_PATTERNS = [
+    // Common build output and dependency directories
+    "node_modules/**",
+    "dist/**",
+    "build/**",
+    "out/**",
+    "target/**",
+    "coverage/**",
+    ".nyc_output/**",
+    // IDE and editor files
+    ".vscode/**",
+    ".idea/**",
+    "*.swp",
+    "*.swo",
+    // Version control
+    ".git/**",
+    ".svn/**",
+    ".hg/**",
+    // Cache directories
+    ".cache/**",
+    "__pycache__/**",
+    ".pytest_cache/**",
+    // Logs and temporary files
+    "logs/**",
+    "tmp/**",
+    "temp/**",
+    "*.log",
+    // Environment and config files
+    ".env",
+    ".env.*",
+    "*.local",
+    // Minified and bundled files
+    "*.min.js",
+    "*.min.css",
+    "*.min.map",
+    "*.bundle.js",
+    "*.bundle.css",
+    "*.chunk.js",
+    "*.vendor.js",
+    "*.polyfills.js",
+    "*.runtime.js",
+    "*.map", // source map files
+    "node_modules",
+    ".git",
+    ".svn",
+    ".hg",
+    "build",
+    "dist",
+    "out",
+    "target",
+    ".vscode",
+    ".idea",
+    "__pycache__",
+    ".pytest_cache",
+    "coverage",
+    ".nyc_output",
+    "logs",
+    "tmp",
+    "temp",
+];
+class Context {
+    constructor(config) {
+        this.synchronizers = new Map();
+        this.name = config.name || "my-context";
+        // Initialize services
+        this.embedding =
+            config.embedding ||
+                new embedding_1.OpenAIEmbedding({
+                    apiKey: env_manager_1.envManager.get("OPENAI_API_KEY") || "your-openai-api-key",
+                    model: "text-embedding-3-small",
+                    ...(env_manager_1.envManager.get("OPENAI_BASE_URL") && {
+                        baseURL: env_manager_1.envManager.get("OPENAI_BASE_URL"),
+                    }),
+                });
+        if (!config.vectorDatabase) {
+            throw new Error("VectorDatabase is required. Please provide a vectorDatabase instance in the config.");
+        }
+        this.vectorDatabase = config.vectorDatabase;
+        this.codeSplitter = config.codeSplitter || new splitter_1.AstCodeSplitter(2500, 300);
+        // Load custom extensions from environment variables
+        const envCustomExtensions = this.getCustomExtensionsFromEnv();
+        // If supportedExtensions is provided, use it as the base (not defaults)
+        // Otherwise use defaults
+        const baseExtensions = config.supportedExtensions || DEFAULT_SUPPORTED_EXTENSIONS;
+        const allSupportedExtensions = [
+            ...baseExtensions,
+            ...(config.customExtensions || []),
+            ...envCustomExtensions,
+        ];
+        // Remove duplicates
+        this.supportedExtensions = [...new Set(allSupportedExtensions)];
+        // Load custom ignore patterns from environment variables
+        const envCustomIgnorePatterns = this.getCustomIgnorePatternsFromEnv();
+        // If ignorePatterns is provided, use it as the base (not defaults)
+        // Otherwise use defaults
+        const baseIgnorePatterns = config.ignorePatterns || DEFAULT_IGNORE_PATTERNS;
+        const allIgnorePatterns = [
+            ...baseIgnorePatterns,
+            ...(config.customIgnorePatterns || []),
+            ...envCustomIgnorePatterns,
+        ];
+        // Remove duplicates
+        this.ignorePatterns = [...new Set(allIgnorePatterns)];
+        console.log(`[Context] 🔧 Initialized with ${this.supportedExtensions.length} supported extensions and ${this.ignorePatterns.length} ignore patterns`);
+        if (envCustomExtensions.length > 0) {
+            console.log(`[Context] 📎 Loaded ${envCustomExtensions.length} custom extensions from environment: ${envCustomExtensions.join(", ")}`);
+        }
+        if (envCustomIgnorePatterns.length > 0) {
+            console.log(`[Context] 🚫 Loaded ${envCustomIgnorePatterns.length} custom ignore patterns from environment: ${envCustomIgnorePatterns.join(", ")}`);
+        }
+    }
+    /**
+     * Get embedding instance
+     */
+    getEmbedding() {
+        return this.embedding;
+    }
+    /**
+     * Get vector database instance
+     */
+    getVectorDatabase() {
+        return this.vectorDatabase;
+    }
+    /**
+     * Get code splitter instance
+     */
+    getCodeSplitter() {
+        return this.codeSplitter;
+    }
+    /**
+     * Get supported extensions
+     */
+    getSupportedExtensions() {
+        return [...this.supportedExtensions];
+    }
+    /**
+     * Get ignore patterns
+     */
+    getIgnorePatterns() {
+        return [...this.ignorePatterns];
+    }
+    /**
+     * Get synchronizers map
+     */
+    getSynchronizers() {
+        return new Map(this.synchronizers);
+    }
+    /**
+     * Set synchronizer for a collection
+     */
+    setSynchronizer(collectionName, synchronizer) {
+        this.synchronizers.set(collectionName, synchronizer);
+    }
+    /**
+     * Public wrapper for loadIgnorePatterns private method
+     */
+    async getLoadedIgnorePatterns(codebasePath) {
+        return this.loadIgnorePatterns(codebasePath);
+    }
+    /**
+     * Public wrapper for prepareCollection private method
+     */
+    async getPreparedCollection(codebasePath) {
+        return this.prepareCollection(codebasePath);
+    }
+    /**
+     * Get isHybrid setting from environment variable with default true
+     */
+    getIsHybrid() {
+        const isHybridEnv = env_manager_1.envManager.get("HYBRID_MODE");
+        if (isHybridEnv === undefined || isHybridEnv === null) {
+            return true; // Default to true
+        }
+        return isHybridEnv.toLowerCase() === "true";
+    }
+    /**
+     * Generate collection name based on codebase path and hybrid mode
+     */
+    getCollectionName() {
+        const isHybrid = this.getIsHybrid();
+        const prefix = isHybrid === true ? "hybrid_code_chunks" : "code_chunks";
+        return `${prefix}_${this.name}`;
+    }
+    /**
+     * Index a codebase for semantic search
+     * @param codebasePath Codebase root path
+     * @param progressCallback Optional progress callback function
+     * @param forceReindex Whether to recreate the collection even if it exists
+     * @returns Indexing statistics
+     */
+    async indexCodebase(codebasePath, progressCallback, forceReindex = false) {
+        const isHybrid = this.getIsHybrid();
+        const searchType = isHybrid === true ? "hybrid search" : "semantic search";
+        console.log(`[Context] 🚀 Starting to index codebase with ${searchType}: ${codebasePath}`);
+        // 1. Load ignore patterns from various ignore files
+        await this.loadIgnorePatterns(codebasePath);
+        // 2. Check and prepare vector collection
+        progressCallback?.({
+            phase: "Preparing collection...",
+            current: 0,
+            total: 100,
+            percentage: 0,
+        });
+        console.log(`Debug2: Preparing vector collection for codebase${forceReindex ? " (FORCE REINDEX)" : ""}`);
+        await this.prepareCollection(codebasePath, forceReindex);
+        // 3. Recursively traverse codebase to get all supported files
+        progressCallback?.({
+            phase: "Scanning files...",
+            current: 5,
+            total: 100,
+            percentage: 5,
+        });
+        const codeFiles = await this.getCodeFiles(codebasePath);
+        console.log(`[Context] 📁 Found ${codeFiles.length} code files`);
+        if (codeFiles.length === 0) {
+            progressCallback?.({
+                phase: "No files to index",
+                current: 100,
+                total: 100,
+                percentage: 100,
+            });
+            return { indexedFiles: 0, totalChunks: 0, status: "completed" };
+        }
+        // 3. Process each file with streaming chunk processing
+        // Reserve 10% for preparation, 90% for actual indexing
+        const indexingStartPercentage = 10;
+        const indexingEndPercentage = 100;
+        const indexingRange = indexingEndPercentage - indexingStartPercentage;
+        const result = await this.processFileList(codeFiles, codebasePath, (filePath, fileIndex, totalFiles) => {
+            // Calculate progress percentage
+            const progressPercentage = indexingStartPercentage + (fileIndex / totalFiles) * indexingRange;
+            console.log(`[Context] 📊 Processed ${fileIndex}/${totalFiles} files`);
+            progressCallback?.({
+                phase: `Processing files (${fileIndex}/${totalFiles})...`,
+                current: fileIndex,
+                total: totalFiles,
+                percentage: Math.round(progressPercentage),
+            });
+        });
+        console.log(`[Context] ✅ Codebase indexing completed! Processed ${result.processedFiles} files in total, generated ${result.totalChunks} code chunks`);
+        progressCallback?.({
+            phase: "Indexing complete!",
+            current: result.processedFiles,
+            total: codeFiles.length,
+            percentage: 100,
+        });
+        return {
+            indexedFiles: result.processedFiles,
+            totalChunks: result.totalChunks,
+            status: result.status,
+        };
+    }
+    async reindexByChange(codebasePath, progressCallback) {
+        const collectionName = this.getCollectionName();
+        const synchronizer = this.synchronizers.get(collectionName);
+        if (!synchronizer) {
+            // Load project-specific ignore patterns before creating FileSynchronizer
+            await this.loadIgnorePatterns(codebasePath);
+            // To be safe, let's initialize if it's not there.
+            const newSynchronizer = new synchronizer_1.FileSynchronizer(codebasePath, this.ignorePatterns);
+            await newSynchronizer.initialize();
+            this.synchronizers.set(collectionName, newSynchronizer);
+        }
+        const currentSynchronizer = this.synchronizers.get(collectionName);
+        progressCallback?.({
+            phase: "Checking for file changes...",
+            current: 0,
+            total: 100,
+            percentage: 0,
+        });
+        const { added, removed, modified } = await currentSynchronizer.checkForChanges();
+        const totalChanges = added.length + removed.length + modified.length;
+        if (totalChanges === 0) {
+            progressCallback?.({
+                phase: "No changes detected",
+                current: 100,
+                total: 100,
+                percentage: 100,
+            });
+            console.log("[Context] ✅ No file changes detected.");
+            return { added: 0, removed: 0, modified: 0 };
+        }
+        console.log(`[Context] 🔄 Found changes: ${added.length} added, ${removed.length} removed, ${modified.length} modified.`);
+        let processedChanges = 0;
+        const updateProgress = (phase) => {
+            processedChanges++;
+            const percentage = Math.round((processedChanges / (removed.length + modified.length + added.length)) *
+                100);
+            progressCallback?.({
+                phase,
+                current: processedChanges,
+                total: totalChanges,
+                percentage,
+            });
+        };
+        // Handle removed files
+        for (const file of removed) {
+            await this.deleteFileChunks(collectionName, file);
+            updateProgress(`Removed ${file}`);
+        }
+        // Handle modified files
+        for (const file of modified) {
+            await this.deleteFileChunks(collectionName, file);
+            updateProgress(`Deleted old chunks for ${file}`);
+        }
+        // Handle added and modified files
+        const filesToIndex = [...added, ...modified].map((f) => path.join(codebasePath, f));
+        if (filesToIndex.length > 0) {
+            await this.processFileList(filesToIndex, codebasePath, (filePath, fileIndex, totalFiles) => {
+                updateProgress(`Indexed ${filePath} (${fileIndex}/${totalFiles})`);
+            });
+        }
+        console.log(`[Context] ✅ Re-indexing complete. Added: ${added.length}, Removed: ${removed.length}, Modified: ${modified.length}`);
+        progressCallback?.({
+            phase: "Re-indexing complete!",
+            current: totalChanges,
+            total: totalChanges,
+            percentage: 100,
+        });
+        return {
+            added: added.length,
+            removed: removed.length,
+            modified: modified.length,
+        };
+    }
+    async deleteFileChunks(collectionName, relativePath) {
+        // Escape backslashes for Milvus query expression (Windows path compatibility)
+        const escapedPath = relativePath.replace(/\\/g, "\\\\");
+        const results = await this.vectorDatabase.query(collectionName, `relativePath == "${escapedPath}"`, ["id"]);
+        if (results.length > 0) {
+            const ids = results.map((r) => r.id).filter((id) => id);
+            if (ids.length > 0) {
+                await this.vectorDatabase.delete(collectionName, ids);
+                console.log(`[Context] Deleted ${ids.length} chunks for file ${relativePath}`);
+            }
+        }
+    }
+    /**
+     * Semantic search with unified implementation
+     * @param query Search query
+     * @param topK Number of results to return
+     * @param threshold Similarity threshold
+     */
+    async semanticSearch(query, topK = 5, threshold = 0.5, filterExpr) {
+        const isHybrid = this.getIsHybrid();
+        const searchType = isHybrid === true ? "hybrid search" : "semantic search";
+        console.log(`[Context] 🔍 Executing ${searchType}: "${query}"`);
+        const collectionName = this.getCollectionName();
+        console.log(`[Context] 🔍 Using collection: ${collectionName}`);
+        // Check if collection exists and has data
+        const hasCollection = await this.vectorDatabase.hasCollection(collectionName);
+        if (!hasCollection) {
+            console.log(`[Context] ⚠️  Collection '${collectionName}' does not exist. Please index the codebase first.`);
+            return [];
+        }
+        if (isHybrid === true) {
+            try {
+                // Check collection stats to see if it has data
+                const _stats = await this.vectorDatabase.query(collectionName, "", ["id"], 1);
+                console.log(`[Context] 🔍 Collection '${collectionName}' exists and appears to have data`);
+            }
+            catch (error) {
+                console.log(`[Context] ⚠️  Collection '${collectionName}' exists but may be empty or not properly indexed:`, error);
+            }
+            // 1. Generate query vector
+            console.log(`[Context] 🔍 Generating embeddings for query: "${query}"`);
+            const queryEmbedding = await this.embedding.embed(query);
+            console.log(`[Context] ✅ Generated embedding vector with dimension: ${queryEmbedding.vector.length}`);
+            console.log(`[Context] 🔍 First 5 embedding values: [${queryEmbedding.vector.slice(0, 5).join(", ")}]`);
+            // 2. Prepare hybrid search requests
+            const searchRequests = [
+                {
+                    data: queryEmbedding.vector,
+                    anns_field: "vector",
+                    param: { nprobe: 10 },
+                    limit: topK,
+                },
+                {
+                    data: query,
+                    anns_field: "sparse_vector",
+                    param: { drop_ratio_search: 0.2 },
+                    limit: topK,
+                },
+            ];
+            console.log(`[Context] 🔍 Search request 1 (dense): anns_field="${searchRequests[0].anns_field}", vector_dim=${queryEmbedding.vector.length}, limit=${searchRequests[0].limit}`);
+            console.log(`[Context] 🔍 Search request 2 (sparse): anns_field="${searchRequests[1].anns_field}", query_text="${query}", limit=${searchRequests[1].limit}`);
+            // 3. Execute hybrid search
+            console.log(`[Context] 🔍 Executing hybrid search with RRF reranking...`);
+            const searchResults = await this.vectorDatabase.hybridSearch(collectionName, searchRequests, {
+                rerank: {
+                    strategy: "rrf",
+                    params: { k: 100 },
+                },
+                limit: topK,
+                filterExpr,
+            });
+            console.log(`[Context] 🔍 Raw search results count: ${searchResults.length}`);
+            // 4. Convert to semantic search result format
+            const results = searchResults.map((result) => ({
+                content: result.document.content,
+                relativePath: result.document.relativePath,
+                startLine: result.document.startLine,
+                endLine: result.document.endLine,
+                language: String(result.document.metadata.language || "unknown"),
+                score: result.score,
+            }));
+            console.log(`[Context] ✅ Found ${results.length} relevant hybrid results`);
+            if (results.length > 0) {
+                console.log(`[Context] 🔍 Top result score: ${results[0].score}, path: ${results[0].relativePath}`);
+            }
+            return results;
+        }
+        else {
+            // Regular semantic search
+            // 1. Generate query vector
+            const queryEmbedding = await this.embedding.embed(query);
+            // 2. Search in vector database
+            const searchResults = await this.vectorDatabase.search(collectionName, queryEmbedding.vector, { topK, threshold, filterExpr });
+            // 3. Convert to semantic search result format
+            const results = searchResults.map((result) => ({
+                content: result.document.content,
+                relativePath: result.document.relativePath,
+                startLine: result.document.startLine,
+                endLine: result.document.endLine,
+                language: String(result.document.metadata.language || "unknown"),
+                score: result.score,
+            }));
+            console.log(`[Context] ✅ Found ${results.length} relevant results`);
+            return results;
+        }
+    }
+    /**
+     * Check if index exists
+     * @returns Whether index exists
+     */
+    async hasIndex() {
+        const collectionName = this.getCollectionName();
+        return await this.vectorDatabase.hasCollection(collectionName);
+    }
+    /**
+     * Clear index
+     * @param codebasePath Codebase path to clear index for
+     * @param progressCallback Optional progress callback function
+     */
+    async clearIndex(codebasePath, progressCallback) {
+        console.log(`[Context] 🧹 Cleaning index data for ${codebasePath}...`);
+        progressCallback?.({
+            phase: "Checking existing index...",
+            current: 0,
+            total: 100,
+            percentage: 0,
+        });
+        const collectionName = this.getCollectionName();
+        const collectionExists = await this.vectorDatabase.hasCollection(collectionName);
+        progressCallback?.({
+            phase: "Removing index data...",
+            current: 50,
+            total: 100,
+            percentage: 50,
+        });
+        if (collectionExists) {
+            await this.vectorDatabase.dropCollection(collectionName);
+        }
+        // Delete snapshot file
+        await synchronizer_1.FileSynchronizer.deleteSnapshot(codebasePath);
+        progressCallback?.({
+            phase: "Index cleared",
+            current: 100,
+            total: 100,
+            percentage: 100,
+        });
+        console.log("[Context] ✅ Index data cleaned");
+    }
+    /**
+     * Update ignore patterns (merges with default patterns and existing patterns)
+     * @param ignorePatterns Array of ignore patterns to add to defaults
+     */
+    updateIgnorePatterns(ignorePatterns) {
+        // Merge with default patterns and any existing custom patterns, avoiding duplicates
+        const mergedPatterns = [...DEFAULT_IGNORE_PATTERNS, ...ignorePatterns];
+        const uniquePatterns = [];
+        const patternSet = new Set(mergedPatterns);
+        patternSet.forEach((pattern) => uniquePatterns.push(pattern));
+        this.ignorePatterns = uniquePatterns;
+        console.log(`[Context] 🚫 Updated ignore patterns: ${ignorePatterns.length} new + ${DEFAULT_IGNORE_PATTERNS.length} default = ${this.ignorePatterns.length} total patterns`);
+    }
+    /**
+     * Add custom ignore patterns (from MCP or other sources) without replacing existing ones
+     * @param customPatterns Array of custom ignore patterns to add
+     */
+    addCustomIgnorePatterns(customPatterns) {
+        if (customPatterns.length === 0)
+            return;
+        // Merge current patterns with new custom patterns, avoiding duplicates
+        const mergedPatterns = [...this.ignorePatterns, ...customPatterns];
+        const uniquePatterns = [];
+        const patternSet = new Set(mergedPatterns);
+        patternSet.forEach((pattern) => uniquePatterns.push(pattern));
+        this.ignorePatterns = uniquePatterns;
+        console.log(`[Context] 🚫 Added ${customPatterns.length} custom ignore patterns. Total: ${this.ignorePatterns.length} patterns`);
+    }
+    /**
+     * Reset ignore patterns to defaults only
+     */
+    resetIgnorePatternsToDefaults() {
+        this.ignorePatterns = [...DEFAULT_IGNORE_PATTERNS];
+        console.log(`[Context] 🔄 Reset ignore patterns to defaults: ${this.ignorePatterns.length} patterns`);
+    }
+    /**
+     * Update embedding instance
+     * @param embedding New embedding instance
+     */
+    updateEmbedding(embedding) {
+        this.embedding = embedding;
+        console.log(`[Context] 🔄 Updated embedding provider: ${embedding.getProvider()}`);
+    }
+    /**
+     * Update vector database instance
+     * @param vectorDatabase New vector database instance
+     */
+    updateVectorDatabase(vectorDatabase) {
+        this.vectorDatabase = vectorDatabase;
+        console.log(`[Context] 🔄 Updated vector database`);
+    }
+    /**
+     * Update splitter instance
+     * @param splitter New splitter instance
+     */
+    updateSplitter(splitter) {
+        this.codeSplitter = splitter;
+        console.log(`[Context] 🔄 Updated splitter instance`);
+    }
+    /**
+     * Prepare vector collection
+     */
+    async prepareCollection(codebasePath, forceReindex = false) {
+        const isHybrid = this.getIsHybrid();
+        const collectionType = isHybrid === true ? "hybrid vector" : "vector";
+        console.log(`[Context] 🔧 Preparing ${collectionType} collection for codebase: ${codebasePath}${forceReindex ? " (FORCE REINDEX)" : ""}`);
+        const collectionName = this.getCollectionName();
+        // Check if collection already exists
+        const collectionExists = await this.vectorDatabase.hasCollection(collectionName);
+        if (collectionExists && !forceReindex) {
+            console.log(`📋 Collection ${collectionName} already exists, skipping creation`);
+            return;
+        }
+        if (collectionExists && forceReindex) {
+            console.log(`[Context] 🗑️  Dropping existing collection ${collectionName} for force reindex...`);
+            await this.vectorDatabase.dropCollection(collectionName);
+            console.log(`[Context] ✅ Collection ${collectionName} dropped successfully`);
+        }
+        console.log(`[Context] 🔍 Detecting embedding dimension for ${this.embedding.getProvider()} provider...`);
+        const dimension = await this.embedding.detectDimension();
+        console.log(`[Context] 📏 Detected dimension: ${dimension} for ${this.embedding.getProvider()}`);
+        const dirName = path.basename(codebasePath);
+        if (isHybrid === true) {
+            await this.vectorDatabase.createHybridCollection(collectionName, dimension, `Hybrid Index for ${dirName}`);
+        }
+        else {
+            await this.vectorDatabase.createCollection(collectionName, dimension, `Index for ${dirName}`);
+        }
+        console.log(`[Context] ✅ Collection ${collectionName} created successfully (dimension: ${dimension})`);
+    }
+    /**
+     * Recursively get all code files in the codebase
+     */
+    async getCodeFiles(codebasePath) {
+        const files = [];
+        const traverseDirectory = async (currentPath) => {
+            const entries = await fs.promises.readdir(currentPath, {
+                withFileTypes: true,
+            });
+            for (const entry of entries) {
+                const fullPath = path.join(currentPath, entry.name);
+                // Check if path matches ignore patterns
+                if (this.matchesIgnorePattern(fullPath, codebasePath)) {
+                    continue;
+                }
+                if (entry.isDirectory()) {
+                    await traverseDirectory(fullPath);
+                }
+                else if (entry.isFile()) {
+                    const ext = path.extname(entry.name);
+                    if (this.supportedExtensions.includes(ext)) {
+                        files.push(fullPath);
+                    }
+                }
+            }
+        };
+        await traverseDirectory(codebasePath);
+        return files;
+    }
+    /**
+     * Process a list of files with streaming chunk processing
+     * @param filePaths Array of file paths to process
+     * @param codebasePath Base path for the codebase
+     * @param onFileProcessed Callback called when each file is processed
+     * @returns Object with processed file count and total chunk count
+     */
+    async processFileList(filePaths, codebasePath, onFileProcessed) {
+        const isHybrid = this.getIsHybrid();
+        const EMBEDDING_BATCH_SIZE = Math.max(1, parseInt(env_manager_1.envManager.get("EMBEDDING_BATCH_SIZE") || "100", 10));
+        const CHUNK_LIMIT = 450000;
+        console.log(`[Context] 🔧 Using EMBEDDING_BATCH_SIZE: ${EMBEDDING_BATCH_SIZE}`);
+        let chunkBuffer = [];
+        let processedFiles = 0;
+        let totalChunks = 0;
+        let limitReached = false;
+        for (let i = 0; i < filePaths.length; i++) {
+            const filePath = filePaths[i];
+            const relativeFilePath = path.relative(codebasePath, filePath);
+            try {
+                const content = await fs.promises.readFile(filePath, "utf-8");
+                const language = this.getLanguageFromExtension(path.extname(filePath));
+                const chunks = await this.codeSplitter.split(content, language, filePath);
+                // Log files with many chunks or large content
+                if (chunks.length > 50) {
+                    console.warn(`[Context] ⚠️  File ${relativeFilePath} generated ${chunks.length} chunks (${Math.round(content.length / 1024)}KB)`);
+                }
+                else if (content.length > 100000) {
+                    console.log(`📄 Large file ${relativeFilePath}: ${Math.round(content.length / 1024)}KB -> ${chunks.length} chunks`);
+                }
+                // Add chunks to buffer
+                for (const chunk of chunks) {
+                    chunkBuffer.push({ chunk, codebasePath });
+                    totalChunks++;
+                    // Process batch when buffer reaches EMBEDDING_BATCH_SIZE
+                    if (chunkBuffer.length >= EMBEDDING_BATCH_SIZE) {
+                        try {
+                            await this.processChunkBuffer(chunkBuffer);
+                        }
+                        catch (error) {
+                            const searchType = isHybrid === true ? "hybrid" : "regular";
+                            console.error(`[Context] ❌ Failed to process chunk batch for ${searchType}:`, error);
+                            if (error instanceof Error) {
+                                console.error("[Context] Stack trace:", error.stack);
+                            }
+                        }
+                        finally {
+                            chunkBuffer = []; // Always clear buffer, even on failure
+                        }
+                    }
+                    // Check if chunk limit is reached
+                    if (totalChunks >= CHUNK_LIMIT) {
+                        console.warn(`[Context] ⚠️  Chunk limit of ${CHUNK_LIMIT} reached. Stopping indexing.`);
+                        limitReached = true;
+                        break; // Exit the inner loop (over chunks)
+                    }
+                }
+                processedFiles++;
+                onFileProcessed?.(relativeFilePath, i + 1, filePaths.length);
+                if (limitReached) {
+                    break; // Exit the outer loop (over files)
+                }
+            }
+            catch (error) {
+                console.warn(`[Context] ⚠️  Skipping file ${relativeFilePath}: ${error}`);
+            }
+        }
+        // Process any remaining chunks in the buffer
+        if (chunkBuffer.length > 0) {
+            const searchType = isHybrid === true ? "hybrid" : "regular";
+            console.log(`📝 Processing final batch of ${chunkBuffer.length} chunks for ${searchType}`);
+            try {
+                await this.processChunkBuffer(chunkBuffer);
+            }
+            catch (error) {
+                console.error(`[Context] ❌ Failed to process final chunk batch for ${searchType}:`, error);
+                if (error instanceof Error) {
+                    console.error("[Context] Stack trace:", error.stack);
+                }
+            }
+        }
+        return {
+            processedFiles,
+            totalChunks,
+            status: limitReached ? "limit_reached" : "completed",
+        };
+    }
+    /**
+     * Process accumulated chunk buffer
+     */
+    async processChunkBuffer(chunkBuffer) {
+        if (chunkBuffer.length === 0)
+            return;
+        // Extract chunks and ensure they all have the same codebasePath
+        const chunks = chunkBuffer.map((item) => item.chunk);
+        const codebasePath = chunkBuffer[0].codebasePath;
+        // Estimate tokens (rough estimation: 1 token ≈ 4 characters)
+        const estimatedTokens = chunks.reduce((sum, chunk) => sum + Math.ceil(chunk.content.length / 4), 0);
+        const isHybrid = this.getIsHybrid();
+        const searchType = isHybrid === true ? "hybrid" : "regular";
+        console.log(`[Context] 🔄 Processing batch of ${chunks.length} chunks (~${estimatedTokens} tokens) for ${searchType}`);
+        await this.processChunkBatch(chunks, codebasePath);
+    }
+    /**
+     * Process a batch of chunks
+     */
+    async processChunkBatch(chunks, codebasePath) {
+        const isHybrid = this.getIsHybrid();
+        // Generate embedding vectors
+        const chunkContents = chunks.map((chunk) => chunk.content);
+        const embeddings = await this.embedding.embedBatch(chunkContents);
+        if (isHybrid === true) {
+            // Create hybrid vector documents
+            const documents = chunks.map((chunk, index) => {
+                if (!chunk.metadata.filePath) {
+                    throw new Error(`Missing filePath in chunk metadata at index ${index}`);
+                }
+                const relativePath = path.relative(codebasePath, chunk.metadata.filePath);
+                const fileExtension = path.extname(chunk.metadata.filePath);
+                const { filePath: _filePath, startLine: _startLine, endLine: _endLine, ...restMetadata } = chunk.metadata;
+                return {
+                    id: this.generateId(relativePath, chunk.metadata.startLine || 0, chunk.metadata.endLine || 0, chunk.content),
+                    content: chunk.content, // Full text content for BM25 and storage
+                    vector: embeddings[index].vector, // Dense vector
+                    relativePath,
+                    startLine: chunk.metadata.startLine || 0,
+                    endLine: chunk.metadata.endLine || 0,
+                    fileExtension,
+                    metadata: {
+                        ...restMetadata,
+                        contextName: this.name,
+                        codebasePath,
+                        language: chunk.metadata.language || "unknown",
+                        chunkIndex: index,
+                    },
+                };
+            });
+            // Store to vector database
+            await this.vectorDatabase.insertHybrid(this.getCollectionName(), documents);
+        }
+        else {
+            // Create regular vector documents
+            const documents = chunks.map((chunk, index) => {
+                if (!chunk.metadata.filePath) {
+                    throw new Error(`Missing filePath in chunk metadata at index ${index}`);
+                }
+                const relativePath = path.relative(codebasePath, chunk.metadata.filePath);
+                const fileExtension = path.extname(chunk.metadata.filePath);
+                const { filePath: _filePath, startLine: _startLine, endLine: _endLine, ...restMetadata } = chunk.metadata;
+                return {
+                    id: this.generateId(relativePath, chunk.metadata.startLine || 0, chunk.metadata.endLine || 0, chunk.content),
+                    vector: embeddings[index].vector,
+                    content: chunk.content,
+                    relativePath,
+                    startLine: chunk.metadata.startLine || 0,
+                    endLine: chunk.metadata.endLine || 0,
+                    fileExtension,
+                    metadata: {
+                        ...restMetadata,
+                        contextName: this.name,
+                        codebasePath,
+                        language: chunk.metadata.language || "unknown",
+                        chunkIndex: index,
+                    },
+                };
+            });
+            // Store to vector database
+            await this.vectorDatabase.insert(this.getCollectionName(), documents);
+        }
+    }
+    /**
+     * Get programming language based on file extension
+     */
+    getLanguageFromExtension(ext) {
+        const languageMap = {
+            ".ts": "typescript",
+            ".tsx": "typescript",
+            ".js": "javascript",
+            ".jsx": "javascript",
+            ".py": "python",
+            ".java": "java",
+            ".cpp": "cpp",
+            ".c": "c",
+            ".h": "c",
+            ".hpp": "cpp",
+            ".cs": "csharp",
+            ".go": "go",
+            ".rs": "rust",
+            ".php": "php",
+            ".rb": "ruby",
+            ".swift": "swift",
+            ".kt": "kotlin",
+            ".scala": "scala",
+            ".m": "objective-c",
+            ".mm": "objective-c",
+            ".ipynb": "jupyter",
+        };
+        return languageMap[ext] || "text";
+    }
+    /**
+     * Generate unique ID based on chunk content and location
+     * @param relativePath Relative path to the file
+     * @param startLine Start line number
+     * @param endLine End line number
+     * @param content Chunk content
+     * @returns Hash-based unique ID
+     */
+    generateId(relativePath, startLine, endLine, content) {
+        const combinedString = `${relativePath}:${startLine}:${endLine}:${content}`;
+        const hash = crypto
+            .createHash("sha256")
+            .update(combinedString, "utf-8")
+            .digest("hex");
+        return `chunk_${hash.substring(0, 16)}`;
+    }
+    /**
+     * Read ignore patterns from file (e.g., .gitignore)
+     * @param filePath Path to the ignore file
+     * @returns Array of ignore patterns
+     */
+    static async getIgnorePatternsFromFile(filePath) {
+        try {
+            const content = await fs.promises.readFile(filePath, "utf-8");
+            return content
+                .split("\n")
+                .map((line) => line.trim())
+                .filter((line) => line && !line.startsWith("#")); // Filter out empty lines and comments
+        }
+        catch (error) {
+            console.warn(`[Context] ⚠️  Could not read ignore file ${filePath}: ${error}`);
+            return [];
+        }
+    }
+    /**
+     * Load ignore patterns from various ignore files in the codebase
+     * This method preserves any existing custom patterns that were added before
+     * @param codebasePath Path to the codebase
+     */
+    async loadIgnorePatterns(codebasePath) {
+        try {
+            const fileBasedPatterns = [];
+            // Load all .xxxignore files in codebase directory
+            const ignoreFiles = await this.findIgnoreFiles(codebasePath);
+            for (const ignoreFile of ignoreFiles) {
+                const patterns = await this.loadIgnoreFile(ignoreFile, path.basename(ignoreFile));
+                fileBasedPatterns.push(...patterns);
+            }
+            // Load global ~/.context/.contextignore
+            const globalIgnorePatterns = await this.loadGlobalIgnoreFile();
+            fileBasedPatterns.push(...globalIgnorePatterns);
+            // Merge file-based patterns with existing patterns (which may include custom MCP patterns)
+            if (fileBasedPatterns.length > 0) {
+                this.addCustomIgnorePatterns(fileBasedPatterns);
+                console.log(`[Context] 🚫 Loaded total ${fileBasedPatterns.length} ignore patterns from all ignore files`);
+            }
+            else {
+                console.log("📄 No ignore files found, keeping existing patterns");
+            }
+        }
+        catch (error) {
+            console.warn(`[Context] ⚠️ Failed to load ignore patterns: ${error}`);
+            // Continue with existing patterns on error - don't reset them
+        }
+    }
+    /**
+     * Find all .xxxignore files in the codebase directory
+     * @param codebasePath Path to the codebase
+     * @returns Array of ignore file paths
+     */
+    async findIgnoreFiles(codebasePath) {
+        try {
+            const entries = await fs.promises.readdir(codebasePath, {
+                withFileTypes: true,
+            });
+            const ignoreFiles = [];
+            for (const entry of entries) {
+                if (entry.isFile() &&
+                    entry.name.startsWith(".") &&
+                    entry.name.endsWith("ignore")) {
+                    ignoreFiles.push(path.join(codebasePath, entry.name));
+                }
+            }
+            if (ignoreFiles.length > 0) {
+                console.log(`📄 Found ignore files: ${ignoreFiles.map((f) => path.basename(f)).join(", ")}`);
+            }
+            return ignoreFiles;
+        }
+        catch (error) {
+            console.warn(`[Context] ⚠️ Failed to scan for ignore files: ${error}`);
+            return [];
+        }
+    }
+    /**
+     * Load global ignore file from ~/.context/.contextignore
+     * @returns Array of ignore patterns
+     */
+    async loadGlobalIgnoreFile() {
+        try {
+            const homeDir = require("os").homedir();
+            const globalIgnorePath = path.join(homeDir, ".context", ".contextignore");
+            return await this.loadIgnoreFile(globalIgnorePath, "global .contextignore");
+        }
+        catch (_error) {
+            // Global ignore file is optional, don't log warnings
+            return [];
+        }
+    }
+    /**
+     * Load ignore patterns from a specific ignore file
+     * @param filePath Path to the ignore file
+     * @param fileName Display name for logging
+     * @returns Array of ignore patterns
+     */
+    async loadIgnoreFile(filePath, fileName) {
+        try {
+            await fs.promises.access(filePath);
+            console.log(`📄 Found ${fileName} file at: ${filePath}`);
+            const ignorePatterns = await Context.getIgnorePatternsFromFile(filePath);
+            if (ignorePatterns.length > 0) {
+                console.log(`[Context] 🚫 Loaded ${ignorePatterns.length} ignore patterns from ${fileName}`);
+                return ignorePatterns;
+            }
+            else {
+                console.log(`📄 ${fileName} file found but no valid patterns detected`);
+                return [];
+            }
+        }
+        catch (_error) {
+            if (fileName.includes("global")) {
+                console.log(`📄 No ${fileName} file found`);
+            }
+            return [];
+        }
+    }
+    /**
+     * Check if a path matches any ignore pattern
+     * @param filePath Path to check
+     * @param basePath Base path for relative pattern matching
+     * @returns True if path should be ignored
+     */
+    matchesIgnorePattern(filePath, basePath) {
+        if (this.ignorePatterns.length === 0) {
+            return false;
+        }
+        const relativePath = path.relative(basePath, filePath);
+        const normalizedPath = relativePath.replace(/\\/g, "/"); // Normalize path separators
+        for (const pattern of this.ignorePatterns) {
+            if (this.isPatternMatch(normalizedPath, pattern)) {
+                return true;
+            }
+        }
+        return false;
+    }
+    /**
+     * Simple glob pattern matching
+     * @param filePath File path to test
+     * @param pattern Glob pattern
+     * @returns True if pattern matches
+     */
+    isPatternMatch(filePath, pattern) {
+        // Handle directory patterns (ending with /)
+        if (pattern.endsWith("/")) {
+            const dirPattern = pattern.slice(0, -1);
+            const pathParts = filePath.split("/");
+            return pathParts.some((part) => this.simpleGlobMatch(part, dirPattern));
+        }
+        // Handle file patterns
+        if (pattern.includes("/")) {
+            // Pattern with path separator - match exact path
+            return this.simpleGlobMatch(filePath, pattern);
+        }
+        else {
+            // Pattern without path separator - match filename in any directory
+            const fileName = path.basename(filePath);
+            return this.simpleGlobMatch(fileName, pattern);
+        }
+    }
+    /**
+     * Simple glob matching supporting * wildcard
+     * @param text Text to test
+     * @param pattern Pattern with * wildcards
+     * @returns True if pattern matches
+     */
+    simpleGlobMatch(text, pattern) {
+        // Convert glob pattern to regex
+        const regexPattern = pattern
+            .replace(/[.+^${}()|[\]\\]/g, "\\$&") // Escape regex special chars except *
+            .replace(/\*/g, ".*"); // Convert * to .*
+        const regex = new RegExp(`^${regexPattern}$`);
+        return regex.test(text);
+    }
+    /**
+     * Get custom extensions from environment variables
+     * Supports CUSTOM_EXTENSIONS as comma-separated list
+     * @returns Array of custom extensions
+     */
+    getCustomExtensionsFromEnv() {
+        const envExtensions = env_manager_1.envManager.get("CUSTOM_EXTENSIONS");
+        if (!envExtensions) {
+            return [];
+        }
+        try {
+            const extensions = envExtensions
+                .split(",")
+                .map((ext) => ext.trim())
+                .filter((ext) => ext.length > 0)
+                .map((ext) => (ext.startsWith(".") ? ext : `.${ext}`)); // Ensure extensions start with dot
+            return extensions;
+        }
+        catch (error) {
+            console.warn(`[Context] ⚠️  Failed to parse CUSTOM_EXTENSIONS: ${error}`);
+            return [];
+        }
+    }
+    /**
+     * Get custom ignore patterns from environment variables
+     * Supports CUSTOM_IGNORE_PATTERNS as comma-separated list
+     * @returns Array of custom ignore patterns
+     */
+    getCustomIgnorePatternsFromEnv() {
+        const envIgnorePatterns = env_manager_1.envManager.get("CUSTOM_IGNORE_PATTERNS");
+        if (!envIgnorePatterns) {
+            return [];
+        }
+        try {
+            const patterns = envIgnorePatterns
+                .split(",")
+                .map((pattern) => pattern.trim())
+                .filter((pattern) => pattern.length > 0);
+            return patterns;
+        }
+        catch (error) {
+            console.warn(`[Context] ⚠️  Failed to parse CUSTOM_IGNORE_PATTERNS: ${error}`);
+            return [];
+        }
+    }
+    /**
+     * Add custom extensions (from MCP or other sources) without replacing existing ones
+     * @param customExtensions Array of custom extensions to add
+     */
+    addCustomExtensions(customExtensions) {
+        if (customExtensions.length === 0)
+            return;
+        // Ensure extensions start with dot
+        const normalizedExtensions = customExtensions.map((ext) => ext.startsWith(".") ? ext : `.${ext}`);
+        // Merge current extensions with new custom extensions, avoiding duplicates
+        const mergedExtensions = [
+            ...this.supportedExtensions,
+            ...normalizedExtensions,
+        ];
+        const uniqueExtensions = [...new Set(mergedExtensions)];
+        this.supportedExtensions = uniqueExtensions;
+        console.log(`[Context] 📎 Added ${customExtensions.length} custom extensions. Total: ${this.supportedExtensions.length} extensions`);
+    }
+    /**
+     * Get current splitter information
+     */
+    getSplitterInfo() {
+        const splitterName = this.codeSplitter.constructor.name;
+        if (splitterName === "AstCodeSplitter") {
+            const { AstCodeSplitter } = require("./splitter/ast-splitter");
+            return {
+                type: "ast",
+                hasBuiltinFallback: true,
+                supportedLanguages: AstCodeSplitter.getSupportedLanguages(),
+            };
+        }
+        else {
+            return {
+                type: "langchain",
+                hasBuiltinFallback: false,
+            };
+        }
+    }
+    /**
+     * Check if current splitter supports a specific language
+     * @param language Programming language
+     */
+    isLanguageSupported(language) {
+        const splitterName = this.codeSplitter.constructor.name;
+        if (splitterName === "AstCodeSplitter") {
+            const { AstCodeSplitter } = require("./splitter/ast-splitter");
+            return AstCodeSplitter.isLanguageSupported(language);
+        }
+        // LangChain splitter supports most languages
+        return true;
+    }
+    /**
+     * Get which strategy would be used for a specific language
+     * @param language Programming language
+     */
+    getSplitterStrategyForLanguage(language) {
+        const splitterName = this.codeSplitter.constructor.name;
+        if (splitterName === "AstCodeSplitter") {
+            const { AstCodeSplitter } = require("./splitter/ast-splitter");
+            const isSupported = AstCodeSplitter.isLanguageSupported(language);
+            return {
+                strategy: isSupported ? "ast" : "langchain",
+                reason: isSupported
+                    ? "Language supported by AST parser"
+                    : "Language not supported by AST, will fallback to LangChain",
+            };
+        }
+        else {
+            return {
+                strategy: "langchain",
+                reason: "Using LangChain splitter directly",
+            };
+        }
+    }
+}
+exports.Context = Context;
+//# sourceMappingURL=context.js.map