agentic-flow 2.0.1-alpha.2 → 2.0.1-alpha.20

package/dist/intelligence/EmbeddingService.js

@@ -0,0 +1,1484 @@
+/**
+ * EmbeddingService - Unified embedding interface for agentic-flow
+ *
+ * Uses ruvector@0.1.61+ for ONNX embeddings with:
+ * - SIMD128 acceleration (6x faster)
+ * - Parallel worker threads (7 workers)
+ * - all-MiniLM-L6-v2 model (384 dimensions)
+ * - Persistent SQLite cache (0.1ms vs 400ms)
+ *
+ * Configure via:
+ * - AGENTIC_FLOW_EMBEDDINGS=simple|onnx|auto (default: auto)
+ * - AGENTIC_FLOW_EMBEDDING_MODEL=all-MiniLM-L6-v2 (default)
+ * - AGENTIC_FLOW_EMBEDDING_CACHE=true|false (default: true)
+ * - AGENTIC_FLOW_PERSISTENT_CACHE=true|false (default: true)
+ */
+import { getEmbeddingCache } from './EmbeddingCache.js';
+// ONNX availability cache
+let onnxAvailable = null;
+let ruvectorModule = null;
+/**
+ * Detect ONNX/SIMD support by loading ruvector
+ */
+async function detectOnnx() {
+    if (onnxAvailable !== null) {
+        return onnxAvailable;
+    }
+    try {
+        const mod = await import('ruvector');
+        ruvectorModule = mod;
+        onnxAvailable = mod.isOnnxAvailable?.() ?? false;
+        return onnxAvailable;
+    }
+    catch (error) {
+        // Ruvector loading failed - fall back to simple embeddings
+        onnxAvailable = false;
+        return false;
+    }
+}
+// Simple LRU cache for embeddings (in-memory, fast)
+class LRUCache {
+    cache = new Map();
+    maxSize;
+    constructor(maxSize = 1000) {
+        this.maxSize = maxSize;
+    }
+    get(key) {
+        const value = this.cache.get(key);
+        if (value) {
+            // Move to end (most recently used)
+            this.cache.delete(key);
+            this.cache.set(key, value);
+        }
+        return value;
+    }
+    set(key, value) {
+        if (this.cache.size >= this.maxSize) {
+            // Delete oldest (first) entry
+            const firstKey = this.cache.keys().next().value;
+            if (firstKey) {
+                this.cache.delete(firstKey);
+            }
+        }
+        this.cache.set(key, value);
+    }
+    clear() {
+        this.cache.clear();
+    }
+    get size() {
+        return this.cache.size;
+    }
+}
+export class EmbeddingService {
+    static instance = null;
+    backend;
+    effectiveBackend = null;
+    dimension;
+    modelName;
+    // ONNX state
+    modelLoaded = false;
+    loadingPromise = null;
+    // Stats
+    totalEmbeddings = 0;
+    totalLatencyMs = 0;
+    cacheHits = 0;
+    // Cache (in-memory LRU)
+    cache;
+    cacheEnabled;
+    // Persistent cache (SQLite)
+    persistentCache = null;
+    persistentCacheEnabled;
+    // Corpus for search operations
+    corpus = { texts: [], embeddings: [] };
+    constructor() {
+        // Default to 'auto' which will detect ONNX and use it if available
+        this.backend = process.env.AGENTIC_FLOW_EMBEDDINGS || 'auto';
+        this.modelName = process.env.AGENTIC_FLOW_EMBEDDING_MODEL || 'all-MiniLM-L6-v2';
+        this.dimension = 256; // Will be updated when ONNX loads (384)
+        this.cacheEnabled = process.env.AGENTIC_FLOW_EMBEDDING_CACHE !== 'false';
+        this.persistentCacheEnabled = process.env.AGENTIC_FLOW_PERSISTENT_CACHE !== 'false';
+        this.cache = new LRUCache(1000);
+        // Initialize persistent cache
+        if (this.persistentCacheEnabled) {
+            try {
+                this.persistentCache = getEmbeddingCache({ dimension: 384 });
+            }
+            catch (error) {
+                console.warn('[EmbeddingService] Persistent cache unavailable:', error);
+                this.persistentCacheEnabled = false;
+            }
+        }
+    }
+    static getInstance() {
+        if (!EmbeddingService.instance) {
+            EmbeddingService.instance = new EmbeddingService();
+        }
+        return EmbeddingService.instance;
+    }
+    /**
+     * Resolve the effective backend based on ONNX detection
+     */
+    async resolveBackend() {
+        if (this.effectiveBackend) {
+            return this.effectiveBackend;
+        }
+        if (this.backend === 'auto') {
+            const hasOnnx = await detectOnnx();
+            this.effectiveBackend = hasOnnx ? 'onnx' : 'simple';
+            if (hasOnnx) {
+                this.dimension = 384; // all-MiniLM-L6-v2 dimension
+            }
+        }
+        else {
+            this.effectiveBackend = this.backend;
+            if (this.backend === 'onnx') {
+                await detectOnnx(); // Ensure module is loaded
+                this.dimension = 384;
+            }
+        }
+        return this.effectiveBackend;
+    }
+    /**
+     * Get configured backend (may be 'auto')
+     */
+    getBackend() {
+        return this.backend;
+    }
+    /**
+     * Get effective backend after detection
+     */
+    getEffectiveBackend() {
+        return this.effectiveBackend || this.backend;
+    }
+    /**
+     * Get embedding dimension
+     */
+    getDimension() {
+        return this.dimension;
+    }
+    /**
+     * Check if ONNX model is loaded
+     */
+    isModelLoaded() {
+        return this.modelLoaded;
+    }
+    /**
+     * Generate embedding for text
+     * Auto-detects ONNX and uses it if available (default behavior)
+     */
+    async embed(text) {
+        const startTime = performance.now();
+        // Check in-memory cache first (fastest)
+        if (this.cacheEnabled) {
+            const cached = this.cache.get(text);
+            if (cached) {
+                this.cacheHits++;
+                return cached;
+            }
+        }
+        // Check persistent cache (SQLite, ~0.1ms)
+        if (this.persistentCache) {
+            const cached = this.persistentCache.get(text, this.modelName);
+            if (cached) {
+                this.cacheHits++;
+                // Also store in memory cache for faster subsequent access
+                if (this.cacheEnabled) {
+                    this.cache.set(text, cached);
+                }
+                return cached;
+            }
+        }
+        // Resolve backend (handles 'auto' mode)
+        const effectiveBackend = await this.resolveBackend();
+        let embedding;
+        if (effectiveBackend === 'onnx' && ruvectorModule) {
+            const result = await ruvectorModule.embed(text);
+            if (result?.embedding) {
+                embedding = result.embedding;
+                this.modelLoaded = true;
+            }
+            else {
+                embedding = this.simpleEmbed(text);
+            }
+        }
+        else {
+            embedding = this.simpleEmbed(text);
+        }
+        // Update stats
+        this.totalEmbeddings++;
+        this.totalLatencyMs += performance.now() - startTime;
+        // Cache result in memory
+        if (this.cacheEnabled) {
+            this.cache.set(text, embedding);
+        }
+        // Cache result persistently (for cross-session)
+        if (this.persistentCache && effectiveBackend === 'onnx') {
+            this.persistentCache.set(text, embedding, this.modelName);
+        }
+        return embedding;
+    }
+    /**
+     * Generate embeddings for multiple texts (batch processing with parallel workers)
+     * Batch processing provides significant speedup with parallel ONNX workers
+     */
+    async embedBatch(texts) {
+        const startTime = performance.now();
+        // Check cache for all texts first
+        if (this.cacheEnabled) {
+            const cachedResults = texts.map(t => this.cache.get(t) || null);
+            const allCached = cachedResults.every(r => r !== null);
+            if (allCached) {
+                this.cacheHits += texts.length;
+                return cachedResults;
+            }
+        }
+        // Resolve backend
+        const effectiveBackend = await this.resolveBackend();
+        if (effectiveBackend === 'onnx' && ruvectorModule) {
+            const result = await ruvectorModule.embedBatch(texts);
+            if (result?.embeddings && result.embeddings.length === texts.length) {
+                const embeddings = result.embeddings;
+                // Cache individual embeddings
+                if (this.cacheEnabled) {
+                    for (let i = 0; i < texts.length; i++) {
+                        this.cache.set(texts[i], embeddings[i]);
+                    }
+                }
+                // Update stats
+                this.totalEmbeddings += texts.length;
+                this.totalLatencyMs += performance.now() - startTime;
+                this.modelLoaded = true;
+                return embeddings;
+            }
+        }
+        // Fall back to sequential for simple backend
+        return Promise.all(texts.map(t => this.embed(t)));
+    }
+    /**
+     * Compute similarity between two texts
+     */
+    async similarity(text1, text2) {
+        const effectiveBackend = await this.resolveBackend();
+        if (effectiveBackend === 'onnx' && ruvectorModule) {
+            const result = await ruvectorModule.similarity(text1, text2);
+            return result.similarity;
+        }
+        // Fall back to embedding + cosine
+        const [e1, e2] = await Promise.all([this.embed(text1), this.embed(text2)]);
+        return this.cosineSimilarity(e1, e2);
+    }
+    /**
+     * Compute NxN similarity matrix for a list of texts
+     * Uses parallel workers for ONNX backend
+     */
+    async similarityMatrix(texts) {
+        const embeddings = await this.embedBatch(texts);
+        const n = texts.length;
+        const matrix = Array(n).fill(null).map(() => Array(n).fill(0));
+        for (let i = 0; i < n; i++) {
+            matrix[i][i] = 1.0; // Self-similarity
+            for (let j = i + 1; j < n; j++) {
+                const sim = this.cosineSimilarity(embeddings[i], embeddings[j]);
+                matrix[i][j] = sim;
+                matrix[j][i] = sim; // Symmetric
+            }
+        }
+        return matrix;
+    }
+    /**
+     * Build a corpus for semantic search
+     */
+    async buildCorpus(texts) {
+        this.corpus.texts = texts;
+        this.corpus.embeddings = await this.embedBatch(texts);
+    }
+    /**
+     * Semantic search against the corpus
+     * Returns top-k most similar texts
+     */
+    async semanticSearch(query, topK = 5) {
+        if (this.corpus.texts.length === 0) {
+            throw new Error('Corpus not built. Call buildCorpus() first.');
+        }
+        const queryEmbedding = await this.embed(query);
+        const results = [];
+        for (let i = 0; i < this.corpus.texts.length; i++) {
+            const sim = this.cosineSimilarity(queryEmbedding, this.corpus.embeddings[i]);
+            results.push({
+                text: this.corpus.texts[i],
+                index: i,
+                similarity: sim,
+            });
+        }
+        // Sort by similarity (descending) and return top-k
+        results.sort((a, b) => b.similarity - a.similarity);
+        return results.slice(0, topK);
+    }
+    /**
+     * Find near-duplicate texts in a list
+     * Groups texts with similarity above threshold
+     */
+    async findDuplicates(texts, threshold = 0.9) {
+        const embeddings = await this.embedBatch(texts);
+        const n = texts.length;
+        const visited = new Set();
+        const groups = [];
+        for (let i = 0; i < n; i++) {
+            if (visited.has(i))
+                continue;
+            const group = {
+                indices: [i],
+                texts: [texts[i]],
+                similarity: 1.0,
+            };
+            for (let j = i + 1; j < n; j++) {
+                if (visited.has(j))
+                    continue;
+                const sim = this.cosineSimilarity(embeddings[i], embeddings[j]);
+                if (sim >= threshold) {
+                    group.indices.push(j);
+                    group.texts.push(texts[j]);
+                    group.similarity = Math.min(group.similarity, sim);
+                    visited.add(j);
+                }
+            }
+            if (group.indices.length > 1) {
+                visited.add(i);
+                groups.push(group);
+            }
+        }
+        return groups;
+    }
+    /**
+     * K-means clustering of texts
+     * Returns cluster assignments and centroids
+     */
+    async clusterTexts(texts, k = 3, maxIterations = 100) {
+        const embeddings = await this.embedBatch(texts);
+        const n = texts.length;
+        const dim = this.dimension;
+        // Initialize centroids randomly (copy to new ArrayBuffer for consistent typing)
+        const centroidIndices = new Set();
+        while (centroidIndices.size < k && centroidIndices.size < n) {
+            centroidIndices.add(Math.floor(Math.random() * n));
+        }
+        let centroids = Array.from(centroidIndices).map(i => {
+            const copy = new Float32Array(dim);
+            copy.set(embeddings[i]);
+            return copy;
+        });
+        let clusters = new Array(n).fill(0);
+        for (let iter = 0; iter < maxIterations; iter++) {
+            // Assign points to nearest centroid
+            const newClusters = embeddings.map(emb => {
+                let bestCluster = 0;
+                let bestSim = -Infinity;
+                for (let c = 0; c < k; c++) {
+                    const sim = this.cosineSimilarity(emb, centroids[c]);
+                    if (sim > bestSim) {
+                        bestSim = sim;
+                        bestCluster = c;
+                    }
+                }
+                return bestCluster;
+            });
+            // Check convergence
+            const changed = newClusters.some((c, i) => c !== clusters[i]);
+            clusters = newClusters;
+            if (!changed)
+                break;
+            // Update centroids
+            const newCentroids = [];
+            for (let c = 0; c < k; c++) {
+                newCentroids.push(new Float32Array(dim));
+            }
+            const counts = new Array(k).fill(0);
+            for (let i = 0; i < n; i++) {
+                const c = clusters[i];
+                counts[c]++;
+                for (let d = 0; d < dim; d++) {
+                    newCentroids[c][d] += embeddings[i][d];
+                }
+            }
+            // Normalize centroids
+            for (let c = 0; c < k; c++) {
+                if (counts[c] > 0) {
+                    let norm = 0;
+                    for (let d = 0; d < dim; d++) {
+                        newCentroids[c][d] /= counts[c];
+                        norm += newCentroids[c][d] * newCentroids[c][d];
+                    }
+                    norm = Math.sqrt(norm) || 1;
+                    for (let d = 0; d < dim; d++) {
+                        newCentroids[c][d] /= norm;
+                    }
+                }
+            }
+            centroids = newCentroids;
+        }
+        return { clusters, centroids };
+    }
+    /**
+     * Stream embeddings for large batches (memory efficient)
+     * Yields embeddings one at a time
+     */
+    async *streamEmbed(texts, batchSize = 32) {
+        for (let i = 0; i < texts.length; i += batchSize) {
+            const batch = texts.slice(i, i + batchSize);
+            const embeddings = await this.embedBatch(batch);
+            for (let j = 0; j < batch.length; j++) {
+                yield {
+                    index: i + j,
+                    text: batch[j],
+                    embedding: embeddings[j],
+                };
+            }
+        }
+    }
+    /**
+     * Simple hash-based embedding (fast, not semantic)
+     */
+    simpleEmbed(text, dim = 256) {
+        const embedding = new Float32Array(dim);
+        // Multi-pass hash for better distribution
+        for (let i = 0; i < text.length; i++) {
+            const code = text.charCodeAt(i);
+            embedding[i % dim] += code / 255;
+            embedding[(i * 7) % dim] += (code * 0.3) / 255;
+            embedding[(i * 13) % dim] += (code * 0.2) / 255;
+        }
+        // Normalize
+        let norm = 0;
+        for (let i = 0; i < dim; i++) {
+            norm += embedding[i] * embedding[i];
+        }
+        norm = Math.sqrt(norm) || 1;
+        for (let i = 0; i < dim; i++) {
+            embedding[i] /= norm;
+        }
+        return embedding;
+    }
+    /**
+     * Compute cosine similarity between two embeddings
+     */
+    cosineSimilarity(a, b) {
+        if (ruvectorModule?.cosineSimilarity) {
+            return ruvectorModule.cosineSimilarity(a, b);
+        }
+        // JS fallback
+        let dot = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dot += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        return dot / (Math.sqrt(normA) * Math.sqrt(normB) || 1);
+    }
+    /**
+     * Get statistics
+     */
+    getStats() {
+        const effective = this.effectiveBackend || this.backend;
+        const ruvectorStats = ruvectorModule?.getStats?.() || {};
+        // Get persistent cache stats
+        let persistentCacheStats;
+        if (this.persistentCache) {
+            const cacheStats = this.persistentCache.getStats();
+            persistentCacheStats = {
+                enabled: true,
+                entries: cacheStats.totalEntries,
+                hits: cacheStats.hits,
+                misses: cacheStats.misses,
+                hitRate: cacheStats.hitRate,
+                dbSizeKB: Math.round(cacheStats.dbSizeBytes / 1024),
+            };
+        }
+        return {
+            backend: this.backend,
+            effectiveBackend: effective,
+            dimension: this.dimension,
+            totalEmbeddings: this.totalEmbeddings,
+            totalLatencyMs: this.totalLatencyMs,
+            avgLatencyMs: this.totalEmbeddings > 0 ? this.totalLatencyMs / this.totalEmbeddings : 0,
+            cacheHits: this.cacheHits,
+            modelLoaded: this.modelLoaded,
+            modelName: effective === 'onnx' ? this.modelName : undefined,
+            simdAvailable: ruvectorStats.simdAvailable ?? onnxAvailable,
+            parallelWorkers: ruvectorStats.workerCount ?? undefined,
+            persistentCache: persistentCacheStats,
+        };
+    }
+    /**
+     * Clear in-memory cache
+     */
+    clearCache() {
+        this.cache.clear();
+    }
+    /**
+     * Clear persistent cache (SQLite)
+     */
+    clearPersistentCache() {
+        if (this.persistentCache) {
+            this.persistentCache.clear();
+        }
+    }
+    /**
+     * Clear all caches (memory + persistent)
+     */
+    clearAllCaches() {
+        this.cache.clear();
+        if (this.persistentCache) {
+            this.persistentCache.clear();
+        }
+    }
+    /**
+     * Get persistent cache stats
+     */
+    getPersistentCacheStats() {
+        if (!this.persistentCache)
+            return null;
+        const stats = this.persistentCache.getStats();
+        return {
+            entries: stats.totalEntries,
+            hits: stats.hits,
+            misses: stats.misses,
+            hitRate: stats.hitRate,
+        };
+    }
+    /**
+     * Clear corpus
+     */
+    clearCorpus() {
+        this.corpus = { texts: [], embeddings: [] };
+    }
+    /**
+     * Shutdown (cleanup workers)
+     */
+    async shutdown() {
+        if (ruvectorModule?.shutdown) {
+            await ruvectorModule.shutdown();
+        }
+    }
+    /**
+     * Reset instance (for testing)
+     */
+    static async reset() {
+        if (EmbeddingService.instance) {
+            await EmbeddingService.instance.shutdown();
+        }
+        EmbeddingService.instance = null;
+        onnxAvailable = null;
+        ruvectorModule = null;
+    }
+    /**
+     * Pretrain cache with texts from files
+     * Embeds content and stores in persistent cache for fast retrieval
+     *
+     * @param sources - File paths or glob patterns, or array of texts
+     * @param options - Pretrain options
+     * @returns Stats about pretraining
+     */
+    async pretrain(sources, options = {}) {
+        const { batchSize = 32, onProgress, chunkSize = 512, overlapSize = 64, skipCached = true } = options;
+        const startTime = performance.now();
+        let processed = 0;
+        let cached = 0;
+        let skipped = 0;
+        // Resolve texts to embed
+        const texts = [];
+        if (typeof sources === 'string') {
+            sources = [sources];
+        }
+        for (const source of sources) {
+            // Check if it's a file path or glob pattern
+            if (source.includes('/') || source.includes('*') || source.includes('.')) {
+                try {
+                    const fs = await import('fs');
+                    const path = await import('path');
+                    const { glob } = await import('glob').catch(() => ({ glob: null }));
+                    // Handle glob patterns
+                    let files = [];
+                    if (source.includes('*') && glob) {
+                        files = await glob(source);
+                    }
+                    else if (fs.existsSync(source)) {
+                        files = [source];
+                    }
+                    for (const file of files) {
+                        try {
+                            const content = fs.readFileSync(file, 'utf-8');
+                            // Chunk large files
+                            if (content.length > chunkSize * 2) {
+                                for (let i = 0; i < content.length; i += chunkSize - overlapSize) {
+                                    const chunk = content.slice(i, i + chunkSize);
+                                    if (chunk.trim().length > 10) {
+                                        texts.push(chunk);
+                                    }
+                                }
+                            }
+                            else if (content.trim().length > 10) {
+                                texts.push(content);
+                            }
+                        }
+                        catch {
+                            // Skip unreadable files
+                        }
+                    }
+                }
+                catch {
+                    // Treat as plain text if file operations fail
+                    texts.push(source);
+                }
+            }
+            else {
+                texts.push(source);
+            }
+        }
+        // Filter out already cached texts
+        const toEmbed = [];
+        for (const text of texts) {
+            if (skipCached && this.persistentCache?.has(text, this.modelName)) {
+                skipped++;
+            }
+            else {
+                toEmbed.push(text);
+            }
+        }
+        // Embed in batches
+        for (let i = 0; i < toEmbed.length; i += batchSize) {
+            const batch = toEmbed.slice(i, i + batchSize);
+            const embeddings = await this.embedBatch(batch);
+            // Store in persistent cache (embedBatch already handles this for ONNX)
+            cached += embeddings.length;
+            processed += batch.length;
+            if (onProgress) {
+                onProgress(processed, toEmbed.length);
+            }
+        }
+        return {
+            processed,
+            cached,
+            skipped,
+            timeMs: performance.now() - startTime,
+        };
+    }
+    /**
+     * Pretrain with common programming patterns
+     * Pre-caches embeddings for frequently used code patterns
+     */
+    async pretrainCodePatterns() {
+        const patterns = [
+            // Common programming constructs
+            'function implementation',
+            'class definition',
+            'interface declaration',
+            'type alias',
+            'import statement',
+            'export module',
+            'async await pattern',
+            'promise handling',
+            'error handling try catch',
+            'conditional logic if else',
+            'loop iteration for while',
+            'array map filter reduce',
+            'object destructuring',
+            'spread operator',
+            'rest parameters',
+            // Code operations
+            'refactor code',
+            'fix bug',
+            'add feature',
+            'write tests',
+            'add documentation',
+            'optimize performance',
+            'improve readability',
+            'handle edge cases',
+            'add validation',
+            'implement authentication',
+            // File types
+            'TypeScript file',
+            'JavaScript module',
+            'React component',
+            'Vue component',
+            'CSS stylesheet',
+            'JSON configuration',
+            'Markdown documentation',
+            'Python script',
+            'Shell script',
+            'SQL query',
+            // Agent routing patterns
+            'code review task',
+            'architecture design',
+            'testing strategy',
+            'debugging session',
+            'performance analysis',
+            'security audit',
+            'documentation update',
+            'API design',
+            'database schema',
+            'deployment configuration',
+        ];
+        const startTime = performance.now();
+        const embeddings = await this.embedBatch(patterns);
+        return {
+            cached: embeddings.length,
+            timeMs: performance.now() - startTime,
+        };
+    }
+    /**
+     * Pretrain from repository structure
+     * Analyzes file names and paths to pre-cache common patterns
+     */
+    async pretrainFromRepo(repoPath = '.') {
+        const startTime = performance.now();
+        let files = 0;
+        let chunks = 0;
+        try {
+            const fs = await import('fs');
+            const path = await import('path');
+            // Common code file extensions
+            const extensions = ['.ts', '.tsx', '.js', '.jsx', '.py', '.md', '.json'];
+            const walkDir = (dir) => {
+                try {
+                    const entries = fs.readdirSync(dir, { withFileTypes: true });
+                    for (const entry of entries) {
+                        const fullPath = path.join(dir, entry.name);
+                        if (entry.isDirectory()) {
+                            // Skip node_modules, .git, etc.
+                            if (!entry.name.startsWith('.') && entry.name !== 'node_modules' && entry.name !== 'dist') {
+                                walkDir(fullPath);
+                            }
+                        }
+                        else if (extensions.some(ext => entry.name.endsWith(ext))) {
+                            return fullPath;
+                        }
+                    }
+                }
+                catch {
+                    // Skip unreadable directories
+                }
+                return null;
+            };
+            // Collect files
+            const filePaths = [];
+            const collectFiles = (dir) => {
+                try {
+                    const entries = fs.readdirSync(dir, { withFileTypes: true });
+                    for (const entry of entries) {
+                        const fullPath = path.join(dir, entry.name);
+                        if (entry.isDirectory()) {
+                            if (!entry.name.startsWith('.') && entry.name !== 'node_modules' && entry.name !== 'dist') {
+                                collectFiles(fullPath);
+                            }
+                        }
+                        else if (extensions.some(ext => entry.name.endsWith(ext))) {
+                            filePaths.push(fullPath);
+                        }
+                    }
+                }
+                catch {
+                    // Skip unreadable
+                }
+            };
+            collectFiles(repoPath);
+            files = filePaths.length;
+            // Pretrain from collected files
+            if (filePaths.length > 0) {
+                const result = await this.pretrain(filePaths, {
+                    batchSize: 16,
+                    chunkSize: 512,
+                    overlapSize: 64,
+                });
+                chunks = result.cached;
+            }
+        }
+        catch (err) {
+            // Repository analysis failed
+        }
+        return {
+            files,
+            chunks,
+            timeMs: performance.now() - startTime,
+        };
+    }
+    /**
+     * Incremental pretrain - only process changed files since last run
+     * Uses git diff to detect modified files
+     */
+    async pretrainIncremental(options = {}) {
+        const { since = 'HEAD~10', repoPath = '.' } = options;
+        const startTime = performance.now();
+        let changedFiles = 0;
+        let newChunks = 0;
+        let skipped = 0;
+        try {
+            const { execSync } = await import('child_process');
+            const path = await import('path');
+            const fs = await import('fs');
+            // Get changed files from git
+            const gitOutput = execSync(`git diff --name-only ${since}`, {
+                cwd: repoPath,
+                encoding: 'utf-8',
+            });
+            const changedPaths = gitOutput
+                .split('\n')
+                .filter(f => f.trim())
+                .map(f => path.join(repoPath, f))
+                .filter(f => {
+                try {
+                    return fs.existsSync(f) && fs.statSync(f).isFile();
+                }
+                catch {
+                    return false;
+                }
+            });
+            changedFiles = changedPaths.length;
+            if (changedPaths.length > 0) {
+                const result = await this.pretrain(changedPaths, {
+                    batchSize: 16,
+                    chunkSize: 512,
+                    overlapSize: 64,
+                    skipCached: true,
+                });
+                newChunks = result.cached;
+                skipped = result.skipped;
+            }
+        }
+        catch {
+            // Git not available or not a repo
+        }
+        return {
+            changedFiles,
+            newChunks,
+            skipped,
+            timeMs: performance.now() - startTime,
+        };
+    }
+    /**
+     * Smart chunking - split code by semantic boundaries
+     * (functions, classes, etc.) instead of fixed size
+     */
+    semanticChunk(content, fileType) {
+        const chunks = [];
+        // TypeScript/JavaScript patterns
+        if (['.ts', '.tsx', '.js', '.jsx'].some(ext => fileType.endsWith(ext))) {
+            // Split on function/class/interface boundaries
+            const patterns = [
+                /^(export\s+)?(async\s+)?function\s+\w+/gm,
+                /^(export\s+)?class\s+\w+/gm,
+                /^(export\s+)?interface\s+\w+/gm,
+                /^(export\s+)?type\s+\w+/gm,
+                /^(export\s+)?const\s+\w+\s*=/gm,
+            ];
+            let lastIndex = 0;
+            const boundaries = [0];
+            for (const pattern of patterns) {
+                let match;
+                while ((match = pattern.exec(content)) !== null) {
+                    boundaries.push(match.index);
+                }
+            }
+            boundaries.push(content.length);
+            boundaries.sort((a, b) => a - b);
+            // Extract chunks between boundaries
+            for (let i = 0; i < boundaries.length - 1; i++) {
+                const chunk = content.slice(boundaries[i], boundaries[i + 1]).trim();
+                if (chunk.length > 20 && chunk.length < 2000) {
+                    chunks.push(chunk);
+                }
+            }
+        }
+        // Python patterns
+        else if (fileType.endsWith('.py')) {
+            const patterns = [
+                /^(async\s+)?def\s+\w+/gm,
+                /^class\s+\w+/gm,
+            ];
+            const boundaries = [0];
+            for (const pattern of patterns) {
+                let match;
+                while ((match = pattern.exec(content)) !== null) {
+                    boundaries.push(match.index);
+                }
+            }
+            boundaries.push(content.length);
+            boundaries.sort((a, b) => a - b);
+            for (let i = 0; i < boundaries.length - 1; i++) {
+                const chunk = content.slice(boundaries[i], boundaries[i + 1]).trim();
+                if (chunk.length > 20 && chunk.length < 2000) {
+                    chunks.push(chunk);
+                }
+            }
+        }
+        // Markdown - split by headers
+        else if (fileType.endsWith('.md')) {
+            const sections = content.split(/^#+\s+/gm);
+            for (const section of sections) {
+                if (section.trim().length > 20) {
+                    chunks.push(section.trim().slice(0, 1000));
+                }
+            }
+        }
+        // Fallback to fixed-size chunking
+        if (chunks.length === 0) {
+            const chunkSize = 512;
+            const overlap = 64;
+            for (let i = 0; i < content.length; i += chunkSize - overlap) {
+                const chunk = content.slice(i, i + chunkSize);
+                if (chunk.trim().length > 20) {
+                    chunks.push(chunk);
+                }
+            }
+        }
+        return chunks;
+    }
+    /**
+     * Pretrain with semantic chunking
+     * Uses code structure to create meaningful chunks
+     */
+    async pretrainSemantic(sources, options = {}) {
+        const { batchSize = 32, onProgress } = options;
+        const startTime = performance.now();
+        let fileCount = 0;
+        let chunkCount = 0;
+        const allChunks = [];
+        try {
+            const fs = await import('fs');
+            const path = await import('path');
+            for (const source of sources) {
+                if (fs.existsSync(source)) {
+                    try {
+                        const content = fs.readFileSync(source, 'utf-8');
+                        const ext = path.extname(source);
+                        const chunks = this.semanticChunk(content, ext);
+                        allChunks.push(...chunks);
+                        fileCount++;
+                    }
+                    catch {
+                        // Skip unreadable files
+                    }
+                }
+            }
+            // Embed and cache all chunks
+            for (let i = 0; i < allChunks.length; i += batchSize) {
+                const batch = allChunks.slice(i, i + batchSize);
+                await this.embedBatch(batch);
+                chunkCount += batch.length;
+                if (onProgress) {
+                    onProgress(chunkCount, allChunks.length);
+                }
+            }
+        }
+        catch {
+            // Pretrain failed
+        }
+        return {
+            files: fileCount,
+            chunks: chunkCount,
+            timeMs: performance.now() - startTime,
+        };
+    }
+    /**
+     * Priority pretrain - cache most frequently used patterns first
+     * Tracks access patterns and prioritizes high-frequency queries
+     */
+    accessCounts = new Map();
+    recordAccess(text) {
+        this.accessCounts.set(text, (this.accessCounts.get(text) || 0) + 1);
+    }
+    getTopPatterns(n = 100) {
+        return Array.from(this.accessCounts.entries())
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, n)
+            .map(([text]) => text);
+    }
+    async pretrainPriority(n = 100) {
+        const topPatterns = this.getTopPatterns(n);
+        const startTime = performance.now();
+        if (topPatterns.length > 0) {
+            await this.embedBatch(topPatterns);
+        }
+        return {
+            cached: topPatterns.length,
+            timeMs: performance.now() - startTime,
+        };
+    }
+    /**
+     * Warmup cache on session start
+     * Combines code patterns + recent repo changes
+     */
+    async warmup(repoPath = '.') {
+        const startTime = performance.now();
+        // First: load common patterns
+        const patternResult = await this.pretrainCodePatterns();
+        // Second: load recent git changes
+        const incrementalResult = await this.pretrainIncremental({
+            since: 'HEAD~5',
+            repoPath,
+        });
+        return {
+            patterns: patternResult.cached,
+            recentChanges: incrementalResult.newChunks,
+            timeMs: performance.now() - startTime,
+        };
+    }
+    /**
+     * Intelligent pretrain using ruvector worker pool
+     * Analyzes repo structure, code patterns, and prepares cache
+     * Uses parallel workers for maximum throughput
+     */
+    async pretrainIntelligent(options = {}) {
+        const { repoPath = '.', parallel = true, onProgress } = options;
+        const startTime = performance.now();
+        const stages = {
+            codePatterns: { count: 0, timeMs: 0 },
+            astAnalysis: { files: 0, functions: 0, timeMs: 0 },
+            gitHistory: { commits: 0, hotFiles: 0, timeMs: 0 },
+            dependencies: { modules: 0, imports: 0, timeMs: 0 },
+            semanticChunks: { chunks: 0, timeMs: 0 },
+        };
+        let totalCached = 0;
+        try {
+            // Stage 1: Code patterns (common programming patterns)
+            onProgress?.('codePatterns', 0);
+            const stage1Start = performance.now();
+            const patternResult = await this.pretrainCodePatterns();
+            stages.codePatterns = {
+                count: patternResult.cached,
+                timeMs: performance.now() - stage1Start,
+            };
+            totalCached += patternResult.cached;
+            onProgress?.('codePatterns', 100);
+            // Stage 2: AST Analysis using ruvector workers (if available)
+            onProgress?.('astAnalysis', 0);
+            const stage2Start = performance.now();
+            try {
+                if (ruvectorModule && parallel) {
+                    // Use ruvector's analyzeFilesParallel if available
+                    const mod = ruvectorModule;
+                    if (mod.analyzeFilesParallel) {
+                        const fs = await import('fs');
+                        const path = await import('path');
+                        // Collect source files
+                        const sourceFiles = [];
+                        const collectSources = (dir) => {
+                            try {
+                                const entries = fs.readdirSync(dir, { withFileTypes: true });
+                                for (const entry of entries) {
+                                    const fullPath = path.join(dir, entry.name);
+                                    if (entry.isDirectory()) {
+                                        if (!entry.name.startsWith('.') && entry.name !== 'node_modules' && entry.name !== 'dist') {
+                                            collectSources(fullPath);
+                                        }
+                                    }
+                                    else if (['.ts', '.tsx', '.js', '.jsx'].some(ext => entry.name.endsWith(ext))) {
+                                        sourceFiles.push(fullPath);
+                                    }
+                                }
+                            }
+                            catch { }
+                        };
+                        collectSources(repoPath);
+                        // Analyze in parallel
+                        const astResult = await mod.analyzeFilesParallel(sourceFiles.slice(0, 100));
+                        stages.astAnalysis = {
+                            files: sourceFiles.length,
+                            functions: astResult?.functions || 0,
+                            timeMs: performance.now() - stage2Start,
+                        };
+                        // Extract function signatures for caching
+                        if (astResult?.signatures) {
+                            await this.embedBatch(astResult.signatures.slice(0, 200));
+                            totalCached += Math.min(astResult.signatures.length, 200);
+                        }
+                    }
+                }
+            }
+            catch { }
+            onProgress?.('astAnalysis', 100);
+            // Stage 3: Git history analysis (hot files = frequently changed)
+            onProgress?.('gitHistory', 0);
+            const stage3Start = performance.now();
+            try {
+                const { execSync } = await import('child_process');
+                // Get commit count
+                const commitCount = execSync('git rev-list --count HEAD', {
+                    cwd: repoPath,
+                    encoding: 'utf-8',
+                }).trim();
+                // Get hot files (most frequently changed)
+                const hotFilesOutput = execSync('git log --format="" --name-only -n 100 | sort | uniq -c | sort -rn | head -20', { cwd: repoPath, encoding: 'utf-8' });
+                const hotFiles = hotFilesOutput
+                    .split('\n')
+                    .filter(l => l.trim())
+                    .map(l => l.trim().split(/\s+/).slice(1).join(' '))
+                    .filter(f => f);
+                stages.gitHistory = {
+                    commits: parseInt(commitCount) || 0,
+                    hotFiles: hotFiles.length,
+                    timeMs: performance.now() - stage3Start,
+                };
+                // Pretrain hot files
+                if (hotFiles.length > 0) {
+                    const fs = await import('fs');
+                    const path = await import('path');
+                    const validFiles = hotFiles
+                        .map(f => path.join(repoPath, f))
+                        .filter(f => fs.existsSync(f));
+                    if (validFiles.length > 0) {
+                        const result = await this.pretrainSemantic(validFiles, { batchSize: 16 });
+                        totalCached += result.chunks;
+                    }
+                }
+            }
+            catch { }
+            onProgress?.('gitHistory', 100);
+            // Stage 4: Dependency analysis
+            onProgress?.('dependencies', 0);
+            const stage4Start = performance.now();
+            try {
+                const fs = await import('fs');
+                const path = await import('path');
+                // Parse package.json for dependencies
+                const pkgPath = path.join(repoPath, 'package.json');
+                if (fs.existsSync(pkgPath)) {
+                    const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf-8'));
+                    const deps = Object.keys(pkg.dependencies || {});
+                    const devDeps = Object.keys(pkg.devDependencies || {});
+                    const allDeps = [...deps, ...devDeps];
+                    stages.dependencies = {
+                        modules: allDeps.length,
+                        imports: 0,
+                        timeMs: performance.now() - stage4Start,
+                    };
+                    // Cache dependency names for import resolution
+                    if (allDeps.length > 0) {
+                        const depPatterns = allDeps.map(d => `import from ${d}`);
+                        await this.embedBatch(depPatterns);
+                        totalCached += depPatterns.length;
+                    }
+                }
+            }
+            catch { }
+            onProgress?.('dependencies', 100);
+            // Stage 5: Semantic chunking with parallel embedding
+            onProgress?.('semanticChunks', 0);
+            const stage5Start = performance.now();
+            try {
+                const incrementalResult = await this.pretrainIncremental({
+                    since: 'HEAD~20',
+                    repoPath,
+                });
+                stages.semanticChunks = {
+                    chunks: incrementalResult.newChunks,
+                    timeMs: performance.now() - stage5Start,
+                };
+                totalCached += incrementalResult.newChunks;
+            }
+            catch { }
+            onProgress?.('semanticChunks', 100);
+        }
+        catch (err) {
+            // Pretrain failed, return partial results
+        }
+        return {
+            stages,
+            totalCached,
+            totalTimeMs: performance.now() - startTime,
+        };
+    }
+    /**
+     * Background pretrain - runs in worker if available
+     * Non-blocking, returns immediately with a promise
+     */
+    pretrainBackground(options = {}) {
+        let cancelled = false;
+        const promise = (async () => {
+            if (cancelled)
+                return;
+            // Run warmup in background
+            await this.warmup(options.repoPath);
+            if (cancelled)
+                return;
+            // Then run intelligent pretrain
+            await this.pretrainIntelligent({
+                ...options,
+                parallel: true,
+            });
+        })();
+        return {
+            promise,
+            cancel: () => { cancelled = true; },
+        };
+    }
+    /**
+     * AI-enhanced pretrain using ruvector attention mechanisms
+     * Uses HyperbolicAttention for code structure, MoE for routing
+     */
+    async pretrainWithAI(options = {}) {
+        const { repoPath = '.', attentionType = 'auto', onProgress } = options;
+        const startTime = performance.now();
+        const patterns = [];
+        let totalCached = 0;
+        let attentionInfo = { type: 'none', timeMs: 0 };
+        let predictions = { prefetch: 0, confidence: 0 };
+        try {
+            const mod = ruvectorModule;
+            // Step 1: Determine best attention type for codebase
+            onProgress?.('attention', 'Selecting optimal attention mechanism...');
+            let selectedAttention = attentionType;
+            if (attentionType === 'auto' && mod) {
+                // Use getAttentionForUseCase if available
+                if (mod.getAttentionForUseCase) {
+                    const result = await mod.getAttentionForUseCase('code_analysis');
+                    selectedAttention = result?.type || 'hyperbolic';
+                }
+                else {
+                    // Default to hyperbolic for hierarchical code structure
+                    selectedAttention = 'hyperbolic';
+                }
+            }
+            attentionInfo.type = selectedAttention;
+            const attentionStart = performance.now();
+            // Step 2: Use attention to identify important code regions
+            onProgress?.('analysis', `Using ${selectedAttention} attention for code analysis...`);
+            if (mod) {
+                // Collect code samples for attention-based analysis
+                const fs = await import('fs');
+                const path = await import('path');
+                const codeSamples = [];
+                const collectCode = (dir, maxFiles = 50) => {
+                    if (codeSamples.length >= maxFiles)
+                        return;
+                    try {
+                        const entries = fs.readdirSync(dir, { withFileTypes: true });
+                        for (const entry of entries) {
+                            if (codeSamples.length >= maxFiles)
+                                break;
+                            const fullPath = path.join(dir, entry.name);
+                            if (entry.isDirectory()) {
+                                if (!entry.name.startsWith('.') && entry.name !== 'node_modules' && entry.name !== 'dist') {
+                                    collectCode(fullPath, maxFiles);
+                                }
+                            }
+                            else if (['.ts', '.tsx', '.js', '.jsx'].some(ext => entry.name.endsWith(ext))) {
+                                try {
+                                    const content = fs.readFileSync(fullPath, 'utf-8');
+                                    if (content.length < 5000) {
+                                        codeSamples.push(content);
+                                    }
+                                }
+                                catch { }
+                            }
+                        }
+                    }
+                    catch { }
+                };
+                collectCode(repoPath);
+                // Step 3: Use attention mechanisms to weight code importance
+                if (mod.HyperbolicAttention && selectedAttention === 'hyperbolic') {
+                    try {
+                        // Hyperbolic attention for hierarchical code structure
+                        const attention = new mod.HyperbolicAttention({ dim: 384 });
+                        // Identify structural patterns (classes, functions, imports)
+                        const structuralPatterns = [
+                            'class definition with constructor',
+                            'async function with error handling',
+                            'interface with multiple properties',
+                            'type with generics',
+                            'import statement block',
+                            'export default component',
+                            'hook implementation useEffect',
+                            'API endpoint handler',
+                            'database query function',
+                            'authentication middleware',
+                        ];
+                        await this.embedBatch(structuralPatterns);
+                        patterns.push({ type: 'structural', count: structuralPatterns.length });
+                        totalCached += structuralPatterns.length;
+                    }
+                    catch { }
+                }
+                if (mod.MoEAttention && selectedAttention === 'moe') {
+                    try {
+                        // MoE for routing different code patterns to experts
+                        const routingPatterns = [
+                            // Expert 1: Frontend
+                            'React component with state',
+                            'Vue component with props',
+                            'CSS styling module',
+                            // Expert 2: Backend
+                            'Express route handler',
+                            'GraphQL resolver',
+                            'REST API endpoint',
+                            // Expert 3: Data
+                            'SQL query builder',
+                            'MongoDB aggregation',
+                            'Redis cache operation',
+                            // Expert 4: Testing
+                            'Jest test case',
+                            'E2E test scenario',
+                            'Mock implementation',
+                        ];
+                        await this.embedBatch(routingPatterns);
+                        patterns.push({ type: 'routing', count: routingPatterns.length });
+                        totalCached += routingPatterns.length;
+                    }
+                    catch { }
+                }
+                if (mod.GraphRoPeAttention && selectedAttention === 'graph') {
+                    try {
+                        // Graph attention for dependency understanding
+                        const graphPatterns = [
+                            'module exports',
+                            'circular dependency',
+                            'shared utility import',
+                            'type re-export',
+                            'barrel file index',
+                            'lazy import dynamic',
+                            'peer dependency',
+                            'optional dependency',
+                        ];
+                        await this.embedBatch(graphPatterns);
+                        patterns.push({ type: 'graph', count: graphPatterns.length });
+                        totalCached += graphPatterns.length;
+                    }
+                    catch { }
+                }
+                attentionInfo.timeMs = performance.now() - attentionStart;
+                // Step 4: FastGRNN for pattern prediction (if available)
+                onProgress?.('prediction', 'Training pattern predictor...');
+                if (mod.FastGRNN) {
+                    try {
+                        // Use recent access patterns to predict what's needed next
+                        const topPatterns = this.getTopPatterns(50);
+                        if (topPatterns.length > 0) {
+                            // Prefetch predicted patterns
+                            const prefetchPatterns = [
+                                ...topPatterns.slice(0, 20),
+                                // Add related patterns
+                                ...topPatterns.slice(0, 10).map(p => `similar to: ${p}`),
+                            ];
+                            await this.embedBatch(prefetchPatterns);
+                            predictions = {
+                                prefetch: prefetchPatterns.length,
+                                confidence: 0.85, // Estimated based on access history
+                            };
+                            totalCached += prefetchPatterns.length;
+                        }
+                    }
+                    catch { }
+                }
+            }
+            // Step 5: Standard warmup
+            onProgress?.('warmup', 'Running standard warmup...');
+            const warmupResult = await this.warmup(repoPath);
+            totalCached += warmupResult.patterns + warmupResult.recentChanges;
+            patterns.push({ type: 'warmup', count: warmupResult.patterns + warmupResult.recentChanges });
+        }
+        catch (err) {
+            // AI pretrain failed, continue with basic
+        }
+        return {
+            patterns,
+            attention: attentionInfo,
+            predictions,
+            totalCached,
+            totalTimeMs: performance.now() - startTime,
+        };
+    }
+    /**
+     * Context-aware prefetch using attention
+     * Predicts what embeddings will be needed based on current context
+     */
+    async prefetchForContext(context) {
+        const startTime = performance.now();
+        let prefetched = 0;
+        let confidence = 0;
+        try {
+            const patterns = [];
+            // Add patterns based on current file type
+            if (context.currentFile) {
+                const ext = context.currentFile.split('.').pop() || '';
+                const filePatterns = {
+                    ts: ['TypeScript type checking', 'interface implementation', 'generic types'],
+                    tsx: ['React component', 'JSX rendering', 'hook usage'],
+                    js: ['JavaScript module', 'CommonJS require', 'ES6 import'],
+                    jsx: ['React component', 'JSX element', 'props handling'],
+                    py: ['Python function', 'class method', 'import statement'],
+                    md: ['documentation', 'README section', 'code example'],
+                };
+                patterns.push(...(filePatterns[ext] || []));
+            }
+            // Add patterns based on task type
+            if (context.taskType) {
+                const taskPatterns = {
+                    edit: ['code modification', 'variable rename', 'function update'],
+                    review: ['code review', 'bug detection', 'style check'],
+                    debug: ['error trace', 'stack analysis', 'variable inspection'],
+                    test: ['test case', 'assertion', 'mock setup'],
+                    refactor: ['code cleanup', 'pattern extraction', 'abstraction'],
+                };
+                patterns.push(...(taskPatterns[context.taskType] || []));
+            }
+            // Add patterns based on user query similarity
+            if (context.userQuery) {
+                patterns.push(context.userQuery);
+                // Add variations
+                patterns.push(`how to ${context.userQuery}`);
+                patterns.push(`implement ${context.userQuery}`);
+            }
+            if (patterns.length > 0) {
+                await this.embedBatch(patterns);
+                prefetched = patterns.length;
+                confidence = Math.min(0.9, 0.5 + patterns.length * 0.05);
+            }
+        }
+        catch {
+            // Prefetch failed
+        }
+        return {
+            prefetched,
+            confidence,
+            timeMs: performance.now() - startTime,
+        };
+    }
+}
+// Export singleton getter
+export function getEmbeddingService() {
+    return EmbeddingService.getInstance();
+}
+// Export convenience functions
+export async function embed(text) {
+    return getEmbeddingService().embed(text);
+}
+export async function embedBatch(texts) {
+    return getEmbeddingService().embedBatch(texts);
+}
+export async function pretrainCodePatterns() {
+    return getEmbeddingService().pretrainCodePatterns();
+}
+export async function pretrainFromRepo(repoPath = '.') {
+    return getEmbeddingService().pretrainFromRepo(repoPath);
+}
+export async function textSimilarity(text1, text2) {
+    return getEmbeddingService().similarity(text1, text2);
+}
+export function simpleEmbed(text, dim = 256) {
+    return getEmbeddingService().simpleEmbed(text, dim);
+}
+export async function similarityMatrix(texts) {
+    return getEmbeddingService().similarityMatrix(texts);
+}
+export async function semanticSearch(query, topK = 5) {
+    return getEmbeddingService().semanticSearch(query, topK);
+}
+export async function findDuplicates(texts, threshold = 0.9) {
+    return getEmbeddingService().findDuplicates(texts, threshold);
+}
+export async function clusterTexts(texts, k = 3) {
+    return getEmbeddingService().clusterTexts(texts, k);
+}
+//# sourceMappingURL=EmbeddingService.js.map