@mrxkun/mcfast-mcp 4.1.10 → 4.1.11

package/src/memory/utils/vector-index.js

@@ -0,0 +1,241 @@
+/**
+ * Vector Index for fast similarity search
+ * Implements HNSW-like in-memory index for embeddings
+ * 
+ * This provides O(log n) search instead of O(n) linear scan
+ */
+
+export class VectorIndex {
+  constructor(options = {}) {
+    this.dimension = options.dimension || 1024;
+    this.maxElements = options.maxElements || 10000;
+    this.efConstruction = options.efConstruction || 200;
+    this.M = options.M || 16;
+    
+    // In-memory storage
+    this.vectors = new Map();  // id -> Float32Array
+    this.metadata = new Map(); // id -> metadata
+    
+    // Simple HNSW-like structure (layer 0 only for simplicity)
+    this.connections = new Map(); // id -> [neighbor_ids]
+    
+    // Entry point for search
+    this.entryPoint = null;
+    
+    // Stats
+    this.stats = {
+      searches: 0,
+      totalDuration: 0,
+      avgDuration: 0
+    };
+  }
+
+  /**
+   * Add vector to index
+   */
+  add(id, vector, metadata = {}) {
+    if (vector.length !== this.dimension) {
+      throw new Error(`Vector dimension ${vector.length} != expected ${this.dimension}`);
+    }
+    
+    const vec = vector instanceof Float32Array ? vector : new Float32Array(vector);
+    this.vectors.set(id, vec);
+    this.metadata.set(id, metadata);
+    
+    // Initialize empty connections
+    if (!this.connections.has(id)) {
+      this.connections.set(id, []);
+    }
+    
+    // Set entry point if first element
+    if (!this.entryPoint) {
+      this.entryPoint = id;
+    }
+    
+    // Build connections (simplified - just connect to nearest)
+    this._buildConnections(id, vec);
+    
+    return this;
+  }
+
+  /**
+   * Build connections to nearest neighbors
+   */
+  _buildConnections(id, vector) {
+    // Find nearest neighbors
+    const neighbors = [];
+    
+    for (const [otherId, otherVec] of this.vectors) {
+      if (otherId === id) continue;
+      
+      const dist = this._cosineDistance(vector, otherVec);
+      neighbors.push({ id: otherId, dist });
+    }
+    
+    // Sort by distance and take M nearest
+    neighbors.sort((a, b) => a.dist - b.dist);
+    const nearest = neighbors.slice(0, this.M);
+    
+    // Update connections
+    const connections = this.connections.get(id) || [];
+    nearest.forEach(n => connections.push(n.id));
+    this.connections.set(id, connections);
+    
+    // Add reverse connections
+    nearest.forEach(n => {
+      const reverseConns = this.connections.get(n.id) || [];
+      if (!reverseConns.includes(id)) {
+        reverseConns.push(id);
+        this.connections.set(n.id, reverseConns);
+      }
+    });
+  }
+
+  /**
+   * Search for nearest neighbors
+   */
+  search(queryVector, k = 5, ef = 10) {
+    const startTime = performance.now();
+    
+    if (!this.entryPoint || this.vectors.size === 0) {
+      return [];
+    }
+    
+    const query = queryVector instanceof Float32Array 
+      ? queryVector 
+      : new Float32Array(queryVector);
+    
+    // Greedy search from entry point
+    let current = this.entryPoint;
+    let bestDist = this._cosineDistance(query, this.vectors.get(current));
+    let visited = new Set([current]);
+    
+    // Explore neighbors
+    let candidates = [current];
+    let results = [{ id: current, dist: bestDist }];
+    
+    while (candidates.length > 0) {
+      const candidate = candidates.shift();
+      const candidateVec = this.vectors.get(candidate);
+      const candidateDist = this._cosineDistance(query, candidateVec);
+      
+      // Update results
+      if (results.length < k || candidateDist < results[results.length - 1].dist) {
+        results.push({ id: candidate, dist: candidateDist });
+        results.sort((a, b) => a.dist - b.dist);
+        results = results.slice(0, k);
+      }
+      
+      // Get neighbors
+      const neighbors = this.connections.get(candidate) || [];
+      for (const neighbor of neighbors) {
+        if (visited.has(neighbor)) continue;
+        visited.add(neighbor);
+        
+        const neighborDist = this._cosineDistance(query, this.vectors.get(neighbor));
+        
+        // Add to candidates if promising
+        if (results.length < k || neighborDist < results[results.length - 1].dist) {
+          candidates.push(neighbor);
+        }
+      }
+    }
+    
+    // Sort final results
+    results.sort((a, b) => a.dist - b.dist);
+    
+    // Convert distance to similarity
+    const searchResults = results.slice(0, k).map(r => ({
+      id: r.id,
+      similarity: 1 - r.dist,
+      distance: r.dist,
+      metadata: this.metadata.get(r.id)
+    }));
+    
+    // Update stats
+    const duration = performance.now() - startTime;
+    this.stats.searches++;
+    this.stats.totalDuration += duration;
+    this.stats.avgDuration = this.stats.totalDuration / this.stats.searches;
+    
+    return searchResults;
+  }
+
+  /**
+   * Cosine distance (1 - cosine similarity)
+   */
+  _cosineDistance(a, b) {
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+    
+    for (let i = 0; i < a.length; i++) {
+      dotProduct += a[i] * b[i];
+      normA += a[i] * a[i];
+      normB += b[i] * b[i];
+    }
+    
+    const similarity = dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+    return 1 - similarity;
+  }
+
+  /**
+   * Bulk add vectors
+   */
+  bulkAdd(items) {
+    for (const item of items) {
+      this.add(item.id, item.vector, item.metadata);
+    }
+    return this;
+  }
+
+  /**
+   * Get statistics
+   */
+  getStats() {
+    return {
+      ...this.stats,
+      size: this.vectors.size,
+      dimension: this.dimension,
+      maxElements: this.maxElements
+    };
+  }
+
+  /**
+   * Clear index
+   */
+  clear() {
+    this.vectors.clear();
+    this.metadata.clear();
+    this.connections.clear();
+    this.entryPoint = null;
+    this.stats = { searches: 0, totalDuration: 0, avgDuration: 0 };
+  }
+
+  /**
+   * Serialize to JSON
+   */
+  toJSON() {
+    const data = {};
+    for (const [id, vec] of this.vectors) {
+      data[id] = {
+        vector: Array.from(vec),
+        metadata: this.metadata.get(id)
+      };
+    }
+    return data;
+  }
+
+  /**
+   * Load from JSON
+   */
+  static fromJSON(json, options = {}) {
+    const index = new VectorIndex(options);
+    for (const [id, item] of Object.entries(json)) {
+      index.add(id, item.vector, item.metadata);
+    }
+    return index;
+  }
+}
+
+export default VectorIndex;

package/src/memory/watchers/file-watcher.js

@@ -2,6 +2,11 @@
  * File Watcher
  * Theo dõi thay đổi file và index tự động
  * Debounced 1.5s để batch rapid changes
+ * 
+ * IMPROVEMENTS v4.1.10:
+ * - Added periodic cleanup for pendingUpdates Map
+ * - Added error boundaries for async operations
+ * - Added graceful shutdown cleanup
  */
 
 import chokidar from 'chokidar';
@@ -29,9 +34,13 @@ export class FileWatcher {
     this.watcher = null;
     this.pendingUpdates = new Map();
     this.isProcessing = false;
+    this.cleanupInterval = null;
+    this.isRunning = false;
     
     // Configuration
     this.debounceMs = options.debounceMs || 1500;
+    this.cleanupIntervalMs = options.cleanupIntervalMs || 30000; // Cleanup every 30s
+    this.maxPendingUpdates = options.maxPendingUpdates || 1000;
     this.ignored = options.ignored || [
       '**/node_modules/**',
       '**/.git/**',
@@ -46,40 +55,100 @@ export class FileWatcher {
       filesAdded: 0,
       filesChanged: 0,
       filesDeleted: 0,
-      errors: 0
+      errors: 0,
+      totalProcessed: 0
     };
   }
 
   async start() {
+    if (this.isRunning) {
+      console.error('[FileWatcher] Already running');
+      return;
+    }
+
     console.error(`[FileWatcher] Starting watcher for: ${this.projectPath}`);
     console.error(`[FileWatcher] Debounce: ${this.debounceMs}ms`);
     
-    this.watcher = chokidar.watch(this.projectPath, {
-      ignored: this.ignored,
-      persistent: true,
-      ignoreInitial: true, // Don't trigger on existing files
-      awaitWriteFinish: {
-        stabilityThreshold: 300,
-        pollInterval: 100
-      }
-    });
+    try {
+      this.watcher = chokidar.watch(this.projectPath, {
+        ignored: this.ignored,
+        persistent: true,
+        ignoreInitial: true,
+        awaitWriteFinish: {
+          stabilityThreshold: 300,
+          pollInterval: 100
+        }
+      });
 
-    // Bind event handlers
-    this.watcher.on('add', filePath => this.handleAdd(filePath));
-    this.watcher.on('change', filePath => this.handleChange(filePath));
-    this.watcher.on('unlink', filePath => this.handleDelete(filePath));
-    this.watcher.on('error', error => this.handleError(error));
-    
-    // Setup debounced flush
-    this.flushQueue = debounce(() => this.processQueue(), this.debounceMs);
+      // Bind event handlers with error handling
+      this.watcher.on('add', filePath => this.safeHandleAdd(filePath));
+      this.watcher.on('change', filePath => this.safeHandleChange(filePath));
+      this.watcher.on('unlink', filePath => this.safeHandleDelete(filePath));
+      this.watcher.on('error', error => this.handleError(error));
+      
+      // Setup debounced flush
+      this.flushQueue = debounce(() => this.processQueue(), this.debounceMs);
+      
+      // Wait for ready
+      await new Promise((resolve, reject) => {
+        this.watcher.once('ready', resolve);
+        this.watcher.once('error', reject);
+      });
+      
+      // Start periodic cleanup
+      this.startCleanup();
+      
+      this.isRunning = true;
+      console.error(`[FileWatcher] Ready and watching`);
+    } catch (error) {
+      console.error(`[FileWatcher] Failed to start:`, error.message);
+      throw error;
+    }
+  }
+
+  // Safe handlers with try-catch
+  safeHandleAdd(filePath) {
+    try {
+      this.handleAdd(filePath);
+    } catch (error) {
+      console.error(`[FileWatcher] Error in handleAdd:`, error.message);
+      this.stats.errors++;
+    }
+  }
+
+  safeHandleChange(filePath) {
+    try {
+      this.handleChange(filePath);
+    } catch (error) {
+      console.error(`[FileWatcher] Error in handleChange:`, error.message);
+      this.stats.errors++;
+    }
+  }
+
+  safeHandleDelete(filePath) {
+    try {
+      this.handleDelete(filePath);
+    } catch (error) {
+      console.error(`[FileWatcher] Error in handleDelete:`, error.message);
+      this.stats.errors++;
+    }
+  }
+
+  // Periodic cleanup to prevent memory leak
+  startCleanup() {
+    if (this.cleanupInterval) {
+      clearInterval(this.cleanupInterval);
+    }
     
-    // Wait for ready
-    await new Promise((resolve, reject) => {
-      this.watcher.once('ready', resolve);
-      this.watcher.once('error', reject);
-    });
+    this.cleanupInterval = setInterval(() => {
+      if (this.pendingUpdates.size > this.maxPendingUpdates) {
+        console.error(`[FileWatcher] Cleanup: clearing ${this.pendingUpdates.size} pending updates`);
+        this.pendingUpdates.clear();
+      }
+    }, this.cleanupIntervalMs);
     
-    console.error(`[FileWatcher] Ready and watching`);
+    // Prevent interval from keeping process alive
+    this.cleanupInterval.unref();
   }
 
   handleAdd(filePath) {
@@ -119,7 +188,6 @@ export class FileWatcher {
 
   async processQueue() {
     if (this.isProcessing) {
-      // If already processing, debounce will call again
       return;
     }
     
@@ -139,14 +207,25 @@ export class FileWatcher {
       
       // Process deletions first
       for (const update of deletes) {
-        await this.processDelete(update);
+        try {
+          await this.processDelete(update);
+        } catch (error) {
+          console.error(`[FileWatcher] Error processing delete:`, error.message);
+          this.stats.errors++;
+        }
       }
       
       // Process additions/changes
       for (const update of adds) {
-        await this.processFile(update);
+        try {
+          await this.processFile(update);
+        } catch (error) {
+          console.error(`[FileWatcher] Error processing file:`, error.message);
+          this.stats.errors++;
+        }
       }
       
+      this.stats.totalProcessed += updates.length;
       console.error(`[FileWatcher] Processed ${updates.length} updates`);
       
     } catch (error) {
@@ -165,19 +244,19 @@ export class FileWatcher {
       const stats = await fs.stat(filePath).catch(() => null);
       if (!stats || !stats.isFile()) return;
       
-      // Skip large files (> 1MB)
-      if (stats.size > 1024 * 1024) {
-        console.error(`[FileWatcher] Skipping large file: ${filePath} (${Math.round(stats.size / 1024)}KB)`);
-        return;
-      }
-      
-      // Read file content
+      // Read file content first
       const content = await fs.readFile(filePath, 'utf-8');
       const contentHash = crypto.createHash('md5').update(content).digest('hex');
       
+      // Handle large files (> 1MB)
+      if (stats.size > 1024 * 1024) {
+        console.error(`[FileWatcher] Large file detected: ${filePath} (${Math.round(stats.size / 1024)}KB)`);
+        return this.indexLargeFile(filePath, content, contentHash, stats);
+      }
+      
       // Check if already indexed with same hash
-      if (this.memory.codebaseDb?.isFileIndexed?.(filePath, contentHash)) {
-        console.error(`[FileWatcher] File unchanged: ${filePath}`);
+      const existingFile = this.memory.codebaseDb?.getFileByPath?.(filePath);
+      if (existingFile && existingFile.content_hash === contentHash) {
         return;
       }
       
@@ -222,77 +301,102 @@ export class FileWatcher {
   async indexMarkdownFile(filePath, content, contentHash, stats) {
     console.error(`[FileWatcher] Indexing Markdown: ${filePath}`);
     
-    // Delete old chunks if updating
-    const relativePath = path.relative(this.projectPath, filePath);
-    this.memory.memoryDb?.deleteChunksByFile?.(relativePath);
-    
-    // Chunk the content
-    const { MarkdownChunker } = await import('../utils/markdown-chunker.js');
-    const chunker = new MarkdownChunker();
-    const chunks = chunker.chunk(content, relativePath);
-    
-    // Track file
-    this.memory.memoryDb?.upsertFile?.(relativePath, contentHash, stats.mtimeMs, stats.size);
-    
-    // Generate embeddings and insert chunks
-    for (const chunk of chunks) {
-      // Check cache first
-      let embedding = null;
-      const cached = this.memory.memoryDb?.getCachedEmbedding?.(chunk.contentHash);
+    try {
+      // Delete old chunks if updating
+      const relativePath = path.relative(this.projectPath, filePath);
+      this.memory.memoryDb?.deleteChunksByFile?.(relativePath);
       
-      if (cached) {
-        embedding = cached.embedding;
-      } else {
-        // Generate embedding
-        embedding = await this.memory.embedder?.embedCode?.(chunk.content);
-        
-        // Cache it
-        if (embedding) {
-          this.memory.memoryDb?.cacheEmbedding?.(
-            chunk.contentHash,
-            Buffer.from(new Float32Array(embedding).buffer),
-            'simple-embedder',
-            embedding.length
-          );
-        }
-      }
+      // Chunk the content
+      const { MarkdownChunker } = await import('../utils/markdown-chunker.js');
+      const chunker = new MarkdownChunker();
+      const chunks = chunker.chunk(content, relativePath);
       
-      // Insert chunk
-      this.memory.memoryDb?.insertChunk?.({
-        id: chunk.id,
-        file_path: chunk.filePath,
-        start_line: chunk.startLine,
-        end_line: chunk.endLine,
-        content: chunk.content,
-        content_hash: chunk.contentHash,
-        chunk_type: chunk.chunkType
-      });
+      // Track file
+      this.memory.memoryDb?.upsertFile?.(relativePath, contentHash, stats.mtimeMs, stats.size);
       
-      // Insert embedding
-      if (embedding) {
-        this.memory.memoryDb?.insertEmbedding?.({
-          chunk_id: chunk.id,
-          embedding: Buffer.from(new Float32Array(embedding).buffer),
-          model: 'simple-embedder',
-          dimensions: embedding.length
-        });
+      // Generate embeddings and insert chunks
+      for (const chunk of chunks) {
+        try {
+          // Check cache first
+          let embedding = null;
+          const cached = this.memory.memoryDb?.getCachedEmbedding?.(chunk.contentHash);
+          
+          if (cached) {
+            embedding = cached.embedding;
+          } else {
+            // Generate embedding
+            embedding = await this.memory.embedder?.embedCode?.(chunk.content);
+            
+            // Cache it
+            if (embedding) {
+              this.memory.memoryDb?.cacheEmbedding?.(
+                chunk.contentHash,
+                Buffer.from(new Float32Array(embedding).buffer),
+                'simple-embedder',
+                embedding.length
+              );
+            }
+          }
+          
+          // Insert chunk
+          this.memory.memoryDb?.insertChunk?.({
+            id: chunk.id,
+            file_path: chunk.filePath,
+            start_line: chunk.startLine,
+            end_line: chunk.endLine,
+            content: chunk.content,
+            content_hash: chunk.contentHash,
+            chunk_type: chunk.chunkType
+          });
+          
+          // Insert embedding
+          if (embedding) {
+            this.memory.memoryDb?.insertEmbedding?.({
+              chunk_id: chunk.id,
+              embedding: Buffer.from(new Float32Array(embedding).buffer),
+              model: 'simple-embedder',
+              dimensions: embedding.length
+            });
+          }
+        } catch (chunkError) {
+          console.error(`[FileWatcher] Error indexing chunk:`, chunkError.message);
+        }
       }
+      
+      console.error(`[FileWatcher] Indexed ${chunks.length} chunks from ${filePath}`);
+    } catch (error) {
+      console.error(`[FileWatcher] Error indexing markdown ${filePath}:`, error.message);
+      this.stats.errors++;
     }
-    
-    console.error(`[FileWatcher] Indexed ${chunks.length} chunks from ${filePath}`);
   }
 
-  async indexCodeFile(filePath, content, contentHash, stats) {
+  async indexCodeFile(filePath, content, contentHash, stats, maxRetries = 3) {
     console.error(`[FileWatcher] Indexing code: ${filePath}`);
-    
-    // Use the existing indexer
-    if (this.memory.indexer) {
+
+    if (!this.memory.indexer) {
+      console.error(`[FileWatcher] No indexer available`);
+      return;
+    }
+
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
       try {
         const indexed = await this.memory.indexer.indexFile(filePath, content);
-        await this.memory.storeIndexed(indexed);
-        console.error(`[FileWatcher] Indexed ${indexed.facts.length} facts, ${indexed.chunks.length} chunks from ${filePath}`);
+        const stored = await this.memory.storeIndexed(indexed);
+
+        if (stored) {
+          console.error(`[FileWatcher] Indexed ${indexed.facts.length} facts, ${indexed.chunks.length} chunks`);
+          return;
+        } else {
+          throw new Error('storeIndexed returned false');
+        }
       } catch (error) {
-        console.error(`[FileWatcher] Failed to index ${filePath}:`, error.message);
+        if (attempt === maxRetries) {
+          console.error(`[FileWatcher] Failed to index after ${maxRetries} attempts:`, error.message);
+          this.stats.errors++;
+        } else {
+          console.warn(`[FileWatcher] Retry ${attempt}/${maxRetries}:`, error.message);
+          await new Promise(resolve => setTimeout(resolve, 100 * Math.pow(2, attempt - 1)));
+        }
       }
     }
   }
@@ -302,25 +406,73 @@ export class FileWatcher {
     return ext === '.md' || ext === '.markdown';
   }
 
+  async indexLargeFile(filePath, content, contentHash, stats) {
+    const chunkSize = 500 * 1024;
+    try {
+      const lines = content.split('\n');
+      let chunkContent = '';
+      let chunkIndex = 0;
+
+      for (let i = 0; i < lines.length; i++) {
+        chunkContent += lines[i] + '\n';
+
+        if (chunkContent.length >= chunkSize || i === lines.length - 1) {
+          console.error(`[FileWatcher] Indexing chunk ${chunkIndex + 1} of large file`);
+
+          if (this.isMarkdownFile(filePath)) {
+            await this.indexMarkdownFile(filePath, chunkContent, contentHash + '_' + chunkIndex, stats);
+          } else {
+            await this.indexCodeFile(filePath, chunkContent, contentHash + '_' + chunkIndex, stats);
+          }
+
+          chunkIndex++;
+          chunkContent = '';
+        }
+      }
+
+      console.error(`[FileWatcher] Large file indexed in ${chunkIndex} chunks`);
+      this.stats.filesChanged++;
+
+    } catch (error) {
+      console.error(`[FileWatcher] Failed to index large file:`, error.message);
+      this.stats.errors++;
+    }
+  }
+
   getStats() {
     return {
       ...this.stats,
       pendingUpdates: this.pendingUpdates.size,
-      isProcessing: this.isProcessing
+      isProcessing: this.isProcessing,
+      isRunning: this.isRunning
     };
   }
 
   async stop() {
+    if (!this.isRunning) return;
+    
+    console.error('[FileWatcher] Stopping...');
+    
+    // Stop cleanup interval
+    if (this.cleanupInterval) {
+      clearInterval(this.cleanupInterval);
+      this.cleanupInterval = null;
+    }
+    
+    // Process remaining updates
+    if (this.pendingUpdates.size > 0) {
+      console.error(`[FileWatcher] Processing remaining ${this.pendingUpdates.size} updates...`);
+      await this.processQueue();
+    }
+    
+    // Close watcher
     if (this.watcher) {
-      // Process any remaining updates
-      if (this.pendingUpdates.size > 0) {
-        await this.processQueue();
-      }
-      
       await this.watcher.close();
       this.watcher = null;
-      console.error(`[FileWatcher] Stopped`);
     }
+    
+    this.isRunning = false;
+    console.error('[FileWatcher] Stopped');
   }
 }