@mrxkun/mcfast-mcp 4.1.10 → 4.1.12

package/src/memory/watchers/file-watcher.js

@@ -2,6 +2,11 @@
  * File Watcher
  * Theo dõi thay đổi file và index tự động
  * Debounced 1.5s để batch rapid changes
+ * 
+ * IMPROVEMENTS v4.1.10:
+ * - Added periodic cleanup for pendingUpdates Map
+ * - Added error boundaries for async operations
+ * - Added graceful shutdown cleanup
  */
 
 import chokidar from 'chokidar';
@@ -29,9 +34,13 @@ export class FileWatcher {
     this.watcher = null;
     this.pendingUpdates = new Map();
     this.isProcessing = false;
+    this.cleanupInterval = null;
+    this.isRunning = false;
     
     // Configuration
     this.debounceMs = options.debounceMs || 1500;
+    this.cleanupIntervalMs = options.cleanupIntervalMs || 30000; // Cleanup every 30s
+    this.maxPendingUpdates = options.maxPendingUpdates || 1000;
     this.ignored = options.ignored || [
       '**/node_modules/**',
       '**/.git/**',
@@ -46,40 +55,100 @@ export class FileWatcher {
       filesAdded: 0,
       filesChanged: 0,
       filesDeleted: 0,
-      errors: 0
+      errors: 0,
+      totalProcessed: 0
     };
   }
 
   async start() {
+    if (this.isRunning) {
+      console.error('[FileWatcher] Already running');
+      return;
+    }
+
     console.error(`[FileWatcher] Starting watcher for: ${this.projectPath}`);
     console.error(`[FileWatcher] Debounce: ${this.debounceMs}ms`);
     
-    this.watcher = chokidar.watch(this.projectPath, {
-      ignored: this.ignored,
-      persistent: true,
-      ignoreInitial: true, // Don't trigger on existing files
-      awaitWriteFinish: {
-        stabilityThreshold: 300,
-        pollInterval: 100
-      }
-    });
+    try {
+      this.watcher = chokidar.watch(this.projectPath, {
+        ignored: this.ignored,
+        persistent: true,
+        ignoreInitial: true,
+        awaitWriteFinish: {
+          stabilityThreshold: 300,
+          pollInterval: 100
+        }
+      });
 
-    // Bind event handlers
-    this.watcher.on('add', filePath => this.handleAdd(filePath));
-    this.watcher.on('change', filePath => this.handleChange(filePath));
-    this.watcher.on('unlink', filePath => this.handleDelete(filePath));
-    this.watcher.on('error', error => this.handleError(error));
-    
-    // Setup debounced flush
-    this.flushQueue = debounce(() => this.processQueue(), this.debounceMs);
+      // Bind event handlers with error handling
+      this.watcher.on('add', filePath => this.safeHandleAdd(filePath));
+      this.watcher.on('change', filePath => this.safeHandleChange(filePath));
+      this.watcher.on('unlink', filePath => this.safeHandleDelete(filePath));
+      this.watcher.on('error', error => this.handleError(error));
+      
+      // Setup debounced flush
+      this.flushQueue = debounce(() => this.processQueue(), this.debounceMs);
+      
+      // Wait for ready
+      await new Promise((resolve, reject) => {
+        this.watcher.once('ready', resolve);
+        this.watcher.once('error', reject);
+      });
+      
+      // Start periodic cleanup
+      this.startCleanup();
+      
+      this.isRunning = true;
+      console.error(`[FileWatcher] Ready and watching`);
+    } catch (error) {
+      console.error(`[FileWatcher] Failed to start:`, error.message);
+      throw error;
+    }
+  }
+
+  // Safe handlers with try-catch
+  safeHandleAdd(filePath) {
+    try {
+      this.handleAdd(filePath);
+    } catch (error) {
+      console.error(`[FileWatcher] Error in handleAdd:`, error.message);
+      this.stats.errors++;
+    }
+  }
+
+  safeHandleChange(filePath) {
+    try {
+      this.handleChange(filePath);
+    } catch (error) {
+      console.error(`[FileWatcher] Error in handleChange:`, error.message);
+      this.stats.errors++;
+    }
+  }
+
+  safeHandleDelete(filePath) {
+    try {
+      this.handleDelete(filePath);
+    } catch (error) {
+      console.error(`[FileWatcher] Error in handleDelete:`, error.message);
+      this.stats.errors++;
+    }
+  }
+
+  // Periodic cleanup to prevent memory leak
+  startCleanup() {
+    if (this.cleanupInterval) {
+      clearInterval(this.cleanupInterval);
+    }
     
-    // Wait for ready
-    await new Promise((resolve, reject) => {
-      this.watcher.once('ready', resolve);
-      this.watcher.once('error', reject);
-    });
+    this.cleanupInterval = setInterval(() => {
+      if (this.pendingUpdates.size > this.maxPendingUpdates) {
+        console.error(`[FileWatcher] Cleanup: clearing ${this.pendingUpdates.size} pending updates`);
+        this.pendingUpdates.clear();
+      }
+    }, this.cleanupIntervalMs);
     
-    console.error(`[FileWatcher] Ready and watching`);
+    // Prevent interval from keeping process alive
+    this.cleanupInterval.unref();
   }
 
   handleAdd(filePath) {
@@ -119,7 +188,6 @@ export class FileWatcher {
 
   async processQueue() {
     if (this.isProcessing) {
-      // If already processing, debounce will call again
       return;
     }
     
@@ -139,14 +207,25 @@ export class FileWatcher {
       
       // Process deletions first
       for (const update of deletes) {
-        await this.processDelete(update);
+        try {
+          await this.processDelete(update);
+        } catch (error) {
+          console.error(`[FileWatcher] Error processing delete:`, error.message);
+          this.stats.errors++;
+        }
       }
       
       // Process additions/changes
       for (const update of adds) {
-        await this.processFile(update);
+        try {
+          await this.processFile(update);
+        } catch (error) {
+          console.error(`[FileWatcher] Error processing file:`, error.message);
+          this.stats.errors++;
+        }
       }
       
+      this.stats.totalProcessed += updates.length;
       console.error(`[FileWatcher] Processed ${updates.length} updates`);
       
     } catch (error) {
@@ -165,19 +244,19 @@ export class FileWatcher {
       const stats = await fs.stat(filePath).catch(() => null);
       if (!stats || !stats.isFile()) return;
       
-      // Skip large files (> 1MB)
-      if (stats.size > 1024 * 1024) {
-        console.error(`[FileWatcher] Skipping large file: ${filePath} (${Math.round(stats.size / 1024)}KB)`);
-        return;
-      }
-      
-      // Read file content
+      // Read file content first
       const content = await fs.readFile(filePath, 'utf-8');
       const contentHash = crypto.createHash('md5').update(content).digest('hex');
       
+      // Handle large files (> 1MB)
+      if (stats.size > 1024 * 1024) {
+        console.error(`[FileWatcher] Large file detected: ${filePath} (${Math.round(stats.size / 1024)}KB)`);
+        return this.indexLargeFile(filePath, content, contentHash, stats);
+      }
+      
       // Check if already indexed with same hash
-      if (this.memory.codebaseDb?.isFileIndexed?.(filePath, contentHash)) {
-        console.error(`[FileWatcher] File unchanged: ${filePath}`);
+      const existingFile = this.memory.codebaseDb?.getFileByPath?.(filePath);
+      if (existingFile && existingFile.content_hash === contentHash) {
         return;
       }
       
@@ -222,77 +301,102 @@ export class FileWatcher {
   async indexMarkdownFile(filePath, content, contentHash, stats) {
     console.error(`[FileWatcher] Indexing Markdown: ${filePath}`);
     
-    // Delete old chunks if updating
-    const relativePath = path.relative(this.projectPath, filePath);
-    this.memory.memoryDb?.deleteChunksByFile?.(relativePath);
-    
-    // Chunk the content
-    const { MarkdownChunker } = await import('../utils/markdown-chunker.js');
-    const chunker = new MarkdownChunker();
-    const chunks = chunker.chunk(content, relativePath);
-    
-    // Track file
-    this.memory.memoryDb?.upsertFile?.(relativePath, contentHash, stats.mtimeMs, stats.size);
-    
-    // Generate embeddings and insert chunks
-    for (const chunk of chunks) {
-      // Check cache first
-      let embedding = null;
-      const cached = this.memory.memoryDb?.getCachedEmbedding?.(chunk.contentHash);
+    try {
+      // Delete old chunks if updating
+      const relativePath = path.relative(this.projectPath, filePath);
+      this.memory.memoryDb?.deleteChunksByFile?.(relativePath);
       
-      if (cached) {
-        embedding = cached.embedding;
-      } else {
-        // Generate embedding
-        embedding = await this.memory.embedder?.embedCode?.(chunk.content);
-        
-        // Cache it
-        if (embedding) {
-          this.memory.memoryDb?.cacheEmbedding?.(
-            chunk.contentHash,
-            Buffer.from(new Float32Array(embedding).buffer),
-            'simple-embedder',
-            embedding.length
-          );
-        }
-      }
+      // Chunk the content
+      const { MarkdownChunker } = await import('../utils/markdown-chunker.js');
+      const chunker = new MarkdownChunker();
+      const chunks = chunker.chunk(content, relativePath);
       
-      // Insert chunk
-      this.memory.memoryDb?.insertChunk?.({
-        id: chunk.id,
-        file_path: chunk.filePath,
-        start_line: chunk.startLine,
-        end_line: chunk.endLine,
-        content: chunk.content,
-        content_hash: chunk.contentHash,
-        chunk_type: chunk.chunkType
-      });
+      // Track file
+      this.memory.memoryDb?.upsertFile?.(relativePath, contentHash, stats.mtimeMs, stats.size);
       
-      // Insert embedding
-      if (embedding) {
-        this.memory.memoryDb?.insertEmbedding?.({
-          chunk_id: chunk.id,
-          embedding: Buffer.from(new Float32Array(embedding).buffer),
-          model: 'simple-embedder',
-          dimensions: embedding.length
-        });
+      // Generate embeddings and insert chunks
+      for (const chunk of chunks) {
+        try {
+          // Check cache first
+          let embedding = null;
+          const cached = this.memory.memoryDb?.getCachedEmbedding?.(chunk.contentHash);
+          
+          if (cached) {
+            embedding = cached.embedding;
+          } else {
+            // Generate embedding
+            embedding = await this.memory.embedder?.embedCode?.(chunk.content);
+            
+            // Cache it
+            if (embedding) {
+              this.memory.memoryDb?.cacheEmbedding?.(
+                chunk.contentHash,
+                Buffer.from(new Float32Array(embedding).buffer),
+                'simple-embedder',
+                embedding.length
+              );
+            }
+          }
+          
+          // Insert chunk
+          this.memory.memoryDb?.insertChunk?.({
+            id: chunk.id,
+            file_path: chunk.filePath,
+            start_line: chunk.startLine,
+            end_line: chunk.endLine,
+            content: chunk.content,
+            content_hash: chunk.contentHash,
+            chunk_type: chunk.chunkType
+          });
+          
+          // Insert embedding
+          if (embedding) {
+            this.memory.memoryDb?.insertEmbedding?.({
+              chunk_id: chunk.id,
+              embedding: Buffer.from(new Float32Array(embedding).buffer),
+              model: 'simple-embedder',
+              dimensions: embedding.length
+            });
+          }
+        } catch (chunkError) {
+          console.error(`[FileWatcher] Error indexing chunk:`, chunkError.message);
+        }
       }
+      
+      console.error(`[FileWatcher] Indexed ${chunks.length} chunks from ${filePath}`);
+    } catch (error) {
+      console.error(`[FileWatcher] Error indexing markdown ${filePath}:`, error.message);
+      this.stats.errors++;
     }
-    
-    console.error(`[FileWatcher] Indexed ${chunks.length} chunks from ${filePath}`);
   }
 
-  async indexCodeFile(filePath, content, contentHash, stats) {
+  async indexCodeFile(filePath, content, contentHash, stats, maxRetries = 3) {
     console.error(`[FileWatcher] Indexing code: ${filePath}`);
-    
-    // Use the existing indexer
-    if (this.memory.indexer) {
+
+    if (!this.memory.indexer) {
+      console.error(`[FileWatcher] No indexer available`);
+      return;
+    }
+
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
       try {
         const indexed = await this.memory.indexer.indexFile(filePath, content);
-        await this.memory.storeIndexed(indexed);
-        console.error(`[FileWatcher] Indexed ${indexed.facts.length} facts, ${indexed.chunks.length} chunks from ${filePath}`);
+        const stored = await this.memory.storeIndexed(indexed);
+
+        if (stored) {
+          console.error(`[FileWatcher] Indexed ${indexed.facts.length} facts, ${indexed.chunks.length} chunks`);
+          return;
+        } else {
+          throw new Error('storeIndexed returned false');
+        }
       } catch (error) {
-        console.error(`[FileWatcher] Failed to index ${filePath}:`, error.message);
+        if (attempt === maxRetries) {
+          console.error(`[FileWatcher] Failed to index after ${maxRetries} attempts:`, error.message);
+          this.stats.errors++;
+        } else {
+          console.warn(`[FileWatcher] Retry ${attempt}/${maxRetries}:`, error.message);
+          await new Promise(resolve => setTimeout(resolve, 100 * Math.pow(2, attempt - 1)));
+        }
       }
     }
   }
@@ -302,25 +406,75 @@ export class FileWatcher {
     return ext === '.md' || ext === '.markdown';
   }
 
+  async indexLargeFile(filePath, content, contentHash, stats) {
+    const chunkSize = 500 * 1024;
+    try {
+      const lines = content.split('\n');
+      let chunkContent = '';
+      let chunkIndex = 0;
+
+      for (let i = 0; i < lines.length; i++) {
+        chunkContent += lines[i] + '\n';
+
+        if (chunkContent.length >= chunkSize || i === lines.length - 1) {
+          console.error(`[FileWatcher] Indexing chunk ${chunkIndex + 1} of large file`);
+
+          if (this.isMarkdownFile(filePath)) {
+            await this.indexMarkdownFile(filePath, chunkContent, contentHash + '_' + chunkIndex, stats);
+          } else {
+            await this.indexCodeFile(filePath, chunkContent, contentHash + '_' + chunkIndex, stats);
+          }
+
+          chunkIndex++;
+          chunkContent = '';
+        }
+      }
+
+      console.error(`[FileWatcher] Large file indexed in ${chunkIndex} chunks`);
+      this.stats.filesChanged++;
+
+    } catch (error) {
+      console.error(`[FileWatcher] Failed to index large file:`, error.message);
+      this.stats.errors++;
+    }
+  }
+
   getStats() {
     return {
       ...this.stats,
       pendingUpdates: this.pendingUpdates.size,
-      isProcessing: this.isProcessing
+      isProcessing: this.isProcessing,
+      isRunning: this.isRunning
     };
   }
 
   async stop() {
+    if (!this.isRunning) return;
+    
+    console.error('[FileWatcher] Stopping...');
+    
+    // 1. Stop cleanup interval
+    if (this.cleanupInterval) {
+      clearInterval(this.cleanupInterval);
+      this.cleanupInterval = null;
+    }
+    
+    // 2. Clear pending updates map (free memory)
+    this.pendingUpdates.clear();
+    
+    // 3. Cancel debounce timeout if exists
+    if (this.flushQueue && this.flushQueue.cancel) {
+      this.flushQueue.cancel();
+    }
+    
+    // 4. Close chokidar watcher (critical - prevents persistent file watching)
     if (this.watcher) {
-      // Process any remaining updates
-      if (this.pendingUpdates.size > 0) {
-        await this.processQueue();
-      }
-      
       await this.watcher.close();
       this.watcher = null;
-      console.error(`[FileWatcher] Stopped`);
     }
+    
+    this.isRunning = false;
+    console.error('[FileWatcher] Stopped');
   }
 }