@mrxkun/mcfast-mcp 4.0.15 → 4.1.0

package/src/memory/watchers/file-watcher.js

@@ -1,6 +1,15 @@
+/**
+ * File Watcher
+ * Theo dõi thay đổi file và index tự động
+ * Debounced 1.5s để batch rapid changes
+ */
+
 import chokidar from 'chokidar';
+import fs from 'fs/promises';
+import path from 'path';
+import crypto from 'crypto';
 
-// Simple debounce implementation to avoid lodash dependency
+// Simple debounce implementation
 function debounce(func, wait) {
   let timeout;
   return function executedFunction(...args) {
@@ -14,47 +23,304 @@ function debounce(func, wait) {
 }
 
 export class FileWatcher {
-  constructor(projectPath, memoryEngine) {
+  constructor(projectPath, memoryEngine, options = {}) {
     this.projectPath = projectPath;
     this.memory = memoryEngine;
     this.watcher = null;
     this.pendingUpdates = new Map();
+    this.isProcessing = false;
+    
+    // Configuration
+    this.debounceMs = options.debounceMs || 1500;
+    this.ignored = options.ignored || [
+      '**/node_modules/**',
+      '**/.git/**',
+      '**/dist/**',
+      '**/build/**',
+      '**/.mcfast/**',
+      '**/*.log'
+    ];
+    
+    // Statistics
+    this.stats = {
+      filesAdded: 0,
+      filesChanged: 0,
+      filesDeleted: 0,
+      errors: 0
+    };
   }
 
   async start() {
+    console.log(`[FileWatcher] Starting watcher for: ${this.projectPath}`);
+    console.log(`[FileWatcher] Debounce: ${this.debounceMs}ms`);
+    
     this.watcher = chokidar.watch(this.projectPath, {
-      ignored: ['**/node_modules/**', '**/.git/**', '**/dist/**'],
-      persistent: true
+      ignored: this.ignored,
+      persistent: true,
+      ignoreInitial: true, // Don't trigger on existing files
+      awaitWriteFinish: {
+        stabilityThreshold: 300,
+        pollInterval: 100
+      }
+    });
+
+    // Bind event handlers
+    this.watcher.on('add', filePath => this.handleAdd(filePath));
+    this.watcher.on('change', filePath => this.handleChange(filePath));
+    this.watcher.on('unlink', filePath => this.handleDelete(filePath));
+    this.watcher.on('error', error => this.handleError(error));
+    
+    // Setup debounced flush
+    this.flushQueue = debounce(() => this.processQueue(), this.debounceMs);
+    
+    // Wait for ready
+    await new Promise((resolve, reject) => {
+      this.watcher.once('ready', resolve);
+      this.watcher.once('error', reject);
+    });
+    
+    console.log(`[FileWatcher] Ready and watching`);
+  }
+
+  handleAdd(filePath) {
+    console.log(`[FileWatcher] File added: ${filePath}`);
+    this.pendingUpdates.set(filePath, {
+      path: filePath,
+      type: 'add',
+      timestamp: Date.now()
+    });
+    this.flushQueue();
+  }
+
+  handleChange(filePath) {
+    console.log(`[FileWatcher] File changed: ${filePath}`);
+    this.pendingUpdates.set(filePath, {
+      path: filePath,
+      type: 'change',
+      timestamp: Date.now()
     });
+    this.flushQueue();
+  }
+
+  handleDelete(filePath) {
+    console.log(`[FileWatcher] File deleted: ${filePath}`);
+    this.pendingUpdates.set(filePath, {
+      path: filePath,
+      type: 'delete',
+      timestamp: Date.now()
+    });
+    this.flushQueue();
+  }
+
+  handleError(error) {
+    console.error(`[FileWatcher] Error:`, error);
+    this.stats.errors++;
+  }
 
-    this.watcher.on('add', path => this.queueUpdate(path, 'add'));
-    this.watcher.on('change', path => this.queueUpdate(path, 'change'));
-    this.watcher.on('unlink', path => this.handleDelete(path));
+  async processQueue() {
+    if (this.isProcessing) {
+      // If already processing, debounce will call again
+      return;
+    }
+    
+    this.isProcessing = true;
+    
+    try {
+      const updates = Array.from(this.pendingUpdates.values());
+      this.pendingUpdates.clear();
+      
+      if (updates.length === 0) return;
+      
+      console.log(`[FileWatcher] Processing ${updates.length} updates...`);
+      
+      // Group by type for efficiency
+      const adds = updates.filter(u => u.type === 'add' || u.type === 'change');
+      const deletes = updates.filter(u => u.type === 'delete');
+      
+      // Process deletions first
+      for (const update of deletes) {
+        await this.processDelete(update);
+      }
+      
+      // Process additions/changes
+      for (const update of adds) {
+        await this.processFile(update);
+      }
+      
+      console.log(`[FileWatcher] Processed ${updates.length} updates`);
+      
+    } catch (error) {
+      console.error(`[FileWatcher] Error processing queue:`, error);
+      this.stats.errors++;
+    } finally {
+      this.isProcessing = false;
+    }
+  }
 
-    this.processQueue = debounce(() => this.flushQueue(), 1500);
+  async processFile(update) {
+    const filePath = update.path;
+    
+    try {
+      // Check if file exists and is readable
+      const stats = await fs.stat(filePath).catch(() => null);
+      if (!stats || !stats.isFile()) return;
+      
+      // Skip large files (> 1MB)
+      if (stats.size > 1024 * 1024) {
+        console.log(`[FileWatcher] Skipping large file: ${filePath} (${Math.round(stats.size / 1024)}KB)`);
+        return;
+      }
+      
+      // Read file content
+      const content = await fs.readFile(filePath, 'utf-8');
+      const contentHash = crypto.createHash('md5').update(content).digest('hex');
+      
+      // Check if already indexed with same hash
+      if (this.memory.codebaseDb?.isFileIndexed?.(filePath, contentHash)) {
+        console.log(`[FileWatcher] File unchanged: ${filePath}`);
+        return;
+      }
+      
+      // Determine file type and index accordingly
+      if (this.isMarkdownFile(filePath)) {
+        await this.indexMarkdownFile(filePath, content, contentHash, stats);
+      } else {
+        await this.indexCodeFile(filePath, content, contentHash, stats);
+      }
+      
+      if (update.type === 'add') {
+        this.stats.filesAdded++;
+      } else {
+        this.stats.filesChanged++;
+      }
+      
+    } catch (error) {
+      console.error(`[FileWatcher] Error processing ${filePath}:`, error.message);
+      this.stats.errors++;
+    }
   }
 
-  queueUpdate(filePath, type) {
-    this.pendingUpdates.set(filePath, { path: filePath, type, timestamp: Date.now() });
-    this.processQueue();
+  async processDelete(update) {
+    const filePath = update.path;
+    
+    try {
+      // Delete from codebase database
+      const file = this.memory.codebaseDb?.getFileByPath?.(filePath);
+      if (file) {
+        this.memory.codebaseDb.deleteFile(file.id);
+        console.log(`[FileWatcher] Deleted from index: ${filePath}`);
+      }
+      
+      this.stats.filesDeleted++;
+      
+    } catch (error) {
+      console.error(`[FileWatcher] Error deleting ${filePath}:`, error.message);
+      this.stats.errors++;
+    }
   }
 
-  async flushQueue() {
-    const updates = Array.from(this.pendingUpdates.values());
-    this.pendingUpdates.clear();
-    await Promise.all(updates.map(u => this.processUpdate(u)));
+  async indexMarkdownFile(filePath, content, contentHash, stats) {
+    console.log(`[FileWatcher] Indexing Markdown: ${filePath}`);
+    
+    // Delete old chunks if updating
+    const relativePath = path.relative(this.projectPath, filePath);
+    this.memory.memoryDb?.deleteChunksByFile?.(relativePath);
+    
+    // Chunk the content
+    const { MarkdownChunker } = await import('../utils/markdown-chunker.js');
+    const chunker = new MarkdownChunker();
+    const chunks = chunker.chunk(content, relativePath);
+    
+    // Track file
+    this.memory.memoryDb?.upsertFile?.(relativePath, contentHash, stats.mtimeMs, stats.size);
+    
+    // Generate embeddings and insert chunks
+    for (const chunk of chunks) {
+      // Check cache first
+      let embedding = null;
+      const cached = this.memory.memoryDb?.getCachedEmbedding?.(chunk.contentHash);
+      
+      if (cached) {
+        embedding = cached.embedding;
+      } else {
+        // Generate embedding
+        embedding = await this.memory.embedder?.embedCode?.(chunk.content);
+        
+        // Cache it
+        if (embedding) {
+          this.memory.memoryDb?.cacheEmbedding?.(
+            chunk.contentHash,
+            Buffer.from(new Float32Array(embedding).buffer),
+            'simple-embedder',
+            embedding.length
+          );
+        }
+      }
+      
+      // Insert chunk
+      this.memory.memoryDb?.insertChunk?.({
+        id: chunk.id,
+        file_path: chunk.filePath,
+        start_line: chunk.startLine,
+        end_line: chunk.endLine,
+        content: chunk.content,
+        content_hash: chunk.contentHash,
+        chunk_type: chunk.chunkType
+      });
+      
+      // Insert embedding
+      if (embedding) {
+        this.memory.memoryDb?.insertEmbedding?.({
+          chunk_id: chunk.id,
+          embedding: Buffer.from(new Float32Array(embedding).buffer),
+          model: 'simple-embedder',
+          dimensions: embedding.length
+        });
+      }
+    }
+    
+    console.log(`[FileWatcher] Indexed ${chunks.length} chunks from ${filePath}`);
   }
 
-  async processUpdate(update) {
-    console.log(`[Watcher] ${update.type}: ${update.path}`);
+  async indexCodeFile(filePath, content, contentHash, stats) {
+    console.log(`[FileWatcher] Indexing code: ${filePath}`);
+    
+    // Use the existing indexer
+    if (this.memory.indexer) {
+      try {
+        const indexed = await this.memory.indexer.indexFile(filePath, content);
+        await this.memory.storeIndexed(indexed);
+        console.log(`[FileWatcher] Indexed ${indexed.facts.length} facts, ${indexed.chunks.length} chunks from ${filePath}`);
+      } catch (error) {
+        console.error(`[FileWatcher] Failed to index ${filePath}:`, error.message);
+      }
+    }
   }
 
-  async handleDelete(filePath) {
-    console.log(`[Watcher] Deleted: ${filePath}`);
+  isMarkdownFile(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    return ext === '.md' || ext === '.markdown';
+  }
+
+  getStats() {
+    return {
+      ...this.stats,
+      pendingUpdates: this.pendingUpdates.size,
+      isProcessing: this.isProcessing
+    };
   }
 
   async stop() {
-    if (this.watcher) await this.watcher.close();
+    if (this.watcher) {
+      // Process any remaining updates
+      if (this.pendingUpdates.size > 0) {
+        await this.processQueue();
+      }
+      
+      await this.watcher.close();
+      this.watcher = null;
+      console.log(`[FileWatcher] Stopped`);
+    }
   }
 }