@arela/uploader 0.2.12 → 0.3.0

package/src/utils/WatchEventHandler.js

@@ -0,0 +1,517 @@
+import fs from 'fs/promises';
+import path from 'path';
+
+import autoProcessingService from '../services/AutoProcessingService.js';
+import logger from '../services/LoggingService.js';
+
+/**
+ * WatchEventHandler - Processes file system events and manages upload strategy
+ * Handles debouncing, file tracking, and determines what files should be uploaded
+ */
+export class WatchEventHandler {
+  constructor() {
+    this.recentFiles = new Map(); // Map<filePath, {event, timestamp, size}>
+    this.eventBatches = new Map(); // Map<batchKey, Set<filePath>>
+    this.fileStates = new Map(); // Map<filePath, {size, mtime}>
+    this.lastProcessedTime = 0;
+    this.minBatchInterval = 100; // Minimum ms between batch processing
+  }
+
+  /**
+   * Register a file event
+   * @param {string} eventType - Type of event (add, change, unlink)
+   * @param {string} filePath - Path to the file
+   * @param {number} timestamp - Event timestamp
+   * @returns {void}
+   */
+  registerEvent(eventType, filePath, timestamp = Date.now()) {
+    try {
+      const normalizedPath = path.normalize(filePath);
+
+      logger.debug(
+        `[EventHandler] Registering ${eventType} event: ${normalizedPath}`,
+      );
+
+      // Update recent files map
+      this.recentFiles.set(normalizedPath, {
+        event: eventType,
+        timestamp,
+        originalPath: filePath,
+      });
+
+      // Add to current batch
+      const batchKey = this.#generateBatchKey(timestamp);
+      if (!this.eventBatches.has(batchKey)) {
+        this.eventBatches.set(batchKey, new Set());
+      }
+      this.eventBatches.get(batchKey).add(normalizedPath);
+
+      logger.debug(
+        `[EventHandler] Batch ${batchKey}: ${this.eventBatches.get(batchKey).size} files`,
+      );
+    } catch (error) {
+      logger.error(`Error registering event: ${error.message}`);
+    }
+  }
+
+  /**
+   * Get files that should be uploaded based on strategy
+   * @param {string} strategy - Upload strategy (individual|batch|full-structure)
+   * @param {Object} options - Additional options
+   * @returns {Promise<Array>} Array of file objects to upload
+   */
+  async getFilesForUpload(strategy, options = {}) {
+    try {
+      logger.debug(
+        `[EventHandler] Getting files for upload with strategy: ${strategy}`,
+      );
+
+      switch (strategy) {
+        case 'individual':
+          return await this.#getIndividualFiles(options);
+        case 'batch':
+          return await this.#getBatchFiles(options);
+        case 'full-structure':
+          return await this.#getFullStructureFiles(options);
+        default:
+          throw new Error(`Unknown strategy: ${strategy}`);
+      }
+    } catch (error) {
+      logger.error(`Error getting files for upload: ${error.message}`);
+      return [];
+    }
+  }
+
+  /**
+   * Process newly added files with the automatic 4-step pipeline
+   * @param {Object} options - Processing options
+   * @param {string} options.watchDir - Watch directory where file was detected
+   * @param {string} options.folderStructure - Folder structure for organization
+   * @param {number} options.batchSize - Batch size for processing
+   * @returns {Promise<Object>} Result from AutoProcessingService
+   */
+  async processNewFilesWithPipeline(options = {}) {
+    try {
+      const { watchDir, folderStructure, batchSize = 10 } = options;
+
+      // Get the most recently added file
+      let newestFile = null;
+      let newestTime = 0;
+
+      for (const [filePath, metadata] of this.recentFiles.entries()) {
+        if (metadata.event === 'add' && metadata.timestamp > newestTime) {
+          newestTime = metadata.timestamp;
+          newestFile = filePath;
+        }
+      }
+
+      if (!newestFile) {
+        logger.warn(
+          '[EventHandler] No newly added files found for pipeline processing',
+        );
+        return {
+          success: false,
+          reason: 'No new files detected',
+        };
+      }
+
+      logger.info(
+        `[EventHandler] Processing newly detected file with pipeline: ${newestFile}`,
+      );
+
+      // Execute the 4-step processing pipeline
+      const result = await autoProcessingService.executeProcessingPipeline({
+        filePath: newestFile,
+        watchDir,
+        folderStructure,
+        batchSize,
+      });
+
+      return result;
+    } catch (error) {
+      logger.error(
+        `[EventHandler] Error processing files with pipeline: ${error.message}`,
+      );
+      return {
+        success: false,
+        error: error.message,
+      };
+    }
+  }
+
+  /**
+   * Get individual file (the most recently modified)
+   * @private
+   * @param {Object} options - Options with sourceDir, etc
+   * @returns {Promise<Array>} Array with single file object
+   */
+  async #getIndividualFiles(options = {}) {
+    try {
+      // Find the most recently modified file
+      let mostRecentFile = null;
+      let mostRecentTime = 0;
+
+      for (const [filePath, metadata] of this.recentFiles.entries()) {
+        if (
+          metadata.event !== 'unlink' &&
+          metadata.timestamp > mostRecentTime
+        ) {
+          mostRecentTime = metadata.timestamp;
+          mostRecentFile = filePath;
+        }
+      }
+
+      if (!mostRecentFile) {
+        logger.warn(
+          '[EventHandler] No valid files found for individual upload',
+        );
+        return [];
+      }
+
+      // Check if file still exists and is readable
+      try {
+        const stats = await fs.stat(mostRecentFile);
+        const fileObj = {
+          path: mostRecentFile,
+          name: path.basename(mostRecentFile),
+          size: stats.size,
+          mtime: stats.mtime,
+          type: 'individual',
+        };
+
+        logger.info(
+          `[EventHandler] Individual file for upload: ${mostRecentFile} (${stats.size} bytes)`,
+        );
+        return [fileObj];
+      } catch (error) {
+        logger.warn(
+          `[EventHandler] File not accessible: ${mostRecentFile} - ${error.message}`,
+        );
+        return [];
+      }
+    } catch (error) {
+      logger.error(`Error in #getIndividualFiles: ${error.message}`);
+      return [];
+    }
+  }
+
+  /**
+   * Get batch of recent files, grouped by directory
+   * When processing batch, ensure all files in a directory are processed together
+   * This is critical for pedimento detection and document grouping
+   * @private
+   * @param {Object} options - Options with batchSize, sourceDir
+   * @returns {Promise<Array>} Array of file objects
+   */
+  async #getBatchFiles(options = {}) {
+    try {
+      const batchSize = options.batchSize || 10;
+      const sourceDir = options.sourceDir;
+      const files = [];
+
+      // Get all recent files (valid, non-deleted ones)
+      const validFiles = Array.from(this.recentFiles.entries())
+        .filter(([_, metadata]) => metadata.event !== 'unlink');
+
+      if (validFiles.length === 0) {
+        logger.debug('[EventHandler] No valid files in batch');
+        return [];
+      }
+
+      // If sourceDir provided, get ALL files from that directory
+      // This ensures when one file is detected, all related files are processed together
+      if (sourceDir) {
+        logger.debug(`[EventHandler] Batch: Processing all files in ${sourceDir}`);
+        
+        for (const [filePath, metadata] of validFiles) {
+          const fileDir = path.dirname(filePath);
+          
+          // Only include files from the source directory or its subdirectories
+          if (filePath.startsWith(sourceDir)) {
+            try {
+              const stats = await fs.stat(filePath);
+              files.push({
+                path: filePath,
+                name: path.basename(filePath),
+                size: stats.size,
+                mtime: stats.mtime,
+                event: metadata.event,
+                type: 'batch',
+              });
+              
+              if (files.length >= batchSize) {
+                logger.debug(`[EventHandler] Batch size limit (${batchSize}) reached`);
+                break;
+              }
+            } catch (error) {
+              logger.debug(
+                `[EventHandler] File not accessible for batch: ${filePath}`,
+              );
+              // Skip files that don't exist or are not accessible
+            }
+          }
+        }
+      } else {
+        // Fallback: Sort by timestamp (newest first) and take up to batchSize
+        const sortedFiles = validFiles.sort(
+          ([_, a], [__, b]) => b.timestamp - a.timestamp
+        );
+
+        for (const [filePath, metadata] of sortedFiles) {
+          try {
+            const stats = await fs.stat(filePath);
+            files.push({
+              path: filePath,
+              name: path.basename(filePath),
+              size: stats.size,
+              mtime: stats.mtime,
+              event: metadata.event,
+              type: 'batch',
+            });
+            
+            if (files.length >= batchSize) {
+              break;
+            }
+          } catch (error) {
+            logger.debug(
+              `[EventHandler] File not accessible for batch: ${filePath}`,
+            );
+            // Skip files that don't exist or are not accessible
+          }
+        }
+      }
+
+      logger.info(
+        `[EventHandler] Batch upload: ${files.length} files (max ${batchSize}) from ${sourceDir || 'all directories'}`,
+      );
+      return files;
+    } catch (error) {
+      logger.error(`Error in #getBatchFiles: ${error.message}`);
+      return [];
+    }
+  }
+
+  /**
+   * Get entire directory structure containing affected files
+   * @private
+   * @param {Object} options - Options with sourceDir
+   * @returns {Promise<Array>} Array of file objects for entire structure
+   */
+  async #getFullStructureFiles(options = {}) {
+    try {
+      const sourceDir = options.sourceDir;
+      if (!sourceDir) {
+        logger.warn(
+          '[EventHandler] sourceDir option required for full-structure strategy',
+        );
+        return [];
+      }
+
+      // Get the top-level directory containing the modified files
+      const affectedDirs = new Set();
+      for (const filePath of this.recentFiles.keys()) {
+        const dir = path.dirname(filePath);
+        affectedDirs.add(dir);
+      }
+
+      logger.debug(`[EventHandler] Affected directories: ${affectedDirs.size}`);
+
+      // For each affected directory, collect all files in its structure
+      const files = [];
+      const processedFiles = new Set();
+
+      for (const affectedDir of affectedDirs) {
+        await this.#collectDirectoryStructure(
+          affectedDir,
+          files,
+          processedFiles,
+        );
+      }
+
+      logger.info(
+        `[EventHandler] Full structure upload: ${files.length} files`,
+      );
+      return files;
+    } catch (error) {
+      logger.error(`Error in #getFullStructureFiles: ${error.message}`);
+      return [];
+    }
+  }
+
+  /**
+   * Recursively collect all files in a directory structure
+   * @private
+   * @param {string} dirPath - Directory path
+   * @param {Array} files - Array to accumulate files
+   * @param {Set} processedFiles - Set of already processed files
+   * @returns {Promise<void>}
+   */
+  async #collectDirectoryStructure(
+    dirPath,
+    files,
+    processedFiles,
+    maxDepth = 5,
+    currentDepth = 0,
+  ) {
+    try {
+      if (currentDepth > maxDepth) {
+        logger.debug(`[EventHandler] Max directory depth reached: ${maxDepth}`);
+        return;
+      }
+
+      const entries = await fs.readdir(dirPath, { withFileTypes: true });
+
+      for (const entry of entries) {
+        // Skip hidden files and common ignore patterns
+        if (
+          entry.name.startsWith('.') ||
+          entry.name === 'node_modules' ||
+          entry.name === '__pycache__'
+        ) {
+          continue;
+        }
+
+        const fullPath = path.join(dirPath, entry.name);
+
+        if (entry.isDirectory()) {
+          // Recursively process subdirectories
+          await this.#collectDirectoryStructure(
+            fullPath,
+            files,
+            processedFiles,
+            maxDepth,
+            currentDepth + 1,
+          );
+        } else if (entry.isFile()) {
+          // Add file if not already processed
+          if (!processedFiles.has(fullPath)) {
+            try {
+              const stats = await fs.stat(fullPath);
+              files.push({
+                path: fullPath,
+                name: entry.name,
+                size: stats.size,
+                mtime: stats.mtime,
+                type: 'full-structure',
+                relativePath: path.relative(path.dirname(dirPath), fullPath),
+              });
+              processedFiles.add(fullPath);
+            } catch (error) {
+              logger.debug(`[EventHandler] Cannot stat file: ${fullPath}`);
+            }
+          }
+        }
+      }
+    } catch (error) {
+      logger.debug(
+        `[EventHandler] Error collecting directory: ${dirPath} - ${error.message}`,
+      );
+      // Don't throw - continue with other directories
+    }
+  }
+
+  /**
+   * Get all recent events
+   * @returns {Array} Array of recent file events
+   */
+  getRecentEvents() {
+    return Array.from(this.recentFiles.entries()).map(
+      ([filePath, metadata]) => ({
+        path: filePath,
+        ...metadata,
+      }),
+    );
+  }
+
+  /**
+   * Clear recent files and batches (after upload)
+   * @returns {void}
+   */
+  clearProcessed() {
+    logger.debug(
+      `[EventHandler] Clearing ${this.recentFiles.size} recent files`,
+    );
+    this.recentFiles.clear();
+    this.eventBatches.clear();
+  }
+
+  /**
+   * Get statistics about current events
+   * @returns {Object} Statistics object
+   */
+  getStats() {
+    const adds = Array.from(this.recentFiles.values()).filter(
+      (m) => m.event === 'add',
+    ).length;
+    const changes = Array.from(this.recentFiles.values()).filter(
+      (m) => m.event === 'change',
+    ).length;
+    const unlinks = Array.from(this.recentFiles.values()).filter(
+      (m) => m.event === 'unlink',
+    ).length;
+
+    return {
+      totalEvents: this.recentFiles.size,
+      adds,
+      changes,
+      unlinks,
+      activeBatches: this.eventBatches.size,
+    };
+  }
+
+  /**
+   * Generate batch key based on time window
+   * @private
+   * @param {number} timestamp - Event timestamp
+   * @returns {string} Batch key
+   */
+  #generateBatchKey(timestamp) {
+    // Group events into 1-second windows
+    const windowSize = 1000;
+    const windowId = Math.floor(timestamp / windowSize);
+    return `batch_${windowId}`;
+  }
+
+  /**
+   * Check if a file should be ignored based on patterns
+   * @param {string} filePath - File path
+   * @param {Array} ignorePatterns - Regex patterns to ignore
+   * @returns {boolean} True if file should be ignored
+   */
+  shouldIgnoreFile(filePath, ignorePatterns = []) {
+    if (!ignorePatterns || ignorePatterns.length === 0) {
+      return false;
+    }
+
+    const fileName = path.basename(filePath);
+    const dirName = path.dirname(filePath);
+
+    return ignorePatterns.some((pattern) => {
+      try {
+        const regex = new RegExp(pattern);
+        return (
+          regex.test(filePath) || regex.test(fileName) || regex.test(dirName)
+        );
+      } catch (error) {
+        logger.warn(
+          `[EventHandler] Invalid ignore pattern: ${pattern} - ${error.message}`,
+        );
+        return false;
+      }
+    });
+  }
+
+  /**
+   * Reset event handler state
+   */
+  reset() {
+    this.recentFiles.clear();
+    this.eventBatches.clear();
+    this.fileStates.clear();
+    this.lastProcessedTime = 0;
+    logger.debug('[EventHandler] Handler reset');
+  }
+}
+
+// Export singleton instance
+export default new WatchEventHandler();