@arela/uploader 0.2.13 → 1.0.0

package/src/services/AutoProcessingService.js

@@ -0,0 +1,749 @@
+import { appConfig } from '../config/config.js';
+import logger from './LoggingService.js';
+
+/**
+ * AutoProcessingService - Handles automatic processing workflow for newly detected files
+ * Executes the 4-step processing pipeline:
+ * 1. Stats collection (stats --stats-only) - uses sourceApi in cross-tenant mode
+ * 2. PDF/Pedimento detection (detect --detect-pdfs) - uses sourceApi in cross-tenant mode
+ * 3. Arela path propagation (detect --propagate-arela-path) - uses sourceApi in cross-tenant mode
+ * 4. RFC-based upload with folder structure (upload --upload-by-rfc --folder-structure) - uses targetApi in cross-tenant mode
+ *
+ * In cross-tenant mode (--source-api and --target-api):
+ * - Phases 1-3 write/update to the SOURCE API
+ * - Phase 4 uploads files to the TARGET API
+ *
+ * In single API mode (--api):
+ * - All phases use the same API
+ */
+export class AutoProcessingService {
+  constructor() {
+    this.isProcessing = false;
+    this.processingStats = {
+      totalProcessed: 0,
+      successfulSteps: 0,
+      failedSteps: 0,
+      errors: [],
+    };
+    this.errorTracker = new Map(); // Track errors by directory
+    this.processingTimeout = null; // Auto-reset processing flag
+    this.processingTimeoutDuration = 30000; // 30 seconds max
+    this.lastProcessedFile = null; // Track last file to avoid duplicates
+    this.processingDebounceMs = 500; // Debounce delay
+  }
+
+  /**
+   * Execute the complete 4-step processing pipeline
+   * @param {Object} options - Configuration options
+   * @param {string} options.filePath - Path of the newly detected file
+   * @param {string} options.watchDir - Watch directory being monitored
+   * @param {string} options.folderStructure - Folder structure for upload
+   * @param {number} options.batchSize - Batch size for processing
+   * @returns {Promise<Object>} Result of the processing pipeline
+   */
+  async executeProcessingPipeline(options = {}) {
+    const { filePath, watchDir, folderStructure, batchSize = 10 } = options;
+
+    const pipelineId = this.#generatePipelineId();
+    const startTime = Date.now();
+
+    logger.debug(`🔄 Pipeline ${pipelineId} started for: ${filePath}`);
+
+    // Prevent concurrent processing with debounce
+    if (this.isProcessing) {
+      // Check if this is a duplicate of the last processed file
+      if (this.lastProcessedFile === filePath) {
+        logger.debug(
+          `⏭️  Skipping duplicate file request (same file within ${this.processingDebounceMs}ms): ${filePath}`,
+        );
+        return {
+          success: false,
+          reason: 'Duplicate file request (debounced)',
+          pipelineId,
+        };
+      }
+
+      logger.warn(
+        '⚠️ Processing pipeline already running, skipping new request',
+      );
+      return {
+        success: false,
+        reason: 'Pipeline already processing',
+        pipelineId,
+      };
+    }
+
+    this.isProcessing = true;
+    this.lastProcessedFile = filePath;
+
+    // Set automatic timeout to reset processing flag (fail-safe)
+    this.#setProcessingTimeout();
+    const results = {
+      pipelineId,
+      filePath,
+      watchDir,
+      folderStructure,
+      steps: {
+        statsOnly: { status: 'pending', error: null },
+        detectPdfs: { status: 'pending', error: null },
+        propagateArelaPath: { status: 'pending', error: null },
+        uploadByRfc: { status: 'pending', error: null },
+      },
+      summary: {},
+      totalDuration: 0,
+    };
+
+    try {
+      // Determine API target for phases 1-3 (write operations)
+      // In cross-tenant mode: use sourceApi for phases 1-3
+      // In single API mode: use activeTarget
+      const isCrossTenant = appConfig.isCrossTenantMode();
+      const sourceApiTarget = isCrossTenant
+        ? appConfig.api.sourceTarget
+        : appConfig.api.activeTarget !== 'default'
+          ? appConfig.api.activeTarget
+          : null;
+
+      if (isCrossTenant) {
+        logger.debug(
+          `🔀 Cross-tenant mode: phases 1-3 will use ${appConfig.api.sourceTarget}, phase 4 will use ${appConfig.api.targetTarget}`,
+        );
+      } else if (sourceApiTarget) {
+        logger.debug(
+          `🎯 Single API mode: all phases will use ${sourceApiTarget}`,
+        );
+      }
+
+      // Step 1: Run stats collection (uses sourceApi in cross-tenant mode)
+      logger.debug(`📊 [Step 1/4] Stats collection...`);
+      results.steps.statsOnly = await this.#executeStatsOnly({
+        filePath,
+        watchDir,
+        batchSize,
+        apiTarget: sourceApiTarget,
+      });
+
+      if (results.steps.statsOnly.status === 'failed') {
+        logger.debug(`Stats failed: ${results.steps.statsOnly.error}`);
+        this.#trackError(watchDir, 'statsOnly', results.steps.statsOnly.error);
+        this.isProcessing = false;
+        results.summary = this.#generateSummary(results);
+        results.totalDuration = Date.now() - startTime;
+        return results;
+      }
+
+      // Step 2: Run PDF detection (uses sourceApi in cross-tenant mode)
+      logger.debug(`🔍 [Step 2/4] PDF detection...`);
+      results.steps.detectPdfs = await this.#executeDetectPdfs({
+        batchSize,
+        apiTarget: sourceApiTarget,
+      });
+
+      if (results.steps.detectPdfs.status === 'failed') {
+        logger.debug(`PDF detection failed: ${results.steps.detectPdfs.error}`);
+        this.#trackError(
+          watchDir,
+          'detectPdfs',
+          results.steps.detectPdfs.error,
+        );
+        this.isProcessing = false;
+        results.summary = this.#generateSummary(results);
+        results.totalDuration = Date.now() - startTime;
+        return results;
+      }
+
+      // Step 3: Propagate Arela path (uses sourceApi in cross-tenant mode)
+      logger.debug(`🔄 [Step 3/4] Arela path propagation...`);
+      results.steps.propagateArelaPath = await this.#executePropagateArelaPath({
+        apiTarget: sourceApiTarget,
+      });
+
+      if (results.steps.propagateArelaPath.status === 'failed') {
+        logger.debug(
+          `Arela path propagation failed: ${results.steps.propagateArelaPath.error}`,
+        );
+        this.#trackError(
+          watchDir,
+          'propagateArelaPath',
+          results.steps.propagateArelaPath.error,
+        );
+        // Don't stop pipeline on this step, as it might succeed partially
+      }
+
+      // Step 4: Upload by RFC with folder structure
+      // In cross-tenant mode: uses sourceApi for reading, targetApi for uploading
+      // In single API mode: uses the same API for both
+      logger.debug(`📤 [Step 4/4] RFC upload...`);
+      results.steps.uploadByRfc = await this.#executeUploadByRfc({
+        batchSize,
+        folderStructure,
+      });
+
+      if (results.steps.uploadByRfc.status === 'failed') {
+        logger.debug(`RFC upload failed: ${results.steps.uploadByRfc.error}`);
+        this.#trackError(
+          watchDir,
+          'uploadByRfc',
+          results.steps.uploadByRfc.error,
+        );
+      }
+
+      results.summary = this.#generateSummary(results);
+      results.totalDuration = Date.now() - startTime;
+
+      // Solo muestra resumen en modo info
+      if (results.summary.success) {
+        logger.info(`✅ Pipeline completed: ${watchDir}`);
+      }
+
+      return results;
+    } catch (error) {
+      logger.error(`Fatal error in pipeline: ${error.message}`);
+      this.#trackError(watchDir, 'fatal', error.message);
+      results.summary = {
+        success: false,
+        message: `Fatal error: ${error.message}`,
+      };
+      results.totalDuration = Date.now() - startTime;
+      return results;
+    } finally {
+      this.isProcessing = false;
+    }
+  }
+
+  /**
+   * Execute stats collection (Step 1)
+   * Scans the entire directory containing the detected file, not just the file itself
+   * @private
+   * @param {Object} options - Options for stats collection
+   * @returns {Promise<Object>} Result of stats collection
+   */
+  async #executeStatsOnly(options = {}) {
+    let stepStartTime = Date.now();
+    try {
+      // Import DatabaseService to collect stats directly
+      // This bypasses UploadCommand which has watch mode restrictions
+      const databaseService = (await import('./DatabaseService.js')).default;
+      const FileOperations = (await import('../utils/FileOperations.js'))
+        .default;
+      const fs = (await import('fs')).default;
+      const path = (await import('path')).default;
+
+      logger.debug(
+        `[AutoProcessingService] Executing stats collection for watch mode`,
+      );
+
+      // Get files from the watch directory
+      const { filePath, watchDir } = options;
+
+      if (!filePath || !watchDir) {
+        throw new Error(
+          'filePath and watchDir are required for stats collection',
+        );
+      }
+
+      // Wait for file to be fully written (with retries)
+      let fileStats = null;
+      let attempts = 0;
+      const maxAttempts = 10; // 10 attempts * 100ms = 1 second maximum wait
+      const retryDelay = 100; // milliseconds
+
+      while (attempts < maxAttempts && !fileStats) {
+        try {
+          if (fs.existsSync(filePath)) {
+            fileStats = FileOperations.getFileStats(filePath);
+          } else {
+            attempts++;
+            if (attempts < maxAttempts) {
+              logger.debug(
+                `File not yet ready (attempt ${attempts}/${maxAttempts}), waiting...`,
+              );
+              await new Promise((resolve) => setTimeout(resolve, retryDelay));
+            }
+          }
+        } catch (error) {
+          attempts++;
+          if (attempts < maxAttempts) {
+            logger.debug(
+              `Error reading file stats (attempt ${attempts}/${maxAttempts}): ${error.message}, retrying...`,
+            );
+            await new Promise((resolve) => setTimeout(resolve, retryDelay));
+          } else {
+            throw error;
+          }
+        }
+      }
+
+      if (!fileStats) {
+        throw new Error(
+          `File not found after ${maxAttempts} retries: ${filePath}`,
+        );
+      }
+
+      // Get the parent directory of the detected file
+      const parentDir = path.dirname(filePath);
+      const fileObjects = [];
+
+      // First, add the detected file
+      fileObjects.push({
+        path: filePath,
+        originalName: path.basename(filePath),
+        stats: fileStats,
+      });
+
+      // Then, scan the entire parent directory for related files
+      logger.debug(`📁 Scanning directory for related files: ${parentDir}`);
+      const relatedFiles = FileOperations.listFilesInDirectory(parentDir, {
+        excludePattern: /(^|[\/\\])\.|node_modules|\.git/,
+        onlyPdf: false,
+      });
+
+      // Add all related files (except the one already detected)
+      for (const relatedFilePath of relatedFiles) {
+        // Skip the file already detected and .DS_Store
+        if (
+          relatedFilePath === filePath ||
+          relatedFilePath.endsWith('.DS_Store')
+        ) {
+          continue;
+        }
+
+        try {
+          const relatedStats = FileOperations.getFileStats(relatedFilePath);
+          if (relatedStats) {
+            fileObjects.push({
+              path: relatedFilePath,
+              originalName: path.basename(relatedFilePath),
+              stats: relatedStats,
+            });
+          }
+        } catch (error) {
+          logger.debug(
+            `Could not read stats for related file: ${relatedFilePath}`,
+          );
+        }
+      }
+
+      logger.debug(
+        `📊 Found ${fileObjects.length} file(s) to process (1 detected + ${fileObjects.length - 1} related)`,
+      );
+
+      // Insert into database directly, using the specified API target
+      const insertOptions = {
+        quietMode: true, // Suppress verbose logging when in auto-pipeline
+      };
+
+      // Pass apiTarget if specified (for cross-tenant or single API mode)
+      if (options.apiTarget) {
+        insertOptions.apiTarget = options.apiTarget;
+        logger.debug(
+          `[AutoProcessingService] Stats using API target: ${options.apiTarget}`,
+        );
+      }
+
+      const result = await databaseService.insertStatsOnlyToUploaderTable(
+        fileObjects,
+        insertOptions,
+      );
+
+      const duration = Date.now() - stepStartTime;
+      logger.info(`✅ Stats collection completed in ${duration}ms`);
+
+      return {
+        status: 'success',
+        duration,
+        result: {
+          totalInserted: result.totalInserted,
+          totalSkipped: result.totalSkipped,
+        },
+        error: null,
+      };
+    } catch (error) {
+      logger.error(
+        `[AutoProcessingService] Stats collection error: ${error.message}`,
+      );
+      return {
+        status: 'failed',
+        duration: Date.now() - stepStartTime,
+        error: error.message,
+        result: null,
+      };
+    }
+  }
+
+  /**
+   * Execute PDF detection (Step 2)
+   * @private
+   * @param {Object} options - Options for PDF detection
+   * @param {string} options.apiTarget - API target to use (for cross-tenant or single API mode)
+   * @returns {Promise<Object>} Result of PDF detection
+   */
+  async #executeDetectPdfs(options = {}) {
+    let stepStartTime = Date.now();
+    try {
+      // Import databaseService singleton instance
+      const databaseService = (await import('./DatabaseService.js')).default;
+
+      logger.debug(
+        `[AutoProcessingService] Executing PDF detection with batch size: ${options.batchSize}`,
+      );
+
+      // Pass apiTarget if specified
+      const detectOptions = {
+        batchSize: options.batchSize || 10,
+      };
+
+      if (options.apiTarget) {
+        detectOptions.apiTarget = options.apiTarget;
+        logger.debug(
+          `[AutoProcessingService] PDF detection using API target: ${options.apiTarget}`,
+        );
+      }
+
+      const result =
+        await databaseService.detectPedimentosInDatabase(detectOptions);
+
+      const duration = Date.now() - stepStartTime;
+      logger.info(`✅ PDF detection completed in ${duration}ms`);
+      logger.info(
+        `   📊 Results: ${result.detectedCount} detected, ${result.processedCount} processed`,
+      );
+
+      return {
+        status: 'success',
+        duration,
+        result,
+        error: null,
+      };
+    } catch (error) {
+      logger.error(
+        `[AutoProcessingService] PDF detection error: ${error.message}`,
+      );
+      return {
+        status: 'failed',
+        duration: Date.now() - stepStartTime,
+        error: error.message,
+        result: null,
+      };
+    }
+  }
+
+  /**
+   * Execute arela_path propagation (Step 3)
+   * @private
+   * @param {Object} options - Options for propagation
+   * @param {string} options.apiTarget - API target to use (for cross-tenant or single API mode)
+   * @returns {Promise<Object>} Result of arela_path propagation
+   */
+  async #executePropagateArelaPath(options = {}) {
+    let stepStartTime = Date.now();
+    try {
+      // Import databaseService singleton instance
+      const databaseService = (await import('./DatabaseService.js')).default;
+
+      logger.debug(`[AutoProcessingService] Executing arela_path propagation`);
+
+      // Pass apiTarget if specified
+      const propagateOptions = {
+        showProgress: true,
+      };
+
+      if (options.apiTarget) {
+        propagateOptions.apiTarget = options.apiTarget;
+        logger.debug(
+          `[AutoProcessingService] Propagation using API target: ${options.apiTarget}`,
+        );
+      }
+
+      const result = await databaseService.propagateArelaPath(propagateOptions);
+
+      const duration = Date.now() - stepStartTime;
+      logger.info(`✅ Arela path propagation completed in ${duration}ms`);
+      logger.info(
+        `   📊 Results: ${result.processedCount} processed, ${result.updatedCount} updated`,
+      );
+
+      return {
+        status: 'success',
+        duration,
+        result,
+        error: null,
+      };
+    } catch (error) {
+      logger.error(
+        `[AutoProcessingService] Arela path propagation error: ${error.message}`,
+      );
+      return {
+        status: 'failed',
+        duration: Date.now() - stepStartTime,
+        error: error.message,
+        result: null,
+      };
+    }
+  }
+
+  /**
+   * Execute RFC-based upload with folder structure (Step 4)
+   * @private
+   * @param {Object} options - Options for RFC upload
+   * @param {string} options.folderStructure - Folder structure for organization
+   * @param {number} options.batchSize - Batch size for upload
+   * @returns {Promise<Object>} Result of RFC upload
+   */
+  async #executeUploadByRfc(options = {}) {
+    let stepStartTime = Date.now();
+    try {
+      // Import databaseService singleton instance
+      const databaseService = (await import('./DatabaseService.js')).default;
+
+      // Build upload options, including cross-tenant config if set globally
+      const uploadOptions = {
+        batchSize: options.batchSize || 10,
+        showProgress: true,
+        folderStructure: options.folderStructure,
+      };
+
+      // Check if cross-tenant mode is enabled globally and pass it to uploadFilesByRfc
+      if (appConfig.isCrossTenantMode()) {
+        // Cross-tenant mode: source API for reading, target API for uploading
+        uploadOptions.sourceApi = appConfig.api.sourceTarget;
+        uploadOptions.targetApi = appConfig.api.targetTarget;
+        logger.debug(
+          `[AutoProcessingService] Cross-tenant upload: source=${uploadOptions.sourceApi}, target=${uploadOptions.targetApi}`,
+        );
+      } else if (
+        appConfig.api.activeTarget &&
+        appConfig.api.activeTarget !== 'default'
+      ) {
+        // Single API mode: use the same API for both reading and uploading
+        uploadOptions.apiTarget = appConfig.api.activeTarget;
+        logger.debug(
+          `[AutoProcessingService] Single API upload: target=${uploadOptions.apiTarget}`,
+        );
+      }
+
+      logger.debug(
+        `[AutoProcessingService] Executing RFC-based upload with options:`,
+        {
+          folderStructure: options.folderStructure,
+          batchSize: options.batchSize,
+          sourceApi: uploadOptions.sourceApi,
+          targetApi: uploadOptions.targetApi,
+          apiTarget: uploadOptions.apiTarget,
+        },
+      );
+
+      const result = await databaseService.uploadFilesByRfc(uploadOptions);
+
+      const duration = Date.now() - stepStartTime;
+      logger.info(`✅ RFC upload completed in ${duration}ms`);
+      logger.info(
+        `   📊 Results: ${result.processedCount} processed, ${result.uploadedCount} uploaded`,
+      );
+
+      return {
+        status: 'success',
+        duration,
+        result,
+        error: null,
+      };
+    } catch (error) {
+      logger.error(
+        `[AutoProcessingService] RFC upload error: ${error.message}`,
+      );
+      return {
+        status: 'failed',
+        duration: Date.now() - stepStartTime,
+        error: error.message,
+        result: null,
+      };
+    }
+  }
+
+  /**
+   * Generate a unique pipeline ID for tracking
+   * @private
+   * @returns {string} Pipeline ID
+   */
+  #generatePipelineId() {
+    return `pipeline-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+  }
+
+  /**
+   * Generate processing summary
+   * @private
+   * @param {Object} results - Processing results
+   * @returns {Object} Summary object
+   */
+  #generateSummary(results) {
+    const steps = Object.entries(results.steps);
+    const successful = steps.filter(
+      ([_, step]) => step.status === 'success',
+    ).length;
+    const failed = steps.filter(([_, step]) => step.status === 'failed').length;
+
+    let message = '';
+    if (failed === 0) {
+      message = `✅ All 4 steps completed successfully!`;
+    } else if (successful > 0) {
+      message = `⚠️ Completed with issues: ${successful}/4 steps successful, ${failed} failed`;
+    } else {
+      message = `❌ Processing failed at step 1`;
+    }
+
+    return {
+      success: failed === 0,
+      totalSteps: steps.length,
+      successfulSteps: successful,
+      failedSteps: failed,
+      message,
+      details: steps.reduce((acc, [stepName, stepResult]) => {
+        acc[stepName] = {
+          status: stepResult.status,
+          duration: stepResult.duration,
+          error: stepResult.error,
+        };
+        return acc;
+      }, {}),
+    };
+  }
+
+  /**
+   * Get processing statistics
+   * @returns {Object} Processing statistics
+   */
+  getStats() {
+    return {
+      ...this.processingStats,
+      isProcessing: this.isProcessing,
+    };
+  }
+
+  /**
+   * Reset processing statistics
+   * @returns {void}
+   */
+  resetStats() {
+    this.processingStats = {
+      totalProcessed: 0,
+      successfulSteps: 0,
+      failedSteps: 0,
+      errors: [],
+    };
+    this.errorTracker.clear();
+  }
+
+  /**
+   * Track errors by directory for summary reporting
+   * @private
+   * @param {string} watchDir - Watch directory
+   * @param {string} step - Failed step
+   * @param {string} error - Error message
+   */
+  #trackError(watchDir, step, error) {
+    if (!this.errorTracker.has(watchDir)) {
+      this.errorTracker.set(watchDir, {
+        count: 0,
+        steps: new Map(),
+        lastError: null,
+      });
+    }
+
+    const dirErrors = this.errorTracker.get(watchDir);
+    dirErrors.count++;
+    dirErrors.lastError = error;
+
+    if (!dirErrors.steps.has(step)) {
+      dirErrors.steps.set(step, 0);
+    }
+    dirErrors.steps.set(step, dirErrors.steps.get(step) + 1);
+  }
+
+  /**
+   * Get error summary by directory
+   * @returns {Object} Errors grouped by directory
+   */
+  getErrorSummary() {
+    const summary = {};
+
+    for (const [dir, errors] of this.errorTracker.entries()) {
+      const stepDetails = {};
+      for (const [step, count] of errors.steps.entries()) {
+        stepDetails[step] = count;
+      }
+
+      summary[dir] = {
+        totalErrors: errors.count,
+        steps: stepDetails,
+        lastError: errors.lastError,
+      };
+    }
+
+    return summary;
+  }
+
+  /**
+   * Print error summary
+   * @returns {void}
+   */
+  printErrorSummary() {
+    const summary = this.getErrorSummary();
+
+    if (Object.keys(summary).length === 0) {
+      logger.info('✅ No errors detected');
+      return;
+    }
+
+    logger.info('\n═══════════════════════════════════════════════════════');
+    logger.info('📋 ERROR SUMMARY BY DIRECTORY');
+    logger.info('═══════════════════════════════════════════════════════');
+
+    for (const [dir, errors] of Object.entries(summary)) {
+      logger.info(`\n📁 ${dir}`);
+      logger.info(`   ❌ Total errors: ${errors.totalErrors}`);
+
+      for (const [step, count] of Object.entries(errors.steps)) {
+        logger.info(`      • ${step}: ${count} failure(s)`);
+      }
+
+      logger.info(`   Last error: ${errors.lastError}`);
+    }
+
+    logger.info('\n═══════════════════════════════════════════════════════\n');
+  }
+
+  /**
+   * Set automatic timeout to reset processing flag (fail-safe)
+   * @private
+   * @returns {void}
+   */
+  #setProcessingTimeout() {
+    this.#clearProcessingTimeout();
+    this.processingTimeout = setTimeout(() => {
+      if (this.isProcessing) {
+        logger.warn(
+          '⚠️ Processing pipeline timeout - forcing reset after ' +
+            `${this.processingTimeoutDuration}ms`,
+        );
+        this.isProcessing = false;
+        this.lastProcessedFile = null;
+      }
+    }, this.processingTimeoutDuration);
+  }
+
+  /**
+   * Clear processing timeout
+   * @private
+   * @returns {void}
+   */
+  #clearProcessingTimeout() {
+    if (this.processingTimeout) {
+      clearTimeout(this.processingTimeout);
+      this.processingTimeout = null;
+    }
+  }
+}
+
+// Export singleton instance
+const autoProcessingService = new AutoProcessingService();
+export default autoProcessingService;