@arela/uploader 1.1.3 → 1.2.0

package/.claude/worktrees/agent-multi-profile/src/commands/UploadCommand.js

@@ -0,0 +1,587 @@
+import cliProgress from 'cli-progress';
+import { globby } from 'globby';
+import mime from 'mime-types';
+import path from 'path';
+
+import databaseService from '../services/DatabaseService.js';
+import logger from '../services/LoggingService.js';
+import watchService from '../services/WatchService.js';
+import uploadServiceFactory from '../services/upload/UploadServiceFactory.js';
+
+import appConfig from '../config/config.js';
+import ErrorHandler from '../errors/ErrorHandler.js';
+import {
+  ConfigurationError,
+  FileOperationError,
+} from '../errors/ErrorTypes.js';
+import FileOperations from '../utils/FileOperations.js';
+import fileSanitizer from '../utils/FileSanitizer.js';
+import pathDetector from '../utils/PathDetector.js';
+
+/**
+ * Upload Command Handler
+ * Handles the main upload functionality
+ */
+export class UploadCommand {
+  constructor() {
+    this.errorHandler = new ErrorHandler(logger);
+  }
+
+  /**
+   * Execute the upload command
+   * @param {Object} options - Command options
+   */
+  async execute(options) {
+    try {
+      // Prevent direct uploads while in watch mode
+      if (watchService.isWatchActive()) {
+        logger.error('❌ Cannot upload directly while in watch mode');
+        logger.info('💡 Files in watch mode are processed automatically');
+        logger.info(
+          '💡 Stop watch mode first (Ctrl+C) before using upload command',
+        );
+        return {
+          success: false,
+          reason: 'Watch mode is active - cannot upload directly',
+          source: null,
+          stats: {
+            successCount: 0,
+            detectedCount: 0,
+            organizedCount: 0,
+            failureCount: 0,
+            skippedCount: 0,
+          },
+        };
+      }
+
+      // Validate configuration
+      this.#validateOptions(options);
+
+      // Initialize services
+      const uploadService = await uploadServiceFactory.getUploadService(
+        options.forceSupabase,
+      );
+      const sources = appConfig.getUploadSources();
+      const basePath = appConfig.getBasePath();
+
+      // Log command start
+      logger.info(`Starting upload with ${uploadService.getServiceName()}`);
+
+      if (options.clearLog) {
+        logger.clearLogFile();
+        logger.info('Log file cleared');
+      }
+
+      // Process each source with configurable concurrency
+      let globalResults = {
+        successCount: 0,
+        detectedCount: 0,
+        organizedCount: 0,
+        failureCount: 0,
+        skippedCount: 0,
+      };
+
+      // Determine processing strategy based on configuration
+      const maxConcurrentSources =
+        appConfig.performance?.maxConcurrentSources || 1;
+
+      if (maxConcurrentSources > 1 && sources.length > 1) {
+        // Parallel source processing
+        logger.info(
+          `Processing ${sources.length} sources with concurrency: ${maxConcurrentSources}`,
+        );
+
+        // Process sources in batches to control concurrency
+        for (let i = 0; i < sources.length; i += maxConcurrentSources) {
+          const sourceBatch = sources.slice(i, i + maxConcurrentSources);
+
+          const sourcePromises = sourceBatch.map(async (source) => {
+            const sourcePath = path
+              .resolve(basePath, source)
+              .replace(/\\/g, '/');
+            logger.info(`Processing folder: ${sourcePath}`);
+
+            try {
+              const files = await this.#discoverFiles(sourcePath);
+              logger.info(`Found ${files.length} files in ${source}`);
+
+              const result = await this.#processFilesInBatches(
+                files,
+                options,
+                uploadService,
+                basePath,
+                source,
+                sourcePath,
+              );
+
+              this.#logSourceSummary(source, result, options);
+              return { success: true, source, result };
+            } catch (error) {
+              this.errorHandler.handleError(error, { source, sourcePath });
+              return { success: false, source, error: error.message };
+            }
+          });
+
+          // Wait for this batch of sources to complete
+          const results = await Promise.allSettled(sourcePromises);
+
+          results.forEach((result) => {
+            if (result.status === 'fulfilled') {
+              const sourceResult = result.value;
+              if (sourceResult.success) {
+                this.#updateGlobalResults(globalResults, sourceResult.result);
+              } else {
+                globalResults.failureCount++;
+              }
+            } else {
+              globalResults.failureCount++;
+            }
+          });
+        }
+      } else {
+        // Sequential source processing (original behavior)
+        for (const source of sources) {
+          const sourcePath = path.resolve(basePath, source).replace(/\\/g, '/');
+          logger.info(`Processing folder: ${sourcePath}`);
+
+          try {
+            const files = await this.#discoverFiles(sourcePath);
+            logger.info(`Found ${files.length} files to process`);
+
+            const result = await this.#processFilesInBatches(
+              files,
+              options,
+              uploadService,
+              basePath,
+              source,
+              sourcePath,
+            );
+
+            this.#updateGlobalResults(globalResults, result);
+            this.#logSourceSummary(source, result, options);
+          } catch (error) {
+            this.errorHandler.handleError(error, { source, sourcePath });
+            globalResults.failureCount++;
+          }
+        }
+      }
+
+      this.#logFinalSummary(globalResults, options, uploadService);
+
+      // Handle additional phases if requested
+      if (options.runAllPhases && options.statsOnly) {
+        await this.#runAdditionalPhases(options);
+      }
+    } catch (error) {
+      this.errorHandler.handleFatalError(error, { command: 'upload', options });
+    }
+  }
+
+  /**
+   * Validate command options
+   * @private
+   * @param {Object} options - Options to validate
+   */
+  #validateOptions(options) {
+    try {
+      appConfig.validateConfiguration(options.forceSupabase);
+    } catch (error) {
+      throw new ConfigurationError(error.message);
+    }
+
+    if (
+      options.batchSize &&
+      (options.batchSize < 1 || options.batchSize > 100)
+    ) {
+      throw new ConfigurationError('Batch size must be between 1 and 100');
+    }
+  }
+
+  /**
+   * Discover files in a source path
+   * @private
+   * @param {string} sourcePath - Path to discover files in
+   * @returns {Promise<string[]>} Array of file paths
+   */
+  async #discoverFiles(sourcePath) {
+    try {
+      if (!FileOperations.fileExists(sourcePath)) {
+        throw new FileOperationError(
+          `Source path does not exist: ${sourcePath}`,
+        );
+      }
+
+      const stats = FileOperations.getFileStats(sourcePath);
+
+      if (stats?.isDirectory()) {
+        return await globby([`${sourcePath}/**/*`], { onlyFiles: true });
+      } else {
+        return [sourcePath];
+      }
+    } catch (error) {
+      throw new FileOperationError(
+        `Failed to discover files in ${sourcePath}`,
+        sourcePath,
+        { originalError: error.message },
+      );
+    }
+  }
+
+  /**
+   * Process files in batches
+   * @private
+   * @param {string[]} files - Files to process
+   * @param {Object} options - Processing options
+   * @param {Object} uploadService - Upload service instance
+   * @param {string} basePath - Base path
+   * @param {string} source - Source name
+   * @param {string} sourcePath - Source path
+   * @returns {Promise<Object>} Processing results
+   */
+  async #processFilesInBatches(
+    files,
+    options,
+    uploadService,
+    basePath,
+    source,
+    sourcePath,
+  ) {
+    const batchSize =
+      parseInt(options.batchSize) || appConfig.performance.batchSize || 50;
+    const results = {
+      successCount: 0,
+      detectedCount: 0,
+      organizedCount: 0,
+      failureCount: 0,
+      skippedCount: 0,
+    };
+
+    // Get processed paths if available
+    const processedPaths = options.skipProcessed
+      ? databaseService.getProcessedPaths()
+      : new Set();
+
+    // Create progress bar
+    const progressBar = new cliProgress.SingleBar({
+      format: `📤 ${source} |{bar}| {percentage}% | {value}/{total} | Success: {success} | Errors: {errors}`,
+      barCompleteChar: '█',
+      barIncompleteChar: '░',
+      hideCursor: true,
+      clearOnComplete: false,
+      stopOnComplete: true,
+      stream: process.stderr, // Use stderr to separate from stdout logging
+    });
+
+    progressBar.start(files.length, 0, { success: 0, errors: 0 });
+
+    // Process files in batches
+    for (let i = 0; i < files.length; i += batchSize) {
+      const batch = files.slice(i, i + batchSize);
+
+      try {
+        const batchResult = await this.#processBatch(
+          batch,
+          options,
+          uploadService,
+          basePath,
+          processedPaths,
+        );
+
+        this.#updateResults(results, batchResult);
+
+        progressBar.update(Math.min(i + batchSize, files.length), {
+          success: results.successCount,
+          errors: results.failureCount,
+        });
+
+        // Delay between batches if configured
+        if (appConfig.performance.batchDelay > 0) {
+          await new Promise((resolve) =>
+            setTimeout(resolve, appConfig.performance.batchDelay),
+          );
+        }
+      } catch (error) {
+        this.errorHandler.handleError(error, {
+          batch: i / batchSize + 1,
+          batchSize,
+        });
+        results.failureCount += batch.length;
+      }
+    }
+
+    progressBar.stop();
+    return results;
+  }
+
+  /**
+   * Process a batch of files
+   * @private
+   * @param {string[]} batch - Files in this batch
+   * @param {Object} options - Processing options
+   * @param {Object} uploadService - Upload service
+   * @param {string} basePath - Base path
+   * @param {Set} processedPaths - Already processed paths
+   * @returns {Promise<Object>} Batch results
+   */
+  async #processBatch(batch, options, uploadService, basePath, processedPaths) {
+    const batchResults = {
+      successCount: 0,
+      detectedCount: 0,
+      organizedCount: 0,
+      failureCount: 0,
+      skippedCount: 0,
+    };
+
+    if (options.statsOnly) {
+      // Stats-only mode: just record file information
+      const fileObjects = batch.map((filePath) => ({
+        path: filePath,
+        originalName: path.basename(filePath),
+        stats: FileOperations.getFileStats(filePath),
+      }));
+
+      try {
+        const result = await databaseService.insertStatsOnlyToUploaderTable(
+          fileObjects,
+          {
+            ...options,
+            quietMode: options.quietMode || false, // Pass through quiet mode flag
+          },
+        );
+        batchResults.successCount = result.totalInserted;
+        batchResults.skippedCount = result.totalSkipped;
+      } catch (error) {
+        throw new Error(`Failed to insert stats: ${error.message}`);
+      }
+    } else {
+      // Upload mode: process files with controlled concurrency to match API replicas
+      const maxConcurrentApiCalls =
+        appConfig.performance?.maxApiConnections || 10;
+
+      // Process batch in chunks to respect API replica limits
+      const allResults = [];
+      for (let i = 0; i < batch.length; i += maxConcurrentApiCalls) {
+        const chunk = batch.slice(i, i + maxConcurrentApiCalls);
+
+        // Process this chunk concurrently (up to API replica count)
+        const chunkPromises = chunk.map(async (filePath) => {
+          try {
+            const result = await this.#processFile(
+              filePath,
+              options,
+              uploadService,
+              basePath,
+              processedPaths,
+            );
+            return { success: true, filePath, result };
+          } catch (error) {
+            this.errorHandler.handleError(error, { filePath });
+            return { success: false, filePath, error: error.message };
+          }
+        });
+
+        // Wait for this chunk to complete before starting the next
+        const chunkResults = await Promise.allSettled(chunkPromises);
+        allResults.push(...chunkResults);
+
+        // Small delay between chunks to prevent overwhelming API
+        if (i + maxConcurrentApiCalls < batch.length) {
+          await new Promise((resolve) => setTimeout(resolve, 50));
+        }
+      }
+
+      // Process all results and update batch results
+      allResults.forEach((result) => {
+        if (result.status === 'fulfilled') {
+          const fileResult = result.value;
+          if (fileResult.success) {
+            if (fileResult.result && fileResult.result.skipped) {
+              batchResults.skippedCount++;
+            } else {
+              batchResults.successCount++;
+              if (fileResult.result && fileResult.result.detectedCount) {
+                batchResults.detectedCount += fileResult.result.detectedCount;
+              }
+              if (fileResult.result && fileResult.result.organizedCount) {
+                batchResults.organizedCount += fileResult.result.organizedCount;
+              }
+            }
+          } else {
+            batchResults.failureCount++;
+          }
+        } else {
+          batchResults.failureCount++;
+        }
+      });
+    }
+
+    return batchResults;
+  }
+
+  /**
+   * Process a single file
+   * @private
+   */
+  async #processFile(
+    filePath,
+    options,
+    uploadService,
+    basePath,
+    processedPaths,
+  ) {
+    // Skip if already processed
+    if (processedPaths.has(filePath)) {
+      return { skipped: true };
+    }
+
+    // Prepare file for upload
+    const sanitizedName = fileSanitizer.sanitizeFileName(
+      path.basename(filePath),
+    );
+    const pathInfo = pathDetector.extractYearAndPedimentoFromPath(
+      filePath,
+      basePath,
+    );
+
+    let uploadPath = sanitizedName;
+    if (pathInfo.detected && options.autoDetectStructure) {
+      uploadPath = `${pathInfo.year}/${pathInfo.pedimento}/${sanitizedName}`;
+    }
+
+    const fileObject = {
+      path: filePath,
+      name: sanitizedName,
+      contentType: this.#getMimeType(filePath),
+    };
+
+    // Upload based on service type
+    let result = { successCount: 1 };
+
+    if (uploadService.getServiceName() === 'Arela API') {
+      result = await uploadService.upload([fileObject], {
+        ...options,
+        uploadPath,
+      });
+    } else {
+      // Supabase direct upload
+      const uploadResult = await uploadService.upload([fileObject], {
+        uploadPath,
+      });
+
+      // Check if upload was successful
+      if (!uploadResult.success) {
+        throw new Error(`Supabase upload failed: ${uploadResult.error}`);
+      }
+
+      result = { successCount: 1 };
+    }
+
+    logger.info(`SUCCESS: ${path.basename(filePath)} -> ${uploadPath}`);
+
+    return {
+      skipped: false,
+      detectedCount: result.detectedCount || 0,
+      organizedCount: result.organizedCount || 0,
+    };
+  }
+
+  /**
+   * Get MIME type for file
+   * @private
+   */
+  #getMimeType(filePath) {
+    return mime.lookup(filePath) || 'application/octet-stream';
+  }
+
+  /**
+   * Update results object
+   * @private
+   */
+  #updateResults(target, source) {
+    target.successCount += source.successCount;
+    target.detectedCount += source.detectedCount;
+    target.organizedCount += source.organizedCount;
+    target.failureCount += source.failureCount;
+    target.skippedCount += source.skippedCount;
+  }
+
+  /**
+   * Update global results
+   * @private
+   */
+  #updateGlobalResults(global, source) {
+    this.#updateResults(global, source);
+  }
+
+  /**
+   * Log source summary
+   * @private
+   */
+  #logSourceSummary(source, result, options) {
+    console.log(`\n📦 Summary for ${source}:`);
+    if (options.statsOnly) {
+      console.log(`   📊 Stats recorded: ${result.successCount}`);
+      console.log(`   ⏭️  Duplicates: ${result.skippedCount}`);
+    } else {
+      console.log(`   ✅ Uploaded: ${result.successCount}`);
+      if (result.detectedCount)
+        console.log(`   🔍 Detected: ${result.detectedCount}`);
+      if (result.organizedCount)
+        console.log(`   📁 Organized: ${result.organizedCount}`);
+      console.log(`   ⏭️  Skipped: ${result.skippedCount}`);
+    }
+    console.log(`   ❌ Errors: ${result.failureCount}`);
+  }
+
+  /**
+   * Log final summary
+   * @private
+   */
+  #logFinalSummary(results, options, uploadService) {
+    console.log(`\n${'='.repeat(60)}`);
+    if (options.statsOnly) {
+      console.log(`📊 STATS COLLECTION COMPLETED`);
+      console.log(`   📊 Total stats recorded: ${results.successCount}`);
+      console.log(`   ⏭️  Total duplicates: ${results.skippedCount}`);
+    } else {
+      console.log(
+        `🎯 ${uploadService.getServiceName().toUpperCase()} UPLOAD COMPLETED`,
+      );
+      console.log(`   ✅ Total uploaded: ${results.successCount}`);
+      if (results.detectedCount)
+        console.log(`   🔍 Total detected: ${results.detectedCount}`);
+      if (results.organizedCount)
+        console.log(`   📁 Total organized: ${results.organizedCount}`);
+      console.log(`   ⏭️  Total skipped: ${results.skippedCount}`);
+    }
+    console.log(`   ❌ Total errors: ${results.failureCount}`);
+    console.log(`   📜 Log file: ${logger.getLogFilePath()}`);
+    console.log(`${'='.repeat(60)}\n`);
+  }
+
+  /**
+   * Run additional phases
+   * @private
+   */
+  async #runAdditionalPhases(options) {
+    try {
+      // Phase 2: PDF Detection
+      console.log('\n🔍 === PHASE 2: PDF Detection ===');
+      const detectionResult = await databaseService.detectPedimentosInDatabase({
+        batchSize:
+          parseInt(options.batchSize) || appConfig.performance.batchSize || 50,
+      });
+      console.log(
+        `✅ Phase 2 Complete: ${detectionResult.detectedCount} detected, ${detectionResult.errorCount} errors`,
+      );
+
+      // Additional phases would be implemented here
+      console.log('\n🎉 All phases completed successfully!');
+    } catch (error) {
+      this.errorHandler.handleError(error, { phase: 'additional-phases' });
+      throw error;
+    }
+  }
+}
+
+export default UploadCommand;