@arela/uploader 1.1.4 → 1.2.0

package/.claude/worktrees/agent-multi-profile/src/commands/PropagateCommand.js

@@ -0,0 +1,636 @@
+import cliProgress from 'cli-progress';
+
+import logger from '../services/LoggingService.js';
+import ScanApiService from '../services/ScanApiService.js';
+
+import appConfig from '../config/config.js';
+
+/**
+ * Propagate Command
+ * Propagates arela_path from detected pedimentos to related files in the same directory
+ * Optimized for large datasets with batch processing and progress tracking
+ */
+export class PropagateCommand {
+  constructor(options = {}) {
+    this.options = {
+      batchSize: parseInt(options.batchSize) || 50, // Process 50 pedimentos at a time
+      showStats: options.showStats || false,
+      api: options.api || 'default',
+      onProgress: options.onProgress || null, // Progress callback for worker mode
+    };
+
+    this.scanApiService = null;
+    this.tableName = null;
+    this.stats = {
+      startTime: Date.now(),
+      pedimentosProcessed: 0,
+      filesUpdated: 0,
+      filesFailed: 0,
+      directoriesProcessed: 0,
+    };
+  }
+
+  /**
+   * Main execution method
+   */
+  async execute() {
+    try {
+      console.log('🔄 Starting arela propagate command\n');
+
+      // Step 1: Validate configuration
+      await this.#validateConfiguration();
+
+      // Step 2: Initialize API service with configured target
+      const apiTarget = this.options.api || 'default';
+      this.scanApiService = new ScanApiService(apiTarget);
+      console.log(`🎯 API Target: ${apiTarget}`);
+
+      // Step 3: Fetch all tables for this instance
+      const scanConfig = appConfig.getScanConfig();
+      const tables = await this.scanApiService.getInstanceTables(
+        scanConfig.companySlug,
+        scanConfig.serverId,
+        scanConfig.basePathFull,
+      );
+
+      if (tables.length === 0) {
+        console.error(
+          '\n❌ No tables found for this instance. Run "arela scan" first.\n',
+        );
+        process.exit(1);
+      }
+
+      console.log(
+        `📋 Found ${tables.length} table${tables.length === 1 ? '' : 's'} to process:`,
+      );
+      for (const table of tables) {
+        console.log(`   - ${table.tableName}`);
+      }
+      console.log();
+
+      // Step 4: Process each table
+      let totalStats = {
+        pedimentosProcessed: 0,
+        filesUpdated: 0,
+        filesFailed: 0,
+        directoriesProcessed: 0,
+      };
+
+      // Report initial progress
+      this.#reportProgress(
+        0,
+        `Starting propagation on ${tables.length} tables`,
+      );
+
+      for (let i = 0; i < tables.length; i++) {
+        const table = tables[i];
+        console.log(`\n🔄 Processing table: ${table.tableName}\n`);
+        this.#reportProgress(
+          Math.round((i / tables.length) * 100),
+          `Processing table ${i + 1}/${tables.length}: ${table.tableName}`,
+        );
+        this.tableName = table.tableName;
+
+        // Process this table
+        const stats = await this.#processTable();
+
+        totalStats.pedimentosProcessed += stats.pedimentosProcessed;
+        totalStats.filesUpdated += stats.filesUpdated;
+        totalStats.filesFailed += stats.filesFailed;
+        totalStats.directoriesProcessed += stats.directoriesProcessed;
+      }
+
+      // Step 5: Cross-table propagation
+      // Match files with detected_pedimento in one table to pedimento sources in other tables
+      const crossTableStats = await this.#processCrossTablePropagation(
+        scanConfig,
+        tables,
+      );
+      totalStats.filesUpdated += crossTableStats.filesUpdated;
+      totalStats.filesFailed += crossTableStats.filesFailed;
+
+      // Show combined results
+      const duration = ((Date.now() - this.stats.startTime) / 1000).toFixed(2);
+      const filesPerSec =
+        totalStats.filesUpdated > 0
+          ? (totalStats.filesUpdated / parseFloat(duration)).toFixed(1)
+          : 0;
+
+      // Report completion
+      this.#reportProgress(
+        100,
+        `Propagation completed: ${totalStats.filesUpdated} files updated`,
+      );
+
+      console.log('\\n✅ Propagation Complete!\\n');
+      console.log(`📊 Total Results:`);
+      console.log(`   Tables Processed: ${tables.length}`);
+      console.log(
+        `   Pedimentos Processed: ${totalStats.pedimentosProcessed.toLocaleString()}`,
+      );
+      console.log(
+        `   Files Updated: ${totalStats.filesUpdated.toLocaleString()}`,
+      );
+      console.log(
+        `   Files Failed: ${totalStats.filesFailed.toLocaleString()}`,
+      );
+      console.log(
+        `   Directories Processed: ${totalStats.directoriesProcessed.toLocaleString()}`,
+      );
+      console.log(`   Duration: ${duration}s`);
+      console.log(`   Speed: ${filesPerSec} files/sec\\n`);
+    } catch (error) {
+      logger.error('Propagation command failed:', error);
+      console.error(`\\n❌ Error: ${error.message}\\n`);
+      if (process.env.VERBOSE || process.env.DEBUG) {
+        console.error('Stack trace:', error.stack);
+      }
+      process.exit(1);
+    }
+  }
+
+  /**
+   * Process a single table
+   * @private
+   * @returns {Promise<Object>} Statistics for this table
+   */
+  async #processTable() {
+    const tableStats = {
+      pedimentosProcessed: 0,
+      filesUpdated: 0,
+      filesFailed: 0,
+      directoriesProcessed: 0,
+    };
+
+    // Show initial statistics
+    const initialStats = await this.#showInitialStats();
+
+    // Mark files needing propagation (if we have pedimento sources)
+    if (initialStats.pedimentoSources > 0) {
+      await this.#markFilesForPropagation();
+    } else {
+      console.log('   ℹ️  No pedimento sources found. Skipping.\\n');
+      return tableStats;
+    }
+
+    // Check if there are files to propagate
+    const statsAfterMarking = await this.scanApiService.getPropagationStats(
+      this.tableName,
+    );
+    if (statsAfterMarking.pending === 0) {
+      console.log(
+        '   ℹ️  All files already have arela_path. Nothing to propagate.\\n',
+      );
+      return tableStats;
+    }
+
+    console.log(
+      `   🚀 Found ${statsAfterMarking.pending.toLocaleString()} files ready for propagation.\\n`,
+    );
+
+    // Process pedimentos and propagate arela_path
+    const stats = await this.#processPropagation();
+    Object.assign(tableStats, stats);
+
+    // Show final statistics for this table
+    await this.#showFinalStats();
+
+    return tableStats;
+  }
+
+  /**
+   * Validate scan configuration
+   * @private
+   */
+  async #validateConfiguration() {
+    logger.debug('Validating scan configuration...');
+
+    // Set API target
+    appConfig.setApiTarget(this.options.api);
+
+    // Validate scan config (same as scan/identify commands)
+    // Note: validateScanConfig() throws on error, doesn't return errors array
+    try {
+      appConfig.validateScanConfig();
+    } catch (error) {
+      console.error(`\\n❌ ${error.message}\\n`);
+      throw new Error('Invalid scan configuration');
+    }
+
+    console.log(`🎯 API Target: ${this.options.api}`);
+    console.log(`📦 Batch Size: ${this.options.batchSize}\\n`);
+
+    logger.debug('Configuration validated');
+  }
+
+  /**
+   * Show initial propagation statistics
+   * @private
+   */
+  async #showInitialStats() {
+    try {
+      const stats = await this.scanApiService.getPropagationStats(
+        this.tableName,
+      );
+
+      console.log('📈 Initial Status:');
+      console.log(`   Total Files: ${stats.totalFiles.toLocaleString()}`);
+      console.log(
+        `   With arela_path: ${stats.withArelaPath.toLocaleString()}`,
+      );
+      console.log(
+        `   Pedimento Sources: ${stats.pedimentoSources.toLocaleString()}`,
+      );
+      console.log(`   Errors: ${stats.errors.toLocaleString()}\n`);
+
+      return stats;
+    } catch (error) {
+      logger.error('Failed to fetch initial stats:', error);
+      throw new Error(`Failed to fetch propagation stats: ${error.message}`);
+    }
+  }
+
+  /**
+   * Mark files that need propagation
+   * This is a preparation step that flags files for efficient processing
+   * @private
+   */
+  async #markFilesForPropagation() {
+    try {
+      console.log('🏷️  Marking files needing propagation...');
+      const result = await this.scanApiService.markFilesNeedingPropagation(
+        this.tableName,
+      );
+      console.log(`✓ Marked ${result.markedCount.toLocaleString()} files\n`);
+    } catch (error) {
+      logger.error('Failed to mark files:', error);
+      throw new Error(`Failed to mark files: ${error.message}`);
+    }
+  }
+
+  /**
+   * Process propagation in batches
+   * Fetches pedimentos and propagates their arela_path to files in the same directory
+   * @private
+   */
+  async #processPropagation() {
+    console.log('🚀 Processing propagation...\n');
+
+    // First, get the total count of pedimento sources
+    const initialStats = await this.scanApiService.getPropagationStats(
+      this.tableName,
+    );
+    const totalPedimentos = initialStats.pedimentoSources;
+
+    if (totalPedimentos === 0) {
+      console.log('ℹ️  No pedimento sources found.\n');
+      return;
+    }
+
+    let offset = 0;
+    let hasMore = true;
+    let processedCount = 0;
+
+    // Create progress bar with actual total
+    const progressBar = new cliProgress.SingleBar(
+      {
+        format:
+          '📄 Propagating |{bar}| {percentage}% | {value}/{total} directories | {speed} files/sec | {filesUpdated} files updated',
+        barCompleteChar: '\u2588',
+        barIncompleteChar: '\u2591',
+        hideCursor: true,
+        clearOnComplete: false,
+        stopOnComplete: true,
+      },
+      cliProgress.Presets.shades_classic,
+    );
+
+    const startTime = Date.now();
+    let filesUpdated = 0;
+
+    // Start progress bar with actual total
+    progressBar.start(totalPedimentos, 0, {
+      speed: '0',
+      filesUpdated: 0,
+    });
+
+    try {
+      while (hasMore) {
+        // Fetch batch of pedimento sources
+        const pedimentos = await this.scanApiService.fetchPedimentoSources(
+          this.tableName,
+          offset,
+          this.options.batchSize,
+        );
+
+        // Validate response
+        if (!pedimentos || !Array.isArray(pedimentos)) {
+          logger.error(
+            'Invalid response from fetchPedimentoSources:',
+            pedimentos,
+          );
+          throw new Error('API returned invalid data format (expected array)');
+        }
+
+        if (pedimentos.length === 0) {
+          hasMore = false;
+          break;
+        }
+
+        // Process each pedimento's directory
+        for (const pedimento of pedimentos) {
+          const {
+            id,
+            directory_path,
+            arela_path,
+            rfc,
+            detected_pedimento,
+            detected_pedimento_year,
+          } = pedimento;
+
+          // Fetch files in the same directory
+          const files =
+            await this.scanApiService.fetchFilesNeedingPropagationByDirectory(
+              this.tableName,
+              directory_path,
+            );
+
+          // Validate response
+          if (!files || !Array.isArray(files)) {
+            logger.error(
+              `Invalid response for directory ${directory_path}:`,
+              files,
+            );
+            this.stats.filesFailed++;
+            continue;
+          }
+
+          if (files.length > 0) {
+            // Prepare batch update
+            const updates = files.map((file) => ({
+              id: file.id,
+              arelaPath: arela_path,
+              rfc: rfc,
+              detectedPedimento: detected_pedimento,
+              detectedPedimentoYear: detected_pedimento_year,
+              propagatedFromId: id,
+              propagationError: null,
+            }));
+
+            // Send batch update to API
+            try {
+              const result = await this.scanApiService.batchUpdatePropagation(
+                this.tableName,
+                updates,
+              );
+
+              filesUpdated += result.updated;
+              this.stats.filesUpdated += result.updated;
+              this.stats.filesFailed += result.errors;
+            } catch (error) {
+              logger.error(
+                `Failed to update files in directory ${directory_path}:`,
+                error,
+              );
+              this.stats.filesFailed += files.length;
+            }
+          }
+
+          this.stats.directoriesProcessed++;
+          this.stats.pedimentosProcessed++;
+          processedCount++;
+
+          // Update progress bar
+          const elapsed = (Date.now() - startTime) / 1000;
+          const speed = elapsed > 0 ? Math.round(filesUpdated / elapsed) : 0;
+
+          progressBar.update(processedCount, {
+            speed: speed.toString(),
+            filesUpdated: filesUpdated.toLocaleString(),
+          });
+        }
+
+        // Move to next batch
+        offset += pedimentos.length;
+
+        // Check if we got fewer results than requested (indicates last batch)
+        if (pedimentos.length < this.options.batchSize) {
+          hasMore = false;
+        }
+      }
+    } finally {
+      progressBar.stop();
+    }
+
+    const duration = ((Date.now() - startTime) / 1000).toFixed(2);
+    const speed =
+      duration > 0 ? Math.round(filesUpdated / parseFloat(duration)) : 0;
+
+    console.log('\n   📊 Results:');
+    console.log(
+      `      Pedimentos Processed: ${this.stats.pedimentosProcessed.toLocaleString()}`,
+    );
+    console.log(
+      `      Directories Processed: ${this.stats.directoriesProcessed.toLocaleString()}`,
+    );
+    console.log(
+      `      Files Updated: ${this.stats.filesUpdated.toLocaleString()}`,
+    );
+    console.log(`      Errors: ${this.stats.filesFailed.toLocaleString()}`);
+    console.log(`      Duration: ${duration}s`);
+    console.log(`      Speed: ${speed} files/sec\n`);
+
+    return {
+      pedimentosProcessed: this.stats.pedimentosProcessed,
+      filesUpdated: this.stats.filesUpdated,
+      filesFailed: this.stats.filesFailed,
+      directoriesProcessed: this.stats.directoriesProcessed,
+    };
+  }
+
+  /**
+   * Cross-table propagation phase
+   * Matches files with detected_pedimento in one table to pedimento sources in other tables.
+   * This enables facturas (in a different directory/table) to get arela_path from their pedimento.
+   * @private
+   * @param {Object} scanConfig - Scan configuration with companySlug, serverId, basePathFull
+   * @param {Array} tables - All tables for this instance
+   * @returns {Promise<Object>} { filesUpdated, filesFailed }
+   */
+  async #processCrossTablePropagation(scanConfig, tables) {
+    console.log('\n🔗 Cross-table propagation phase...\n');
+
+    const stats = { filesUpdated: 0, filesFailed: 0 };
+
+    // Step 1: Fetch all pedimento sources across all tables
+    const pedimentoSources =
+      await this.scanApiService.fetchCrossTablePedimentoSources(
+        scanConfig.companySlug,
+        scanConfig.serverId,
+        scanConfig.basePathFull,
+      );
+
+    if (pedimentoSources.length === 0) {
+      console.log(
+        '   ℹ️  No pedimento sources found across tables. Skipping cross-table phase.\n',
+      );
+      return stats;
+    }
+
+    // Build a map: detected_pedimento → source info
+    const sourceMap = new Map();
+    for (const source of pedimentoSources) {
+      sourceMap.set(source.detected_pedimento, source);
+    }
+
+    console.log(
+      `   📋 Found ${sourceMap.size} unique pedimento sources across ${tables.length} tables`,
+    );
+
+    // Step 2: For each table, find orphan files (have pedimento, no arela_path)
+    let totalOrphans = 0;
+
+    for (const table of tables) {
+      let offset = 0;
+      let hasMore = true;
+
+      while (hasMore) {
+        const orphanFiles =
+          await this.scanApiService.fetchFilesWithPedimentoNoArelaPath(
+            table.tableName,
+            offset,
+            this.options.batchSize,
+          );
+
+        if (orphanFiles.length === 0) {
+          hasMore = false;
+          break;
+        }
+
+        // Step 3: Match orphans against pedimento source map
+        const updates = [];
+        for (const file of orphanFiles) {
+          const source = sourceMap.get(file.detected_pedimento);
+          if (source) {
+            updates.push({
+              id: file.id,
+              arelaPath: source.arela_path,
+              rfc: source.rfc,
+              detectedPedimento: file.detected_pedimento,
+              detectedPedimentoYear: source.detected_pedimento_year,
+              propagatedFromId: source.source_id,
+              propagatedFromTable: source.source_table,
+              propagationError: null,
+            });
+          }
+        }
+
+        totalOrphans += orphanFiles.length;
+
+        // Step 4: Batch update matched files
+        if (updates.length > 0) {
+          try {
+            const result = await this.scanApiService.batchUpdatePropagation(
+              table.tableName,
+              updates,
+            );
+            stats.filesUpdated += result.updated;
+            stats.filesFailed += result.errors;
+          } catch (error) {
+            logger.error(
+              `Failed cross-table update on ${table.tableName}:`,
+              error,
+            );
+            stats.filesFailed += updates.length;
+          }
+        }
+
+        offset += orphanFiles.length;
+        if (orphanFiles.length < this.options.batchSize) {
+          hasMore = false;
+        }
+      }
+    }
+
+    console.log(`   📊 Cross-table results:`);
+    console.log(`      Orphan files checked: ${totalOrphans}`);
+    console.log(`      Files updated: ${stats.filesUpdated}`);
+    console.log(`      Files failed: ${stats.filesFailed}\n`);
+
+    return stats;
+  }
+
+  /**
+   * Show final propagation statistics
+   * @private
+   */
+  async #showFinalStats() {
+    try {
+      const stats = await this.scanApiService.getPropagationStats(
+        this.tableName,
+      );
+
+      console.log('📈 Final Status:');
+      console.log(`   Total Files: ${stats.totalFiles.toLocaleString()}`);
+      console.log(
+        `   With arela_path: ${stats.withArelaPath.toLocaleString()}`,
+      );
+      console.log(
+        `   Needs Propagation: ${stats.needsPropagation.toLocaleString()}`,
+      );
+      console.log(`   Pending: ${stats.pending.toLocaleString()}`);
+      console.log(`   Errors: ${stats.errors.toLocaleString()}`);
+
+      if (stats.maxAttemptsReached > 0) {
+        console.log(
+          `\n⚠️  ${stats.maxAttemptsReached} files reached max propagation attempts.`,
+        );
+        console.log(
+          '   Run with increased max_propagation_attempts if needed, or review propagation errors.',
+        );
+      }
+
+      if (this.options.showStats) {
+        const duration = ((Date.now() - this.stats.startTime) / 1000).toFixed(
+          2,
+        );
+        console.log('\n💻 Performance Stats:');
+        console.log(`   Total Duration: ${duration}s`);
+        console.log(`   Memory Used: ${this.#getMemoryUsage()}`);
+      }
+    } catch (error) {
+      logger.error('Failed to fetch final stats:', error);
+      // Don't throw - command was successful even if we can't fetch final stats
+    }
+  }
+
+  /**
+   * Get formatted memory usage
+   * @private
+   */
+  #getMemoryUsage() {
+    const used = process.memoryUsage();
+    return `${Math.round(used.heapUsed / 1024 / 1024)}MB`;
+  }
+
+  /**
+   * Report progress to callback if available (for worker mode)
+   * @private
+   * @param {number} percent - Progress percentage (0-100)
+   * @param {string} message - Progress message
+   */
+  #reportProgress(percent, message) {
+    if (
+      this.options.onProgress &&
+      typeof this.options.onProgress === 'function'
+    ) {
+      try {
+        this.options.onProgress(percent, message);
+      } catch (error) {
+        logger.debug(`Progress callback error: ${error.message}`);
+      }
+    }
+  }
+}
+
+export default PropagateCommand;