@arela/uploader 1.0.2 → 1.0.4

package/src/commands/PushCommand.js

@@ -0,0 +1,473 @@
+import cliProgress from 'cli-progress';
+import FormData from 'form-data';
+import fs from 'fs';
+import fetch from 'node-fetch';
+import path from 'path';
+
+import logger from '../services/LoggingService.js';
+import ScanApiService from '../services/ScanApiService.js';
+
+import appConfig from '../config/config.js';
+
+/**
+ * Push Command Handler
+ * Uploads files with arela_path to storage API
+ */
+export class PushCommand {
+  constructor() {
+    this.scanApiService = new ScanApiService();
+  }
+
+  /**
+   * Execute the push command
+   * @param {Object} options - Command options
+   */
+  async execute(options) {
+    try {
+      console.log('\n🚀 Starting arela push command\n');
+
+      // Validate scan configuration (same config as scan/identify/propagate)
+      const errors = this.#validateConfig();
+      if (errors.length > 0) {
+        console.error('⚠️ Configuration errors:');
+        errors.forEach((err) => console.error(`   - ${err}`));
+        process.exit(1);
+      }
+
+      // Get configuration
+      const scanConfig = appConfig.getScanConfig();
+      const pushConfig = appConfig.getPushConfig();
+      const tableName = scanConfig.tableName;
+
+      // Set API target for scan/push operations
+      const scanApiTarget = options.api || options.scanApi || 'default';
+      const pushApiTarget = options.pushApi || scanApiTarget;
+
+      if (scanApiTarget !== 'default') {
+        appConfig.setApiTarget(scanApiTarget);
+        this.scanApiService = new ScanApiService(); // Reinitialize with new target
+      }
+
+      // Get upload API configuration
+      const uploadApiConfig = appConfig.getApiConfig(pushApiTarget);
+
+      console.log(`🎯 Scan API Target: ${scanApiTarget}`);
+      console.log(
+        `🎯 Upload API Target: ${pushApiTarget} → ${uploadApiConfig.baseUrl}`,
+      );
+      console.log(`📦 Fetch Batch Size: ${options.batchSize}`);
+      console.log(`📤 Upload Batch Size: ${options.uploadBatchSize}`);
+
+      // Apply filters
+      const filters = {
+        rfcs: options.rfcs || pushConfig.rfcs || [],
+        years: options.years || pushConfig.years || [],
+      };
+
+      if (filters.rfcs.length > 0) {
+        console.log(`🔍 RFC Filter: ${filters.rfcs.join(', ')}`);
+      }
+      if (filters.years.length > 0) {
+        console.log(`🔍 Year Filter: ${filters.years.join(', ')}`);
+      }
+
+      // Fetch all tables for this instance
+      console.log('\\n📊 Fetching instance tables...');
+      const tables = await this.scanApiService.getInstanceTables(
+        scanConfig.companySlug,
+        scanConfig.serverId,
+        scanConfig.basePathFull,
+      );
+
+      if (tables.length === 0) {
+        console.error(
+          '\\n❌ No tables found for this instance. Run \"arela scan\" first.\\n',
+        );
+        process.exit(1);
+      }
+
+      console.log(
+        `📋 Found ${tables.length} table${tables.length === 1 ? '' : 's'} to process:`,
+      );
+      for (const table of tables) {
+        console.log(`   - ${table.tableName}`);
+      }
+
+      // Process each table
+      let totalResults = {
+        processed: 0,
+        uploaded: 0,
+        errors: 0,
+        startTime: Date.now(),
+      };
+
+      for (const table of tables) {
+        console.log(`\\n🚀 Processing table: ${table.tableName}\\n`);
+
+        // Get initial statistics for this table
+        const initialStats = await this.scanApiService.getPushStats(
+          table.tableName,
+        );
+        this.#displayStats('  Table Status', initialStats);
+
+        if (initialStats.pending === 0) {
+          console.log('  ✅ No files pending upload. Skipping.\\n');
+          continue;
+        }
+
+        console.log(
+          `\\n  🚀 Uploading ${initialStats.pending} pending files...\\n`,
+        );
+
+        // Process files for this table
+        const results = await this.#processFiles(
+          table.tableName,
+          filters,
+          parseInt(options.batchSize),
+          parseInt(options.uploadBatchSize),
+          uploadApiConfig,
+        );
+
+        totalResults.processed += results.processed;
+        totalResults.uploaded += results.uploaded;
+        totalResults.errors += results.errors;
+
+        // Display results for this table
+        console.log('\\n  📊 Table Results:');
+        console.log(`     Files Processed: ${results.processed}`);
+        console.log(`     Uploaded: ${results.uploaded}`);
+        console.log(`     Errors: ${results.errors}\\n`);
+      }
+
+      // Display combined results
+      const duration = ((Date.now() - totalResults.startTime) / 1000).toFixed(
+        1,
+      );
+      const speed =
+        totalResults.processed > 0
+          ? (totalResults.processed / parseFloat(duration)).toFixed(0)
+          : 0;
+
+      console.log('\\n✅ Push Complete!\\n');
+      console.log('📊 Total Results:');
+      console.log(`   Tables Processed: ${tables.length}`);
+      console.log(`   Files Processed: ${totalResults.processed}`);
+      console.log(`   Uploaded: ${totalResults.uploaded}`);
+      console.log(`   Errors: ${totalResults.errors}`);
+      console.log(`   Duration: ${duration}s`);
+      console.log(`   Speed: ${speed} files/sec\\n`);
+    } catch (error) {
+      console.error('\n❌ Push failed:', error.message);
+      logger.error('Push command error:', error);
+      process.exit(1);
+    }
+  }
+
+  /**
+   * Validate configuration
+   * @private
+   * @returns {string[]} Array of error messages
+   */
+  #validateConfig() {
+    const errors = [];
+
+    try {
+      appConfig.validateScanConfig();
+    } catch (err) {
+      return [err.message];
+    }
+
+    const scanConfig = appConfig.getScanConfig();
+    if (!scanConfig.tableName) {
+      errors.push('Could not generate table name from configuration');
+    }
+
+    return errors;
+  }
+
+  /**
+   * Display statistics
+   * @private
+   */
+  #displayStats(title, stats) {
+    console.log(`\n📈 ${title}:`);
+    console.log(`   Total with arela_path: ${stats.totalWithArelaPath}`);
+    console.log(`   Uploaded: ${stats.uploaded}`);
+    console.log(`   Pending: ${stats.pending}`);
+    if (stats.errors > 0) {
+      console.log(`   Errors: ${stats.errors}`);
+    }
+    if (stats.maxAttemptsReached > 0) {
+      console.log(`   Max Attempts Reached: ${stats.maxAttemptsReached}`);
+    }
+
+    // Show top RFCs if available
+    if (stats.byRfc && stats.byRfc.length > 0) {
+      console.log('\n   📊 Top RFCs:');
+      stats.byRfc.slice(0, 5).forEach((rfc) => {
+        const percent = ((rfc.uploaded / rfc.total) * 100).toFixed(1);
+        console.log(
+          `      ${rfc.rfc}: ${rfc.uploaded}/${rfc.total} (${percent}%)`,
+        );
+      });
+    }
+  }
+
+  /**
+   * Process files in batches
+   * @private
+   */
+  async #processFiles(
+    tableName,
+    filters,
+    batchSize,
+    uploadBatchSize,
+    uploadApiConfig,
+  ) {
+    const results = {
+      processed: 0,
+      uploaded: 0,
+      errors: 0,
+      startTime: Date.now(),
+    };
+
+    // Get total count first for accurate progress
+    const initialStats = await this.scanApiService.getPushStats(tableName);
+    const totalToProcess = initialStats.pending;
+
+    let hasMore = true;
+
+    // Create progress bar with known total
+    const progressBar = new cliProgress.SingleBar({
+      format:
+        '📤 |{bar}| {percentage}% | {value}/{total} | {speed}/s | ✓{uploaded} ✗{errors}',
+      barCompleteChar: '█',
+      barIncompleteChar: '░',
+      hideCursor: true,
+      clearOnComplete: false,
+      stopOnComplete: true,
+    });
+
+    // Always use offset=0 because uploaded files are removed from pending query results
+    // After each batch upload, those files are no longer "pending", so the next query
+    // at offset=0 will naturally return the next batch of unprocessed files
+
+    // Start progress bar with known total
+    progressBar.start(totalToProcess, 0, {
+      speed: 0,
+      uploaded: 0,
+      errors: 0,
+    });
+
+    while (hasMore) {
+      // Fetch batch of files (always offset=0 since uploaded files are removed from pending)
+      const files = await this.scanApiService.fetchFilesForPush(tableName, {
+        ...filters,
+        offset: 0,
+        limit: batchSize,
+      });
+
+      if (files.length === 0) {
+        hasMore = false;
+        break;
+      }
+
+      // Upload files in smaller batches
+      for (let i = 0; i < files.length; i += uploadBatchSize) {
+        const uploadBatch = files.slice(i, i + uploadBatchSize);
+        const batchResults = await this.#uploadBatch(
+          uploadBatch,
+          uploadApiConfig,
+        );
+
+        // Update results in database
+        await this.scanApiService.batchUpdateUpload(tableName, batchResults);
+
+        // Update counters
+        batchResults.forEach((result) => {
+          results.processed++;
+          if (result.uploaded) {
+            results.uploaded++;
+          } else {
+            results.errors++;
+          }
+        });
+
+        // Update progress bar
+        const elapsed = (Date.now() - results.startTime) / 1000;
+        const speed =
+          elapsed > 0 ? (results.processed / elapsed).toFixed(0) : 0;
+        progressBar.update(results.processed, {
+          speed,
+          uploaded: results.uploaded,
+          errors: results.errors,
+        });
+      }
+
+      // Check if there are more files
+      // If we got fewer files than requested, we've processed all pending files
+      if (files.length < batchSize) {
+        hasMore = false;
+      }
+    }
+
+    progressBar.stop();
+
+    return results;
+  }
+
+  /**
+   * Upload a batch of files
+   * @private
+   */
+  async #uploadBatch(files, uploadApiConfig) {
+    const uploadPromises = files.map((file) =>
+      this.#uploadFile(file, uploadApiConfig),
+    );
+    return Promise.all(uploadPromises);
+  }
+
+  /**
+   * Upload a single file
+   * @private
+   */
+  async #uploadFile(file, uploadApiConfig) {
+    const result = {
+      id: file.id,
+      uploaded: false,
+      uploadError: null,
+      uploadPath: null,
+      uploadedToStorageId: null,
+    };
+
+    try {
+      // Check if file exists
+      if (!fs.existsSync(file.absolute_path)) {
+        result.uploadError =
+          'FILE_NOT_FOUND: File does not exist on filesystem';
+        return result;
+      }
+
+      // Get file stats
+      const stats = fs.statSync(file.absolute_path);
+      if (!stats.isFile()) {
+        result.uploadError = 'NOT_A_FILE: Path is not a regular file';
+        return result;
+      }
+
+      // Construct upload path using arela_path
+      // arela_path format: RFC/Year/Patente/Aduana/Pedimento/
+      const uploadPath = `${file.arela_path}${file.file_name}`;
+      result.uploadPath = uploadPath;
+
+      // Upload file using storage API
+      const response = await this.#uploadToStorageApi(
+        file.absolute_path,
+        uploadPath,
+        uploadApiConfig,
+        {
+          rfc: file.rfc,
+          year: file.detected_pedimento_year,
+          originalPath: file.relative_path,
+        },
+      );
+
+      if (response.success) {
+        result.uploaded = true;
+        result.uploadedToStorageId = response.fileId;
+        logger.info(`✓ Uploaded: ${file.file_name} → ${uploadPath}`);
+      } else {
+        result.uploadError = `UPLOAD_FAILED: ${response.error || 'Unknown error'}`;
+        logger.error(`✗ Failed: ${file.file_name} - ${result.uploadError}`);
+      }
+    } catch (error) {
+      result.uploadError = `UPLOAD_ERROR: ${error.message}`;
+      logger.error(`✗ Error uploading ${file.file_name}:`, error.message);
+    }
+
+    return result;
+  }
+
+  /**
+   * Upload file to storage API
+   * @private
+   */
+  async #uploadToStorageApi(filePath, uploadPath, apiConfig, metadata = {}) {
+    try {
+      // Create form data
+      const form = new FormData();
+      form.append('files', fs.createReadStream(filePath));
+
+      // uploadPath format: RFC/Year/Patente/Aduana/Pedimento/filename
+      // Extract folder structure (arela_path without filename)
+      const folderStructure = path.dirname(uploadPath);
+
+      // Use batch-upload-and-process endpoint (same as legacy upload)
+      form.append('bucket', appConfig.getPushConfig().bucket);
+      form.append('folderStructure', folderStructure);
+
+      // Add RFC for multi-database routing
+      if (metadata.rfc) {
+        form.append('rfc', metadata.rfc);
+      }
+
+      // Enable auto-detection
+      form.append('autoDetect', 'true');
+      form.append('autoOrganize', 'false');
+      form.append('batchSize', '1');
+      form.append('clientVersion', appConfig.packageVersion);
+
+      // Upload file
+      const response = await fetch(
+        `${apiConfig.baseUrl}/api/storage/batch-upload-and-process`,
+        {
+          method: 'POST',
+          headers: {
+            'x-api-key': apiConfig.token,
+            ...form.getHeaders(),
+          },
+          body: form,
+        },
+      );
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        return {
+          success: false,
+          error: `HTTP ${response.status}: ${errorText}`,
+        };
+      }
+
+      const result = await response.json();
+
+      // batch-upload-and-process returns { uploaded: [], detected: [], errors: [] }
+      // Check if upload was successful
+      if (result.uploaded && result.uploaded.length > 0) {
+        const uploadedFile = result.uploaded[0];
+        return {
+          success: true,
+          fileId: uploadedFile.fileId,
+          path: uploadedFile.path,
+        };
+      } else if (result.errors && result.errors.length > 0) {
+        const error = result.errors[0];
+        return {
+          success: false,
+          error: error.error || 'Upload failed',
+        };
+      } else {
+        return {
+          success: false,
+          error: 'Unknown upload error - no files uploaded',
+        };
+      }
+    } catch (error) {
+      return {
+        success: false,
+        error: error.message,
+      };
+    }
+  }
+}
+
+export default PushCommand;