@arela/uploader 1.1.4 → 1.2.0

package/.claude/worktrees/agent-multi-profile/src/commands/PushCommand.js

@@ -0,0 +1,743 @@
+import cliProgress from 'cli-progress';
+import FormData from 'form-data';
+import fs from 'fs';
+import fetch from 'node-fetch';
+import path from 'path';
+
+import logger from '../services/LoggingService.js';
+import ScanApiService from '../services/ScanApiService.js';
+
+import appConfig from '../config/config.js';
+
+/**
+ * Push Command Handler
+ * Uploads files with arela_path to storage API
+ */
+export class PushCommand {
+  constructor() {
+    // ScanApiService will be initialized in execute() with proper API target
+    this.scanApiService = null;
+    this.onProgress = null; // Progress callback for worker mode
+  }
+
+  /**
+   * Execute the push command
+   * @param {Object} options - Command options
+   * @param {Function} options.onProgress - Optional callback for progress updates (percent, message)
+   */
+  async execute(options) {
+    this.onProgress = options.onProgress || null;
+    try {
+      console.log('\n🚀 Starting arela push command\n');
+
+      // Validate scan configuration (same config as scan/identify/propagate)
+      const errors = this.#validateConfig();
+      if (errors.length > 0) {
+        console.error('⚠️ Configuration errors:');
+        errors.forEach((err) => console.error(`   - ${err}`));
+        process.exit(1);
+      }
+
+      // Get configuration
+      const scanConfig = appConfig.getScanConfig();
+      const pushConfig = appConfig.getPushConfig();
+
+      // Override folderStructure from command option if provided
+      if (options.folderStructure) {
+        // Clean the folder structure: remove leading/trailing slashes
+        pushConfig.folderStructure = options.folderStructure
+          .trim()
+          .replace(/^\/+/, '')
+          .replace(/\/+$/, '');
+      }
+
+      // Determine API targets for scan (read) and push (upload) operations
+      // Priority: explicit scan-api/push-api > source-api/target-api > api > default
+      const scanApiTarget =
+        options.scanApi || options.sourceApi || options.api || 'default';
+      const pushApiTarget =
+        options.pushApi || options.targetApi || options.api || scanApiTarget;
+
+      // Initialize ScanApiService with the scan API target
+      this.scanApiService = new ScanApiService(scanApiTarget);
+
+      // Get upload API configuration for the push target
+      const uploadApiConfig = appConfig.getApiConfig(pushApiTarget);
+
+      // Display API configuration
+      const isCrossTenant = scanApiTarget !== pushApiTarget;
+      if (isCrossTenant) {
+        console.log('🔗 Cross-tenant mode enabled:');
+        console.log(
+          `   📖 Scan API (read): ${scanApiTarget} → ${appConfig.getApiConfig(scanApiTarget).baseUrl}`,
+        );
+        console.log(
+          `   📝 Push API (upload): ${pushApiTarget} → ${uploadApiConfig.baseUrl}`,
+        );
+      } else {
+        console.log(
+          `🎯 API Target: ${scanApiTarget} → ${uploadApiConfig.baseUrl}`,
+        );
+      }
+      console.log(`📦 Fetch Batch Size: ${options.batchSize}`);
+      console.log(`📤 Upload Batch Size: ${options.uploadBatchSize}`);
+      if (pushConfig.folderStructure) {
+        console.log(
+          `📁 Folder Structure Prefix: ${pushConfig.folderStructure}`,
+        );
+      }
+
+      // Apply filters
+      const filters = {
+        rfcs: options.rfcs || pushConfig.rfcs || [],
+        years: options.years || pushConfig.years || [],
+      };
+
+      if (filters.rfcs.length > 0) {
+        console.log(`🔍 RFC Filter: ${filters.rfcs.join(', ')}`);
+      }
+      if (filters.years.length > 0) {
+        console.log(`🔍 Year Filter: ${filters.years.join(', ')}`);
+      }
+
+      // Fetch all tables for this instance
+      console.log('\\n📊 Fetching instance tables...');
+      const tables = await this.scanApiService.getInstanceTables(
+        scanConfig.companySlug,
+        scanConfig.serverId,
+        scanConfig.basePathFull,
+      );
+
+      if (tables.length === 0) {
+        console.error(
+          '\\n❌ No tables found for this instance. Run \"arela scan\" first.\\n',
+        );
+        process.exit(1);
+      }
+
+      console.log(
+        `📋 Found ${tables.length} table${tables.length === 1 ? '' : 's'} to process:`,
+      );
+      for (const table of tables) {
+        console.log(`   - ${table.tableName}`);
+      }
+
+      // Process each table
+      let totalResults = {
+        processed: 0,
+        uploaded: 0,
+        errors: 0,
+        startTime: Date.now(),
+      };
+
+      // Report initial progress
+      this.#reportProgress(0, `Starting push on ${tables.length} tables`);
+
+      for (let i = 0; i < tables.length; i++) {
+        const table = tables[i];
+        console.log(`\\n🚀 Processing table: ${table.tableName}\\n`);
+        this.#reportProgress(
+          Math.round((i / tables.length) * 100),
+          `Processing table ${i + 1}/${tables.length}: ${table.tableName}`,
+        );
+
+        // Get initial statistics for this table
+        const initialStats = await this.scanApiService.getPushStats(
+          table.tableName,
+        );
+        this.#displayStats('  Table Status', initialStats);
+
+        if (initialStats.pending === 0) {
+          console.log('  ✅ No files pending upload. Skipping.\\n');
+          continue;
+        }
+
+        console.log(
+          `\\n  🚀 Uploading ${initialStats.pending} pending files...\\n`,
+        );
+
+        // Process files for this table
+        const results = await this.#processFiles(
+          table.tableName,
+          filters,
+          parseInt(options.batchSize),
+          parseInt(options.uploadBatchSize),
+          uploadApiConfig,
+          options,
+        );
+
+        totalResults.processed += results.processed;
+        totalResults.uploaded += results.uploaded;
+        totalResults.errors += results.errors;
+
+        // Display results for this table
+        console.log('\\n  📊 Table Results:');
+        console.log(`     Files Processed: ${results.processed}`);
+        console.log(`     Uploaded: ${results.uploaded}`);
+        console.log(`     Errors: ${results.errors}\\n`);
+      }
+
+      // Display combined results
+      const duration = ((Date.now() - totalResults.startTime) / 1000).toFixed(
+        1,
+      );
+      const speed =
+        totalResults.processed > 0
+          ? (totalResults.processed / parseFloat(duration)).toFixed(0)
+          : 0;
+
+      // Report completion
+      this.#reportProgress(
+        100,
+        `Push completed: ${totalResults.uploaded} files uploaded`,
+      );
+
+      console.log('\\n✅ Push Complete!\\n');
+      console.log('📊 Total Results:');
+      console.log(`   Tables Processed: ${tables.length}`);
+      console.log(`   Files Processed: ${totalResults.processed}`);
+      console.log(`   Uploaded: ${totalResults.uploaded}`);
+      console.log(`   Errors: ${totalResults.errors}`);
+      console.log(`   Duration: ${duration}s`);
+      console.log(`   Speed: ${speed} files/sec\\n`);
+    } catch (error) {
+      console.error('\n❌ Push failed:', error.message);
+      logger.error('Push command error:', error);
+      process.exit(1);
+    }
+  }
+
+  /**
+   * Validate configuration
+   * @private
+   * @returns {string[]} Array of error messages
+   */
+  #validateConfig() {
+    const errors = [];
+
+    try {
+      appConfig.validateScanConfig();
+    } catch (err) {
+      return [err.message];
+    }
+
+    const scanConfig = appConfig.getScanConfig();
+    // When basePathFull is '*' (cross-directory wildcard), tableName is intentionally null.
+    // The push command fetches tables dynamically via getInstanceTables, so a static
+    // tableName is not required — only companySlug, serverId, and basePathFull matter.
+    if (!scanConfig.tableName && scanConfig.basePathFull !== '*') {
+      errors.push('Could not generate table name from configuration');
+    }
+
+    return errors;
+  }
+
+  /**
+   * Display statistics
+   * @private
+   */
+  #displayStats(title, stats) {
+    console.log(`\n📈 ${title}:`);
+    console.log(`   Total with arela_path: ${stats.totalWithArelaPath}`);
+    console.log(`   Uploaded: ${stats.uploaded}`);
+    console.log(`   Pending: ${stats.pending}`);
+    if (stats.errors > 0) {
+      console.log(`   Errors: ${stats.errors}`);
+    }
+    if (stats.maxAttemptsReached > 0) {
+      console.log(`   Max Attempts Reached: ${stats.maxAttemptsReached}`);
+    }
+
+    // Show top RFCs if available
+    if (stats.byRfc && stats.byRfc.length > 0) {
+      console.log('\n   📊 Top RFCs:');
+      stats.byRfc.slice(0, 5).forEach((rfc) => {
+        const percent = ((rfc.uploaded / rfc.total) * 100).toFixed(1);
+        console.log(
+          `      ${rfc.rfc}: ${rfc.uploaded}/${rfc.total} (${percent}%)`,
+        );
+      });
+    }
+  }
+
+  /**
+   * Process files in batches
+   * @private
+   */
+  async #processFiles(
+    tableName,
+    filters,
+    batchSize,
+    uploadBatchSize,
+    uploadApiConfig,
+    options = {},
+  ) {
+    const results = {
+      processed: 0,
+      uploaded: 0,
+      errors: 0,
+      startTime: Date.now(),
+    };
+
+    // Get total count first for accurate progress (filtered by RFC/year)
+    const initialStats = await this.scanApiService.getPushStats(
+      tableName,
+      filters,
+    );
+    const totalToProcess = initialStats.pending;
+
+    let hasMore = true;
+
+    // Create progress bar with known total
+    const progressBar = new cliProgress.SingleBar({
+      format:
+        '📤 |{bar}| {percentage}% | {value}/{total} | {speed}/s | ✓{uploaded} ✗{errors}',
+      barCompleteChar: '█',
+      barIncompleteChar: '░',
+      hideCursor: true,
+      clearOnComplete: false,
+      stopOnComplete: true,
+    });
+
+    // Always use offset=0 because uploaded files are removed from pending query results
+    // After each batch upload, those files are no longer "pending", so the next query
+    // at offset=0 will naturally return the next batch of unprocessed files
+
+    // Track seen file IDs to detect infinite loops (scan table update failures)
+    const seenFileIds = new Set();
+    let consecutiveRepeats = 0;
+    const MAX_CONSECUTIVE_REPEATS = 3;
+
+    // Start progress bar with known total
+    progressBar.start(totalToProcess, 0, {
+      speed: 0,
+      uploaded: 0,
+      errors: 0,
+    });
+
+    while (hasMore) {
+      // Fetch batch of files (always offset=0 since uploaded files are removed from pending)
+      const files = await this.scanApiService.fetchFilesForPush(tableName, {
+        ...filters,
+        offset: 0,
+        limit: batchSize,
+      });
+
+      if (files.length === 0) {
+        hasMore = false;
+        break;
+      }
+
+      // Infinite loop protection: if the same files keep coming back,
+      // the scan table update is failing and they stay "pending" forever.
+      const allSeen = files.every((f) => seenFileIds.has(f.id));
+      if (allSeen) {
+        consecutiveRepeats++;
+        if (consecutiveRepeats >= MAX_CONSECUTIVE_REPEATS) {
+          const msg = `Aborting: same ${files.length} files returned ${MAX_CONSECUTIVE_REPEATS} times — scan table updates are likely failing.`;
+          logger.error(msg);
+          console.error(`\n⚠  ${msg}`);
+          hasMore = false;
+          break;
+        }
+      } else {
+        consecutiveRepeats = 0;
+      }
+      files.forEach((f) => seenFileIds.add(f.id));
+
+      // Upload files in smaller batches using new CLI upload endpoint
+      for (let i = 0; i < files.length; i += uploadBatchSize) {
+        const uploadBatch = files.slice(i, i + uploadBatchSize);
+        const batchResults = await this.#uploadBatchViaCli(
+          tableName,
+          uploadBatch,
+          uploadApiConfig,
+          options,
+        );
+
+        // Update counters from API response
+        batchResults.forEach((result) => {
+          results.processed++;
+          if (result.uploaded) {
+            results.uploaded++;
+          } else {
+            results.errors++;
+          }
+        });
+
+        // Update scan table via scanApi (supports cross-tenant mode)
+        // This is called on scanApi, not pushApi, because scan tables live on scanApi
+        try {
+          await this.scanApiService.batchUpdateUpload(tableName, batchResults);
+        } catch (updateError) {
+          const msg = `Failed to update scan table for batch: ${updateError.message}`;
+          logger.error(msg);
+          console.error(`\n⚠  ${msg}`);
+          // Don't fail the entire process, just log the error
+        }
+
+        // Update progress bar
+        const elapsed = (Date.now() - results.startTime) / 1000;
+        const speed =
+          elapsed > 0 ? (results.processed / elapsed).toFixed(0) : 0;
+        progressBar.update(results.processed, {
+          speed,
+          uploaded: results.uploaded,
+          errors: results.errors,
+        });
+      }
+
+      // Check if there are more files
+      // If we got fewer files than requested, we've processed all pending files
+      if (files.length < batchSize) {
+        hasMore = false;
+      }
+    }
+
+    progressBar.stop();
+
+    return results;
+  }
+
+  /**
+   * Upload a batch of files using the new CLI upload endpoint
+   * Note: Scan table is updated separately via scanApi after this batch completes
+   * @private
+   */
+  async #uploadBatchViaCli(tableName, files, uploadApiConfig, options = {}) {
+    const pushConfig = appConfig.getPushConfig();
+    const results = [];
+
+    // Process files one by one (simpler for now, can optimize to true batch later)
+    for (const file of files) {
+      const result = await this.#uploadFileViaCli(
+        tableName,
+        file,
+        uploadApiConfig,
+        pushConfig,
+        options,
+      );
+      results.push(result);
+    }
+
+    return results;
+  }
+
+  /**
+   * Upload a single file using the CLI upload endpoint
+   * @private
+   */
+  async #uploadFileViaCli(
+    tableName,
+    file,
+    uploadApiConfig,
+    pushConfig,
+    options = {},
+  ) {
+    const result = {
+      id: file.id,
+      uploaded: false,
+      uploadError: null,
+      uploadPath: null,
+      uploadedToStorageId: null,
+    };
+
+    try {
+      // Check if file exists
+      if (!fs.existsSync(file.absolute_path)) {
+        result.uploadError =
+          'FILE_NOT_FOUND: File does not exist on filesystem';
+        return result;
+      }
+
+      // Get file stats
+      const stats = fs.statSync(file.absolute_path);
+      if (!stats.isFile()) {
+        result.uploadError = 'NOT_A_FILE: Path is not a regular file';
+        return result;
+      }
+
+      // Construct upload path using arela_path
+      // arela_path format: RFC/Year/Patente/Aduana/Pedimento/
+      const uploadPath = `${file.arela_path}${file.file_name}`;
+      result.uploadPath = uploadPath;
+
+      // Create form data for CLI upload endpoint
+      const form = new FormData();
+
+      // Build folder structure: optionally prefix with PUSH_FOLDER_STRUCTURE env var
+      // arela_path already contains the logical path (RFC/Year/Patente/Aduana/Pedimento)
+      // folderStructure from config is the bucket prefix (e.g., "documents" or "uploads/2024")
+      let arelaPath = file.arela_path.endsWith('/')
+        ? file.arela_path.slice(0, -1)
+        : file.arela_path;
+
+      // Prepend folder structure prefix if configured
+      const folderStructure = pushConfig.folderStructure
+        ? `${pushConfig.folderStructure}/${arelaPath}`
+        : arelaPath;
+
+      // Create a read stream with the original filename (no encoding needed)
+      const fileStream = fs.createReadStream(file.absolute_path);
+      form.append('files', fileStream, {
+        filename: file.file_name,
+        contentType: this.#getMimeType(file.file_extension),
+      });
+
+      // Add required fields for CLI upload
+      form.append('tableName', tableName);
+      form.append('fileId', file.id);
+      form.append('folderStructure', folderStructure);
+      form.append('rfc', file.rfc);
+      form.append('bucket', pushConfig.bucket);
+      form.append('autoDetect', 'true');
+      form.append(
+        'autoOrganize',
+        options.autoOrganize !== false ? 'true' : 'false',
+      );
+      form.append('batchSize', '1');
+      form.append('clientVersion', appConfig.packageVersion);
+
+      // Upload file using new CLI upload endpoint
+      const response = await fetch(
+        `${uploadApiConfig.baseUrl}/api/storage/cli-upload`,
+        {
+          method: 'POST',
+          headers: {
+            'x-api-key': uploadApiConfig.token,
+            ...form.getHeaders(),
+          },
+          body: form,
+        },
+      );
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        result.uploadError = `HTTP ${response.status}: ${errorText}`;
+        logger.error(`✗ Failed: ${file.file_name} - ${result.uploadError}`);
+        return result;
+      }
+
+      const apiResult = await response.json();
+
+      // Check response from CLI upload endpoint
+      if (apiResult.uploaded && apiResult.uploaded.length > 0) {
+        const uploadedFile = apiResult.uploaded[0];
+        result.uploaded = true;
+        // Only assign storageId if it is a valid UUID; ignore placeholder values
+        const UUID_RE =
+          /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+        result.uploadedToStorageId =
+          uploadedFile.storageId && UUID_RE.test(uploadedFile.storageId)
+            ? uploadedFile.storageId
+            : null;
+        logger.info(`✓ Uploaded: ${file.file_name} → ${uploadPath}`);
+      } else if (apiResult.errors && apiResult.errors.length > 0) {
+        const error = apiResult.errors[0];
+        result.uploadError = `UPLOAD_FAILED: ${error.error || 'Upload failed'}`;
+        logger.error(`✗ Failed: ${file.file_name} - ${result.uploadError}`);
+      } else {
+        result.uploadError = 'Unknown upload error - no files uploaded';
+        logger.error(`✗ Failed: ${file.file_name} - ${result.uploadError}`);
+      }
+    } catch (error) {
+      result.uploadError = `UPLOAD_ERROR: ${error.message}`;
+      logger.error(`✗ Error uploading ${file.file_name}:`, error.message);
+    }
+
+    return result;
+  }
+
+  /**
+   * Get MIME type from file extension
+   * @private
+   */
+  #getMimeType(extension) {
+    const mimeTypes = {
+      pdf: 'application/pdf',
+      doc: 'application/msword',
+      docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+      xls: 'application/vnd.ms-excel',
+      xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+      txt: 'text/plain',
+      jpg: 'image/jpeg',
+      jpeg: 'image/jpeg',
+      png: 'image/png',
+      gif: 'image/gif',
+      xml: 'application/xml',
+    };
+    return mimeTypes[extension?.toLowerCase()] || 'application/octet-stream';
+  }
+
+  /**
+   * Upload a batch of files (legacy - kept for compatibility)
+   * @private
+   * @deprecated Use #uploadBatchViaCli instead
+   */
+  async #uploadBatch(files, uploadApiConfig) {
+    const uploadPromises = files.map((file) =>
+      this.#uploadFile(file, uploadApiConfig),
+    );
+    return Promise.all(uploadPromises);
+  }
+
+  /**
+   * Upload a single file
+   * @private
+   */
+  async #uploadFile(file, uploadApiConfig) {
+    const result = {
+      id: file.id,
+      uploaded: false,
+      uploadError: null,
+      uploadPath: null,
+      uploadedToStorageId: null,
+    };
+
+    try {
+      // Check if file exists
+      if (!fs.existsSync(file.absolute_path)) {
+        result.uploadError =
+          'FILE_NOT_FOUND: File does not exist on filesystem';
+        return result;
+      }
+
+      // Get file stats
+      const stats = fs.statSync(file.absolute_path);
+      if (!stats.isFile()) {
+        result.uploadError = 'NOT_A_FILE: Path is not a regular file';
+        return result;
+      }
+
+      // Construct upload path using arela_path
+      // arela_path format: RFC/Year/Patente/Aduana/Pedimento/
+      const uploadPath = `${file.arela_path}${file.file_name}`;
+      result.uploadPath = uploadPath;
+
+      // Upload file using storage API
+      const response = await this.#uploadToStorageApi(
+        file.absolute_path,
+        uploadPath,
+        uploadApiConfig,
+        {
+          rfc: file.rfc,
+          year: file.detected_pedimento_year,
+          originalPath: file.relative_path,
+        },
+      );
+
+      if (response.success) {
+        result.uploaded = true;
+        result.uploadedToStorageId = response.fileId;
+        logger.info(`✓ Uploaded: ${file.file_name} → ${uploadPath}`);
+      } else {
+        result.uploadError = `UPLOAD_FAILED: ${response.error || 'Unknown error'}`;
+        logger.error(`✗ Failed: ${file.file_name} - ${result.uploadError}`);
+      }
+    } catch (error) {
+      result.uploadError = `UPLOAD_ERROR: ${error.message}`;
+      logger.error(`✗ Error uploading ${file.file_name}:`, error.message);
+    }
+
+    return result;
+  }
+
+  /**
+   * Upload file to storage API
+   * @private
+   */
+  async #uploadToStorageApi(filePath, uploadPath, apiConfig, metadata = {}) {
+    try {
+      // Create form data
+      const form = new FormData();
+      form.append('files', fs.createReadStream(filePath));
+
+      // uploadPath format: RFC/Year/Patente/Aduana/Pedimento/filename
+      // Extract folder structure (arela_path without filename)
+      const folderStructure = path.dirname(uploadPath);
+
+      // Use batch-upload-and-process endpoint (same as legacy upload)
+      form.append('bucket', appConfig.getPushConfig().bucket);
+      form.append('folderStructure', folderStructure);
+
+      // Add RFC for multi-database routing
+      if (metadata.rfc) {
+        form.append('rfc', metadata.rfc);
+      }
+
+      // Enable auto-detection
+      form.append('autoDetect', 'true');
+      form.append('autoOrganize', 'false');
+      form.append('batchSize', '1');
+      form.append('clientVersion', appConfig.packageVersion);
+
+      // Upload file
+      const response = await fetch(
+        `${apiConfig.baseUrl}/api/storage/batch-upload-and-process`,
+        {
+          method: 'POST',
+          headers: {
+            'x-api-key': apiConfig.token,
+            ...form.getHeaders(),
+          },
+          body: form,
+        },
+      );
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        return {
+          success: false,
+          error: `HTTP ${response.status}: ${errorText}`,
+        };
+      }
+
+      const result = await response.json();
+
+      // batch-upload-and-process returns { uploaded: [], detected: [], errors: [] }
+      // Check if upload was successful
+      if (result.uploaded && result.uploaded.length > 0) {
+        const uploadedFile = result.uploaded[0];
+        return {
+          success: true,
+          fileId: uploadedFile.fileId,
+          path: uploadedFile.path,
+        };
+      } else if (result.errors && result.errors.length > 0) {
+        const error = result.errors[0];
+        return {
+          success: false,
+          error: error.error || 'Upload failed',
+        };
+      } else {
+        return {
+          success: false,
+          error: 'Unknown upload error - no files uploaded',
+        };
+      }
+    } catch (error) {
+      return {
+        success: false,
+        error: error.message,
+      };
+    }
+  }
+
+  /**
+   * Report progress to callback if available (for worker mode)
+   * @private
+   * @param {number} percent - Progress percentage (0-100)
+   * @param {string} message - Progress message
+   */
+  #reportProgress(percent, message) {
+    if (this.onProgress && typeof this.onProgress === 'function') {
+      try {
+        this.onProgress(percent, message);
+      } catch (error) {
+        logger.debug(`Progress callback error: ${error.message}`);
+      }
+    }
+  }
+}
+
+export default PushCommand;