@arela/uploader 1.0.14 → 1.0.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arela/uploader",
-  "version": "1.0.14",
+  "version": "1.0.16",
   "description": "CLI to upload files/directories to Arela",
   "bin": {
     "arela": "./src/index.js"
@@ -31,6 +31,7 @@
   "homepage": "https://github.com/inspiraCode/arela-uploader#readme",
   "dependencies": {
     "@supabase/supabase-js": "2.49.4",
+    "bullmq": "^5.71.0",
     "chokidar": "^4.0.3",
     "cli-progress": "3.12.0",
     "commander": "13.1.0",
@@ -38,6 +39,7 @@
     "form-data": "4.0.4",
     "formdata-node": "^6.0.3",
     "globby": "14.1.0",
+    "ioredis": "^5.10.0",
     "mime-types": "3.0.1",
     "node-fetch": "3.3.2",
     "office-text-extractor": "3.0.3",

package/src/commands/IdentifyCommand.js

@@ -34,9 +34,11 @@ export class IdentifyCommand {
    * @param {string} options.api - API target (default, agencia, cliente)
    * @param {number} options.batchSize - Batch size for API operations
    * @param {boolean} options.showStats - Show performance statistics
+   * @param {Function} options.onProgress - Optional callback for progress updates (percent, message)
    */
   async execute(options = {}) {
     const startTime = Date.now();
+    this.onProgress = options.onProgress || null;
 
     try {
       // Validate scan configuration (need same config as scan command)
@@ -88,8 +90,19 @@ export class IdentifyCommand {
         pending: 0,
       };
 
-      for (const table of tables) {
+      // Report initial progress
+      this.#reportProgress(
+        0,
+        `Starting identification on ${tables.length} tables`,
+      );
+
+      for (let i = 0; i < tables.length; i++) {
+        const table = tables[i];
         logger.info(`\n🔍 Processing table: ${table.tableName}`);
+        this.#reportProgress(
+          Math.round((i / tables.length) * 100),
+          `Processing table ${i + 1}/${tables.length}: ${table.tableName}`,
+        );
 
         // Get detection statistics for this table
         const stats = await this.#processTable(
@@ -108,6 +121,12 @@ export class IdentifyCommand {
       const avgSpeed =
         duration > 0 ? Math.round(totalStats.processed / duration) : 0;
 
+      // Report completion
+      this.#reportProgress(
+        100,
+        `Identification completed: ${totalStats.detected} pedimentos detected`,
+      );
+
       logger.success(`\n✅ Identification Complete!`);
       logger.info(`\n📊 Total Results:`);
       logger.info(`   Tables Processed: ${tables.length}`);
@@ -479,6 +498,22 @@ export class IdentifyCommand {
       `   Heap Total: ${(memUsage.heapTotal / 1024 / 1024).toFixed(2)} MB`,
     );
   }
+
+  /**
+   * Report progress to callback if available (for worker mode)
+   * @private
+   * @param {number} percent - Progress percentage (0-100)
+   * @param {string} message - Progress message
+   */
+  #reportProgress(percent, message) {
+    if (this.onProgress && typeof this.onProgress === 'function') {
+      try {
+        this.onProgress(percent, message);
+      } catch (error) {
+        logger.debug(`Progress callback error: ${error.message}`);
+      }
+    }
+  }
 }
 
 // Export singleton instance

package/src/commands/PollWorkerCommand.js

@@ -0,0 +1,391 @@
+import logger from '../services/LoggingService.js';
+import { PipelineApiService } from '../services/PipelineApiService.js';
+
+import appConfig from '../config/config.js';
+import ErrorHandler from '../errors/ErrorHandler.js';
+
+/**
+ * Poll Worker Command Handler
+ * Runs arela-uploader as an HTTP-polling worker, fetching jobs from the API.
+ * Alternative to BullMQ/Redis-based worker for environments without Redis access.
+ *
+ * Usage:
+ *   arela worker --poll
+ *   arela worker --poll --interval 10000
+ *
+ * Environment variables:
+ *   WORKER_POLL_INTERVAL - Polling interval in ms (default: 5000)
+ *   WORKER_POLL_API_TARGET - API target for polling (default: 'agencia')
+ *   ARELA_SERVER_ID - Server identifier for job assignment
+ */
+export class PollWorkerCommand {
+  constructor() {
+    this.errorHandler = new ErrorHandler(logger);
+    this.isShuttingDown = false;
+    this.currentJob = null;
+    this.pipelineApi = null;
+    this.pollTimer = null;
+  }
+
+  /**
+   * Execute the poll worker command
+   * @param {Object} options - Command options
+   * @param {number} options.interval - Polling interval in milliseconds
+   * @param {string} options.api - API target for polling
+   */
+  async execute(options = {}) {
+    try {
+      const workerConfig = appConfig.getWorkerConfig();
+      const pollConfig = workerConfig.poll;
+
+      // Get configuration
+      const interval = parseInt(options.interval) || pollConfig.interval;
+      const apiTarget = options.api || pollConfig.apiTarget;
+      const serverId = appConfig.getServerId();
+
+      if (!serverId) {
+        throw new Error(
+          'ARELA_SERVER_ID is required for poll mode. Set it in your .env file.',
+        );
+      }
+
+      // Initialize Pipeline API service
+      this.pipelineApi = new PipelineApiService(apiTarget);
+
+      console.log('\n🔧 Starting Arela Poll Worker');
+      console.log(`📡 API Target: ${apiTarget}`);
+      console.log(`🖥️  Server ID: ${serverId}`);
+      console.log(`⏱️  Poll Interval: ${interval}ms`);
+
+      logger.info('🔧 Starting Arela Poll Worker');
+      logger.info(`📡 API Target: ${apiTarget}`);
+      logger.info(`🖥️  Server ID: ${serverId}`);
+      logger.info(`⏱️  Poll Interval: ${interval}ms`);
+
+      // Setup graceful shutdown handlers
+      this.#setupShutdownHandlers();
+
+      // Start polling loop
+      await this.#startPolling(serverId, interval);
+    } catch (error) {
+      logger.error('❌ Poll Worker failed to start:', error.message);
+      throw error;
+    }
+  }
+
+  /**
+   * Start the polling loop
+   * @private
+   */
+  async #startPolling(serverId, interval) {
+    console.log('\n✅ Poll Worker is running. Press Ctrl+C to stop.\n');
+    logger.success('Poll Worker is running');
+
+    while (!this.isShuttingDown) {
+      try {
+        // Check for next job
+        const job = await this.pipelineApi.getNextJob(serverId);
+
+        if (job && job.id) {
+          console.log(`📥 Got job: ${job.type} for RFC ${job.rfc}`);
+          logger.info(`📥 Got job: ${job.id} - ${job.type} for RFC ${job.rfc}`);
+          await this.#processJob(job);
+        } else {
+          // Only log to file, not console (too noisy)
+          logger.debug('📭 No jobs available, waiting...');
+        }
+      } catch (error) {
+        console.error(`❌ Polling error: ${error.message}`);
+        logger.error(`❌ Polling error: ${error.message}`);
+      }
+
+      // Wait for next poll (unless shutting down)
+      if (!this.isShuttingDown) {
+        await this.#sleep(interval);
+      }
+    }
+  }
+
+  /**
+   * Process a job
+   * @private
+   */
+  async #processJob(job) {
+    this.currentJob = job;
+
+    try {
+      logger.info(`🔄 Processing ${job.type} job ${job.id}`);
+
+      // Create progress callback that reports to API
+      const onProgress = async (percent, message) => {
+        await this.pipelineApi.updateProgress(
+          job.id,
+          percent,
+          message,
+          null,
+          job.type,
+        );
+      };
+
+      // Configure API targets for this job
+      this.#configureApiTargets(job);
+
+      // Execute the appropriate command
+      let result;
+      switch (job.type) {
+        case 'scan':
+          result = await this.#processScanJob(job, onProgress);
+          break;
+        case 'identify':
+          result = await this.#processIdentifyJob(job, onProgress);
+          break;
+        case 'propagate':
+          result = await this.#processPropagateJob(job, onProgress);
+          break;
+        case 'push':
+          result = await this.#processPushJob(job, onProgress);
+          break;
+        case 'full':
+          result = await this.#processFullPipeline(job, onProgress);
+          break;
+        default:
+          throw new Error(`Unknown job type: ${job.type}`);
+      }
+
+      // Mark job as completed
+      await this.pipelineApi.completeJob(job.id, result);
+      logger.success(`✅ Job ${job.id} completed successfully`);
+    } catch (error) {
+      logger.error(`❌ Job ${job.id} failed: ${error.message}`);
+      await this.pipelineApi.failJob(job.id, error.message);
+    } finally {
+      this.currentJob = null;
+      // Reset API configuration
+      this.#resetApiConfig();
+    }
+  }
+
+  /**
+   * Configure API targets for cross-tenant operations
+   * @private
+   */
+  #configureApiTargets(job) {
+    if (job.sourceApi && job.targetApi) {
+      appConfig.setCrossTenantTargets(job.sourceApi, job.targetApi);
+    } else {
+      appConfig.setApiTarget(job.sourceApi || 'agencia');
+    }
+  }
+
+  /**
+   * Reset API configuration after job
+   * @private
+   */
+  #resetApiConfig() {
+    // Clear cross-tenant mode
+    appConfig.api.sourceTarget = null;
+    appConfig.api.targetTarget = null;
+  }
+
+  /**
+   * Override scan configuration from job data
+   * @private
+   */
+  #overrideScanConfig(job) {
+    if (job.scanConfig) {
+      if (job.scanConfig.companySlug) {
+        process.env.ARELA_COMPANY_SLUG = job.scanConfig.companySlug;
+      }
+      if (job.scanConfig.serverId) {
+        process.env.ARELA_SERVER_ID = job.scanConfig.serverId;
+      }
+      if (job.scanConfig.basePath) {
+        process.env.UPLOAD_BASE_PATH = job.scanConfig.basePath;
+        process.env.ARELA_BASE_PATH_LABEL = job.scanConfig.basePath;
+      }
+      if (job.scanConfig.directoryLevel !== undefined) {
+        process.env.SCAN_DIRECTORY_LEVEL = String(
+          job.scanConfig.directoryLevel,
+        );
+      }
+    }
+
+    // Override scan directories if provided
+    if (job.scanDirectories && job.scanDirectories.length > 0) {
+      process.env.UPLOAD_SOURCES = job.scanDirectories.join('|');
+    }
+
+    // Override file extensions if provided
+    if (job.fileExtensions && job.fileExtensions.length > 0) {
+      process.env.UPLOAD_FILE_EXTENSIONS = job.fileExtensions.join(',');
+    }
+  }
+
+  /**
+   * Process a scan job
+   * @private
+   */
+  async #processScanJob(job, onProgress) {
+    const { ScanCommand } = await import('./ScanCommand.js');
+    const scanCommand = new ScanCommand();
+
+    this.#overrideScanConfig(job);
+
+    const options = {
+      api: job.sourceApi || 'agencia',
+      countFirst: false,
+      stream: true,
+      onProgress,
+    };
+
+    return scanCommand.execute(options);
+  }
+
+  /**
+   * Process an identify job
+   * @private
+   */
+  async #processIdentifyJob(job, onProgress) {
+    const { IdentifyCommand } = await import('./IdentifyCommand.js');
+    const identifyCommand = new IdentifyCommand();
+
+    this.#overrideScanConfig(job);
+
+    const options = {
+      api: job.sourceApi || 'agencia',
+      batchSize: 100,
+      showStats: false,
+      onProgress,
+    };
+
+    return identifyCommand.execute(options);
+  }
+
+  /**
+   * Process a propagate job
+   * @private
+   */
+  async #processPropagateJob(job, onProgress) {
+    const { PropagateCommand } = await import('./PropagateCommand.js');
+
+    this.#overrideScanConfig(job);
+
+    const options = {
+      api: job.sourceApi || 'agencia',
+      batchSize: 50,
+      showStats: false,
+      onProgress,
+    };
+
+    const propagateCommand = new PropagateCommand(options);
+    return propagateCommand.execute();
+  }
+
+  /**
+   * Process a push job
+   * @private
+   */
+  async #processPushJob(job, onProgress) {
+    const { PushCommand } = await import('./PushCommand.js');
+    const pushCommand = new PushCommand();
+
+    this.#overrideScanConfig(job);
+
+    const options = {
+      scanApi: job.sourceApi || 'agencia',
+      pushApi: job.targetApi || 'cliente',
+      batchSize: 100,
+      uploadBatchSize: 10,
+      folderStructure: job.folderStructure || 'pedimento',
+      autoOrganize: true,
+      showStats: false,
+      onProgress,
+    };
+
+    return pushCommand.execute(options);
+  }
+
+  /**
+   * Process full pipeline (scan → identify → propagate → push)
+   * @private
+   */
+  async #processFullPipeline(job, onProgress) {
+    const results = {};
+
+    // Step 1: Scan (0-25%)
+    logger.info('📁 Step 1/4: Scanning files...');
+    const wrappedProgress1 = (pct, msg) =>
+      onProgress(pct * 0.25, `[Scan] ${msg}`);
+    results.scan = await this.#processScanJob(job, wrappedProgress1);
+
+    // Step 2: Identify (25-50%)
+    logger.info('🔍 Step 2/4: Identifying documents...');
+    const wrappedProgress2 = (pct, msg) =>
+      onProgress(25 + pct * 0.25, `[Identify] ${msg}`);
+    results.identify = await this.#processIdentifyJob(job, wrappedProgress2);
+
+    // Step 3: Propagate (50-75%)
+    logger.info('🔄 Step 3/4: Propagating paths...');
+    const wrappedProgress3 = (pct, msg) =>
+      onProgress(50 + pct * 0.25, `[Propagate] ${msg}`);
+    results.propagate = await this.#processPropagateJob(job, wrappedProgress3);
+
+    // Step 4: Push (75-100%)
+    logger.info('📤 Step 4/4: Pushing to storage...');
+    const wrappedProgress4 = (pct, msg) =>
+      onProgress(75 + pct * 0.25, `[Push] ${msg}`);
+    results.push = await this.#processPushJob(job, wrappedProgress4);
+
+    return results;
+  }
+
+  /**
+   * Sleep for specified milliseconds
+   * @private
+   */
+  #sleep(ms) {
+    return new Promise((resolve) => {
+      this.pollTimer = setTimeout(resolve, ms);
+    });
+  }
+
+  /**
+   * Setup graceful shutdown handlers
+   * @private
+   */
+  #setupShutdownHandlers() {
+    const shutdown = async (signal) => {
+      if (this.isShuttingDown) return;
+      this.isShuttingDown = true;
+
+      logger.info(`\n👋 Received ${signal}. Shutting down poll worker...`);
+
+      // Clear poll timer
+      if (this.pollTimer) {
+        clearTimeout(this.pollTimer);
+      }
+
+      // Wait for current job to finish if any
+      if (this.currentJob) {
+        logger.info(
+          `  ⏳ Waiting for current job to finish: ${this.currentJob.id}`,
+        );
+        // Give it a moment to complete
+        await this.#sleep(2000);
+      }
+
+      // Cleanup
+      if (this.pipelineApi) {
+        this.pipelineApi.destroy();
+      }
+
+      logger.info('✅ Poll worker stopped gracefully');
+      process.exit(0);
+    };
+
+    process.on('SIGINT', () => shutdown('SIGINT'));
+    process.on('SIGTERM', () => shutdown('SIGTERM'));
+  }
+}
+
+export default new PollWorkerCommand();

package/src/commands/PropagateCommand.js

@@ -16,6 +16,7 @@ export class PropagateCommand {
       batchSize: parseInt(options.batchSize) || 50, // Process 50 pedimentos at a time
       showStats: options.showStats || false,
       api: options.api || 'default',
+      onProgress: options.onProgress || null, // Progress callback for worker mode
     };
 
     this.scanApiService = null;
@@ -75,8 +76,19 @@ export class PropagateCommand {
         directoriesProcessed: 0,
       };
 
-      for (const table of tables) {
+      // Report initial progress
+      this.#reportProgress(
+        0,
+        `Starting propagation on ${tables.length} tables`,
+      );
+
+      for (let i = 0; i < tables.length; i++) {
+        const table = tables[i];
         console.log(`\n🔄 Processing table: ${table.tableName}\n`);
+        this.#reportProgress(
+          Math.round((i / tables.length) * 100),
+          `Processing table ${i + 1}/${tables.length}: ${table.tableName}`,
+        );
         this.tableName = table.tableName;
 
         // Process this table
@@ -95,6 +107,12 @@ export class PropagateCommand {
           ? (totalStats.filesUpdated / parseFloat(duration)).toFixed(1)
           : 0;
 
+      // Report completion
+      this.#reportProgress(
+        100,
+        `Propagation completed: ${totalStats.filesUpdated} files updated`,
+      );
+
       console.log('\\n✅ Propagation Complete!\\n');
       console.log(`📊 Total Results:`);
       console.log(`   Tables Processed: ${tables.length}`);
@@ -473,6 +491,25 @@ export class PropagateCommand {
     const used = process.memoryUsage();
     return `${Math.round(used.heapUsed / 1024 / 1024)}MB`;
   }
+
+  /**
+   * Report progress to callback if available (for worker mode)
+   * @private
+   * @param {number} percent - Progress percentage (0-100)
+   * @param {string} message - Progress message
+   */
+  #reportProgress(percent, message) {
+    if (
+      this.options.onProgress &&
+      typeof this.options.onProgress === 'function'
+    ) {
+      try {
+        this.options.onProgress(percent, message);
+      } catch (error) {
+        logger.debug(`Progress callback error: ${error.message}`);
+      }
+    }
+  }
 }
 
 export default PropagateCommand;

package/src/commands/PushCommand.js

@@ -17,13 +17,16 @@ export class PushCommand {
   constructor() {
     // ScanApiService will be initialized in execute() with proper API target
     this.scanApiService = null;
+    this.onProgress = null; // Progress callback for worker mode
   }
 
   /**
    * Execute the push command
    * @param {Object} options - Command options
+   * @param {Function} options.onProgress - Optional callback for progress updates (percent, message)
    */
   async execute(options) {
+    this.onProgress = options.onProgress || null;
     try {
       console.log('\n🚀 Starting arela push command\n');
 
@@ -127,8 +130,16 @@ export class PushCommand {
         startTime: Date.now(),
       };
 
-      for (const table of tables) {
+      // Report initial progress
+      this.#reportProgress(0, `Starting push on ${tables.length} tables`);
+
+      for (let i = 0; i < tables.length; i++) {
+        const table = tables[i];
         console.log(`\\n🚀 Processing table: ${table.tableName}\\n`);
+        this.#reportProgress(
+          Math.round((i / tables.length) * 100),
+          `Processing table ${i + 1}/${tables.length}: ${table.tableName}`,
+        );
 
         // Get initial statistics for this table
         const initialStats = await this.scanApiService.getPushStats(
@@ -175,6 +186,12 @@ export class PushCommand {
           ? (totalResults.processed / parseFloat(duration)).toFixed(0)
           : 0;
 
+      // Report completion
+      this.#reportProgress(
+        100,
+        `Push completed: ${totalResults.uploaded} files uploaded`,
+      );
+
       console.log('\\n✅ Push Complete!\\n');
       console.log('📊 Total Results:');
       console.log(`   Tables Processed: ${tables.length}`);
@@ -283,6 +300,11 @@ export class PushCommand {
     // After each batch upload, those files are no longer "pending", so the next query
     // at offset=0 will naturally return the next batch of unprocessed files
 
+    // Track seen file IDs to detect infinite loops (scan table update failures)
+    const seenFileIds = new Set();
+    let consecutiveRepeats = 0;
+    const MAX_CONSECUTIVE_REPEATS = 3;
+
     // Start progress bar with known total
     progressBar.start(totalToProcess, 0, {
       speed: 0,
@@ -303,6 +325,23 @@ export class PushCommand {
         break;
       }
 
+      // Infinite loop protection: if the same files keep coming back,
+      // the scan table update is failing and they stay "pending" forever.
+      const allSeen = files.every((f) => seenFileIds.has(f.id));
+      if (allSeen) {
+        consecutiveRepeats++;
+        if (consecutiveRepeats >= MAX_CONSECUTIVE_REPEATS) {
+          const msg = `Aborting: same ${files.length} files returned ${MAX_CONSECUTIVE_REPEATS} times — scan table updates are likely failing.`;
+          logger.error(msg);
+          console.error(`\n⚠  ${msg}`);
+          hasMore = false;
+          break;
+        }
+      } else {
+        consecutiveRepeats = 0;
+      }
+      files.forEach((f) => seenFileIds.add(f.id));
+
       // Upload files in smaller batches using new CLI upload endpoint
       for (let i = 0; i < files.length; i += uploadBatchSize) {
         const uploadBatch = files.slice(i, i + uploadBatchSize);
@@ -328,9 +367,9 @@ export class PushCommand {
         try {
           await this.scanApiService.batchUpdateUpload(tableName, batchResults);
         } catch (updateError) {
-          logger.error(
-            `Failed to update scan table for batch: ${updateError.message}`,
-          );
+          const msg = `Failed to update scan table for batch: ${updateError.message}`;
+          logger.error(msg);
+          console.error(`\n⚠  ${msg}`);
           // Don't fail the entire process, just log the error
         }
 
@@ -482,7 +521,13 @@ export class PushCommand {
       if (apiResult.uploaded && apiResult.uploaded.length > 0) {
         const uploadedFile = apiResult.uploaded[0];
         result.uploaded = true;
-        result.uploadedToStorageId = uploadedFile.storageId;
+        // Only assign storageId if it is a valid UUID; ignore placeholder values
+        const UUID_RE =
+          /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+        result.uploadedToStorageId =
+          uploadedFile.storageId && UUID_RE.test(uploadedFile.storageId)
+            ? uploadedFile.storageId
+            : null;
         logger.info(`✓ Uploaded: ${file.file_name} → ${uploadPath}`);
       } else if (apiResult.errors && apiResult.errors.length > 0) {
         const error = apiResult.errors[0];
@@ -674,6 +719,22 @@ export class PushCommand {
       };
     }
   }
+
+  /**
+   * Report progress to callback if available (for worker mode)
+   * @private
+   * @param {number} percent - Progress percentage (0-100)
+   * @param {string} message - Progress message
+   */
+  #reportProgress(percent, message) {
+    if (this.onProgress && typeof this.onProgress === 'function') {
+      try {
+        this.onProgress(percent, message);
+      } catch (error) {
+        logger.debug(`Progress callback error: ${error.message}`);
+      }
+    }
+  }
 }
 
 export default PushCommand;