@arela/uploader 1.0.15 → 1.0.16

package/src/commands/WorkerCommand.js

@@ -0,0 +1,334 @@
+import { Worker } from 'bullmq';
+
+import logger from '../services/LoggingService.js';
+
+import appConfig from '../config/config.js';
+import ErrorHandler from '../errors/ErrorHandler.js';
+
+/**
+ * Worker Command Handler
+ * Runs arela-uploader as a BullMQ worker, listening for jobs from the API
+ *
+ * This enables UI-controlled execution of:
+ * - scan: Filesystem scanning and file cataloging
+ * - identify: Document type detection (pedimento-simplificado)
+ * - propagate: Spread arela_path to related files
+ * - push: Upload files to storage
+ */
+export class WorkerCommand {
+  constructor() {
+    this.errorHandler = new ErrorHandler(logger);
+    this.workers = new Map();
+    this.isShuttingDown = false;
+  }
+
+  /**
+   * Execute the worker command
+   * @param {Object} options - Command options
+   * @param {string} options.queues - Comma-separated list of queues to listen to
+   * @param {number} options.concurrency - Number of concurrent jobs per queue
+   */
+  async execute(options = {}) {
+    try {
+      // Load configuration
+      const redisConfig = appConfig.getRedisConfig();
+      const workerConfig = appConfig.getWorkerConfig();
+
+      // Determine which queues to listen to
+      const queueNames = options.queues
+        ? options.queues.split(',').map((q) => q.trim())
+        : workerConfig.queues;
+
+      const concurrency =
+        parseInt(options.concurrency) || workerConfig.concurrency;
+
+      logger.info('🔧 Starting Arela Worker');
+      logger.info(`📡 Redis: ${redisConfig.host}:${redisConfig.port}`);
+      logger.info(`📋 Queues: ${queueNames.join(', ')}`);
+      logger.info(`⚡ Concurrency: ${concurrency}`);
+
+      // Setup graceful shutdown handlers
+      this.#setupShutdownHandlers();
+
+      // Start workers for each queue
+      for (const queueName of queueNames) {
+        await this.#startWorker(queueName, redisConfig, concurrency);
+      }
+
+      logger.success('\n✅ Worker is running. Press Ctrl+C to stop.\n');
+
+      // Keep the process alive
+      await this.#keepAlive();
+    } catch (error) {
+      logger.error('❌ Worker failed to start:', error.message);
+      throw error;
+    }
+  }
+
+  /**
+   * Start a worker for a specific queue
+   * @private
+   */
+  async #startWorker(queueName, redisConfig, concurrency) {
+    const processor = this.#getProcessor(queueName);
+
+    if (!processor) {
+      logger.warn(`⚠️ No processor found for queue: ${queueName}`);
+      return;
+    }
+
+    const worker = new Worker(queueName, processor, {
+      connection: redisConfig,
+      concurrency,
+      lockDuration: appConfig.getWorkerConfig().lockDuration,
+      stalledInterval: appConfig.getWorkerConfig().stalledInterval,
+    });
+
+    // Event handlers
+    worker.on('completed', (job) => {
+      logger.success(`✅ Job ${job.id} completed on ${queueName}`);
+    });
+
+    worker.on('failed', (job, err) => {
+      logger.error(`❌ Job ${job?.id} failed on ${queueName}: ${err.message}`);
+    });
+
+    worker.on('progress', (job, progress) => {
+      logger.info(`📊 Job ${job.id} progress: ${JSON.stringify(progress)}`);
+    });
+
+    worker.on('error', (err) => {
+      logger.error(`🔴 Worker error on ${queueName}: ${err.message}`);
+    });
+
+    this.workers.set(queueName, worker);
+    logger.info(`  ✓ Worker started for queue: ${queueName}`);
+  }
+
+  /**
+   * Get the processor function for a queue
+   * @private
+   */
+  #getProcessor(queueName) {
+    const processors = {
+      'pipeline-scan': this.#processScanJob.bind(this),
+      'pipeline-identify': this.#processIdentifyJob.bind(this),
+      'pipeline-propagate': this.#processPropagateJob.bind(this),
+      'pipeline-push': this.#processPushJob.bind(this),
+    };
+
+    return processors[queueName];
+  }
+
+  /**
+   * Process a scan job
+   * @private
+   */
+  async #processScanJob(job) {
+    logger.info(`🔍 Processing scan job ${job.id}`);
+
+    const { ScanCommand } = await import('./ScanCommand.js');
+    const scanCommand = new ScanCommand();
+
+    // Create progress callback for the job
+    const onProgress = (percent, message) => {
+      job.updateProgress({ percent, message });
+      job.log(message);
+    };
+
+    try {
+      // Extract options from job data
+      const options = {
+        api: job.data.apiTarget || 'default',
+        countFirst: job.data.countFirst || false,
+        stream: job.data.stream !== false,
+        // Pass progress callback
+        onProgress,
+      };
+
+      // Override scan config if provided in job
+      if (job.data.scanConfig) {
+        this.#overrideScanConfig(job.data.scanConfig);
+      }
+
+      const result = await scanCommand.execute(options);
+      return { success: true, ...result };
+    } catch (error) {
+      logger.error(`Scan job ${job.id} failed: ${error.message}`);
+      throw error;
+    }
+  }
+
+  /**
+   * Process an identify job
+   * @private
+   */
+  async #processIdentifyJob(job) {
+    logger.info(`🔍 Processing identify job ${job.id}`);
+
+    const { IdentifyCommand } = await import('./IdentifyCommand.js');
+    const identifyCommand = new IdentifyCommand();
+
+    const onProgress = (percent, message) => {
+      job.updateProgress({ percent, message });
+      job.log(message);
+    };
+
+    try {
+      const options = {
+        api: job.data.apiTarget || 'default',
+        batchSize: job.data.batchSize || 100,
+        showStats: job.data.showStats || false,
+        onProgress,
+      };
+
+      if (job.data.scanConfig) {
+        this.#overrideScanConfig(job.data.scanConfig);
+      }
+
+      const result = await identifyCommand.execute(options);
+      return { success: true, ...result };
+    } catch (error) {
+      logger.error(`Identify job ${job.id} failed: ${error.message}`);
+      throw error;
+    }
+  }
+
+  /**
+   * Process a propagate job
+   * @private
+   */
+  async #processPropagateJob(job) {
+    logger.info(`🔄 Processing propagate job ${job.id}`);
+
+    const { PropagateCommand } = await import('./PropagateCommand.js');
+
+    const onProgress = (percent, message) => {
+      job.updateProgress({ percent, message });
+      job.log(message);
+    };
+
+    try {
+      const options = {
+        api: job.data.apiTarget || 'default',
+        batchSize: job.data.batchSize || 50,
+        showStats: job.data.showStats || false,
+        onProgress,
+      };
+
+      if (job.data.scanConfig) {
+        this.#overrideScanConfig(job.data.scanConfig);
+      }
+
+      const propagateCommand = new PropagateCommand(options);
+      const result = await propagateCommand.execute();
+      return { success: true, ...result };
+    } catch (error) {
+      logger.error(`Propagate job ${job.id} failed: ${error.message}`);
+      throw error;
+    }
+  }
+
+  /**
+   * Process a push job
+   * @private
+   */
+  async #processPushJob(job) {
+    logger.info(`📤 Processing push job ${job.id}`);
+
+    const { PushCommand } = await import('./PushCommand.js');
+    const pushCommand = new PushCommand();
+
+    const onProgress = (percent, message) => {
+      job.updateProgress({ percent, message });
+      job.log(message);
+    };
+
+    try {
+      const options = {
+        api: job.data.apiTarget || 'default',
+        scanApi: job.data.scanApi,
+        pushApi: job.data.pushApi,
+        batchSize: job.data.batchSize || 100,
+        uploadBatchSize: job.data.uploadBatchSize || 10,
+        rfcs: job.data.rfcs,
+        years: job.data.years,
+        folderStructure: job.data.folderStructure,
+        autoOrganize: job.data.autoOrganize !== false,
+        showStats: job.data.showStats || false,
+        onProgress,
+      };
+
+      if (job.data.scanConfig) {
+        this.#overrideScanConfig(job.data.scanConfig);
+      }
+
+      const result = await pushCommand.execute(options);
+      return { success: true, ...result };
+    } catch (error) {
+      logger.error(`Push job ${job.id} failed: ${error.message}`);
+      throw error;
+    }
+  }
+
+  /**
+   * Override scan configuration from job data
+   * @private
+   */
+  #overrideScanConfig(scanConfig) {
+    // Temporarily override environment variables for this job
+    if (scanConfig.companySlug) {
+      process.env.ARELA_COMPANY_SLUG = scanConfig.companySlug;
+    }
+    if (scanConfig.serverId) {
+      process.env.ARELA_SERVER_ID = scanConfig.serverId;
+    }
+    if (scanConfig.basePath) {
+      process.env.UPLOAD_BASE_PATH = scanConfig.basePath;
+      process.env.ARELA_BASE_PATH_LABEL = scanConfig.basePath;
+    }
+    if (scanConfig.directoryLevel !== undefined) {
+      process.env.SCAN_DIRECTORY_LEVEL = String(scanConfig.directoryLevel);
+    }
+  }
+
+  /**
+   * Setup graceful shutdown handlers
+   * @private
+   */
+  #setupShutdownHandlers() {
+    const shutdown = async (signal) => {
+      if (this.isShuttingDown) return;
+      this.isShuttingDown = true;
+
+      logger.info(`\n👋 Received ${signal}. Shutting down workers...`);
+
+      // Close all workers gracefully
+      const closePromises = [];
+      for (const [queueName, worker] of this.workers) {
+        logger.info(`  ⏳ Closing worker: ${queueName}`);
+        closePromises.push(worker.close());
+      }
+
+      await Promise.all(closePromises);
+      logger.info('✅ All workers closed gracefully');
+      process.exit(0);
+    };
+
+    process.on('SIGINT', () => shutdown('SIGINT'));
+    process.on('SIGTERM', () => shutdown('SIGTERM'));
+  }
+
+  /**
+   * Keep the process alive while workers are running
+   * @private
+   */
+  async #keepAlive() {
+    return new Promise((resolve) => {
+      // This promise never resolves, keeping the process alive
+      // The process will exit via the shutdown handlers
+    });
+  }
+}
+
+export default new WorkerCommand();

package/src/config/config.js

@@ -22,6 +22,8 @@ class Config {
     this.performance = this.#loadPerformanceConfig();
     this.logging = this.#loadLoggingConfig();
     this.watch = this.#loadWatchConfig();
+    this.redis = this.#loadRedisConfig();
+    this.worker = this.#loadWorkerConfig();
   }
 
   /**
@@ -34,10 +36,10 @@ class Config {
       const __dirname = path.dirname(__filename);
       const packageJsonPath = path.resolve(__dirname, '../../package.json');
       const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
-      return packageJson.version || '1.0.15';
+      return packageJson.version || '1.0.16';
     } catch (error) {
       console.warn('⚠️ Could not read package.json version, using fallback');
-      return '1.0.15';
+      return '1.0.16';
     }
   }
 
@@ -78,6 +80,11 @@ class Config {
           token:
             process.env.ARELA_API_CLIENTE_TOKEN || process.env.ARELA_API_TOKEN,
         },
+        // Dynamic targets - add more as needed via ARELA_API_{NAME}_URL
+        ktj: {
+          baseUrl: process.env.ARELA_API_KTJ_URL || process.env.ARELA_API_URL,
+          token: process.env.ARELA_API_KTJ_TOKEN || process.env.ARELA_API_TOKEN,
+        },
       },
       // Current active target (can be changed at runtime)
       activeTarget: process.env.ARELA_API_TARGET || 'default',
@@ -118,7 +125,7 @@ class Config {
    * @param {string} target - API target: 'default', 'agencia', or 'cliente'
    */
   setApiTarget(target) {
-    const validTargets = ['default', 'agencia', 'cliente'];
+    const validTargets = ['default', 'agencia', 'cliente', 'ktj'];
     if (!validTargets.includes(target.toLowerCase())) {
       throw new Error(
         `Invalid API target '${target}'. Must be one of: ${validTargets.join(', ')}`,
@@ -135,7 +142,7 @@ class Config {
    * @param {string} targetTarget - Target API target (for writing data)
    */
   setCrossTenantTargets(sourceTarget, targetTarget) {
-    const validTargets = ['default', 'agencia', 'cliente'];
+    const validTargets = ['default', 'agencia', 'cliente', 'ktj'];
 
     if (!validTargets.includes(sourceTarget.toLowerCase())) {
       throw new Error(
@@ -496,6 +503,81 @@ class Config {
     return !!(this.supabase.url && this.supabase.key && this.supabase.bucket);
   }
 
+  /**
+   * Load Redis configuration for BullMQ worker mode
+   * @private
+   */
+  #loadRedisConfig() {
+    return {
+      host: process.env.REDIS_HOST || 'localhost',
+      port: parseInt(process.env.REDIS_PORT) || 6379,
+      password: process.env.REDIS_PASSWORD || undefined,
+      db: parseInt(process.env.REDIS_DB) || 0,
+      maxRetriesPerRequest: null, // Required by BullMQ
+    };
+  }
+
+  /**
+   * Load worker configuration
+   * @private
+   */
+  #loadWorkerConfig() {
+    const queues = process.env.WORKER_QUEUES?.split(',')
+      .map((q) => q.trim())
+      .filter(Boolean) || [
+      'pipeline-scan',
+      'pipeline-identify',
+      'pipeline-propagate',
+      'pipeline-push',
+    ];
+
+    return {
+      queues,
+      concurrency: parseInt(process.env.WORKER_CONCURRENCY) || 1,
+      lockDuration: parseInt(process.env.WORKER_LOCK_DURATION) || 30000,
+      stalledInterval: parseInt(process.env.WORKER_STALLED_INTERVAL) || 30000,
+      // HTTP Polling mode configuration
+      poll: {
+        enabled: process.env.WORKER_POLL_MODE === 'true',
+        interval: parseInt(process.env.WORKER_POLL_INTERVAL) || 5000,
+        apiTarget: process.env.WORKER_POLL_API_TARGET || 'agencia',
+      },
+    };
+  }
+
+  /**
+   * Get Redis configuration
+   * @returns {Object} Redis connection options for ioredis/BullMQ
+   */
+  getRedisConfig() {
+    return this.redis;
+  }
+
+  /**
+   * Get worker configuration
+   * @returns {Object} Worker settings
+   */
+  getWorkerConfig() {
+    return this.worker;
+  }
+
+  /**
+   * Get the server ID for this instance
+   * Used for worker job assignment in poll mode
+   * @returns {string|null} Server ID or null if not configured
+   */
+  getServerId() {
+    return process.env.ARELA_SERVER_ID || null;
+  }
+
+  /**
+   * Check if worker mode is available (Redis configured)
+   * @returns {boolean}
+   */
+  isWorkerModeAvailable() {
+    return !!(this.redis.host && this.redis.port);
+  }
+
   /**
    * Validate configuration for the requested mode
    * @param {boolean} forceSupabase - Whether to force Supabase mode

package/src/index.js

@@ -2,11 +2,13 @@
 import { Command } from 'commander';
 
 import identifyCommand from './commands/IdentifyCommand.js';
+import pollWorkerCommand from './commands/PollWorkerCommand.js';
 import PropagateCommand from './commands/PropagateCommand.js';
 import PushCommand from './commands/PushCommand.js';
 import scanCommand from './commands/ScanCommand.js';
 import UploadCommand from './commands/UploadCommand.js';
 import watchCommand from './commands/WatchCommand.js';
+import workerCommand from './commands/WorkerCommand.js';
 import appConfig from './config/config.js';
 import ErrorHandler from './errors/ErrorHandler.js';
 import logger from './services/LoggingService.js';
@@ -522,6 +524,46 @@ class ArelaUploaderCLI {
         }
       });
 
+    // ============================================================================
+    // WORKER MODE - BullMQ job processor
+    // ============================================================================
+
+    // Worker command - process jobs from BullMQ queues
+    this.program
+      .command('worker')
+      .description('🔧 Run as BullMQ worker to process pipeline jobs from UI')
+      .option(
+        '--queues <queues>',
+        'Comma-separated queues to listen to (default: all pipeline queues)',
+      )
+      .option(
+        '-c, --concurrency <number>',
+        'Number of concurrent jobs per queue',
+        '1',
+      )
+      .option(
+        '--poll',
+        'Use HTTP polling mode instead of BullMQ (for environments without Redis)',
+      )
+      .option(
+        '--api <target>',
+        'API target for polling (e.g., agencia, cliente, ktj)',
+        'agencia',
+      )
+      .action(async (options) => {
+        try {
+          if (options.poll) {
+            // HTTP polling mode - for Windows Server or environments without Redis
+            await pollWorkerCommand.execute(options);
+          } else {
+            // BullMQ mode - default for environments with Redis
+            await workerCommand.execute(options);
+          }
+        } catch (error) {
+          this.errorHandler.handleFatalError(error, { command: 'worker' });
+        }
+      });
+
     // Version command (already handled by program.version())
 
     // Help command