@arela/uploader 1.1.3 → 1.2.0

package/.claude/worktrees/agent-multi-profile/src/commands/WorkerCommand.js

@@ -0,0 +1,337 @@
+import { Worker } from 'bullmq';
+
+import logger from '../services/LoggingService.js';
+
+import appConfig from '../config/config.js';
+import ErrorHandler from '../errors/ErrorHandler.js';
+
+/**
+ * Worker Command Handler
+ * Runs arela-uploader as a BullMQ worker, listening for jobs from the API
+ *
+ * This enables UI-controlled execution of:
+ * - scan: Filesystem scanning and file cataloging
+ * - identify: Document type detection (pedimento-simplificado)
+ * - propagate: Spread arela_path to related files
+ * - push: Upload files to storage
+ */
+export class WorkerCommand {
+  constructor() {
+    this.errorHandler = new ErrorHandler(logger);
+    this.workers = new Map();
+    this.isShuttingDown = false;
+  }
+
+  /**
+   * Execute the worker command
+   * @param {Object} options - Command options
+   * @param {string} options.queues - Comma-separated list of queues to listen to
+   * @param {number} options.concurrency - Number of concurrent jobs per queue
+   */
+  async execute(options = {}) {
+    try {
+      // Load configuration
+      const redisConfig = appConfig.getRedisConfig();
+      const workerConfig = appConfig.getWorkerConfig();
+
+      // Determine which queues to listen to
+      const queueNames = options.queues
+        ? options.queues.split(',').map((q) => q.trim())
+        : workerConfig.queues;
+
+      const concurrency =
+        parseInt(options.concurrency) || workerConfig.concurrency;
+
+      logger.info('🔧 Starting Arela Worker');
+      logger.info(`📡 Redis: ${redisConfig.host}:${redisConfig.port}`);
+      logger.info(`📋 Queues: ${queueNames.join(', ')}`);
+      logger.info(`⚡ Concurrency: ${concurrency}`);
+
+      // Setup graceful shutdown handlers
+      this.#setupShutdownHandlers();
+
+      // Start workers for each queue
+      for (const queueName of queueNames) {
+        await this.#startWorker(queueName, redisConfig, concurrency);
+      }
+
+      logger.success('\n✅ Worker is running. Press Ctrl+C to stop.\n');
+
+      // Keep the process alive
+      await this.#keepAlive();
+    } catch (error) {
+      logger.error('❌ Worker failed to start:', error.message);
+      throw error;
+    }
+  }
+
+  /**
+   * Start a worker for a specific queue
+   * @private
+   */
+  async #startWorker(queueName, redisConfig, concurrency) {
+    const processor = this.#getProcessor(queueName);
+
+    if (!processor) {
+      logger.warn(`⚠️ No processor found for queue: ${queueName}`);
+      return;
+    }
+
+    const worker = new Worker(queueName, processor, {
+      connection: redisConfig,
+      concurrency,
+      lockDuration: appConfig.getWorkerConfig().lockDuration,
+      stalledInterval: appConfig.getWorkerConfig().stalledInterval,
+    });
+
+    // Event handlers
+    worker.on('completed', (job) => {
+      logger.success(`✅ Job ${job.id} completed on ${queueName}`);
+    });
+
+    worker.on('failed', (job, err) => {
+      logger.error(`❌ Job ${job?.id} failed on ${queueName}: ${err.message}`);
+    });
+
+    worker.on('progress', (job, progress) => {
+      logger.info(`📊 Job ${job.id} progress: ${JSON.stringify(progress)}`);
+    });
+
+    worker.on('error', (err) => {
+      logger.error(`🔴 Worker error on ${queueName}: ${err.message}`);
+    });
+
+    this.workers.set(queueName, worker);
+    logger.info(`  ✓ Worker started for queue: ${queueName}`);
+  }
+
+  /**
+   * Get the processor function for a queue
+   * @private
+   */
+  #getProcessor(queueName) {
+    const processors = {
+      'pipeline-scan': this.#processScanJob.bind(this),
+      'pipeline-identify': this.#processIdentifyJob.bind(this),
+      'pipeline-propagate': this.#processPropagateJob.bind(this),
+      'pipeline-push': this.#processPushJob.bind(this),
+    };
+
+    return processors[queueName];
+  }
+
+  /**
+   * Process a scan job
+   * @private
+   */
+  async #processScanJob(job) {
+    logger.info(`🔍 Processing scan job ${job.id}`);
+
+    const { ScanCommand } = await import('./ScanCommand.js');
+    const scanCommand = new ScanCommand();
+
+    // Create progress callback for the job
+    const onProgress = (percent, message) => {
+      job.updateProgress({ percent, message });
+      job.log(message);
+    };
+
+    try {
+      // Extract options from job data
+      const options = {
+        api: job.data.apiTarget || 'default',
+        countFirst: job.data.countFirst || false,
+        stream: job.data.stream !== false,
+        // Pass progress callback
+        onProgress,
+      };
+
+      // Override scan config if provided in job
+      if (job.data.scanConfig) {
+        this.#overrideScanConfig(job.data.scanConfig);
+      }
+
+      const result = await scanCommand.execute(options);
+      return { success: true, ...result };
+    } catch (error) {
+      logger.error(`Scan job ${job.id} failed: ${error.message}`);
+      throw error;
+    }
+  }
+
+  /**
+   * Process an identify job
+   * @private
+   */
+  async #processIdentifyJob(job) {
+    logger.info(`🔍 Processing identify job ${job.id}`);
+
+    const { IdentifyCommand } = await import('./IdentifyCommand.js');
+    const identifyCommand = new IdentifyCommand();
+
+    const onProgress = (percent, message) => {
+      job.updateProgress({ percent, message });
+      job.log(message);
+    };
+
+    try {
+      const options = {
+        api: job.data.apiTarget || 'default',
+        batchSize: job.data.batchSize || 100,
+        showStats: job.data.showStats || false,
+        onProgress,
+      };
+
+      if (job.data.scanConfig) {
+        this.#overrideScanConfig(job.data.scanConfig);
+      }
+
+      const result = await identifyCommand.execute(options);
+      return { success: true, ...result };
+    } catch (error) {
+      logger.error(`Identify job ${job.id} failed: ${error.message}`);
+      throw error;
+    }
+  }
+
+  /**
+   * Process a propagate job
+   * @private
+   */
+  async #processPropagateJob(job) {
+    logger.info(`🔄 Processing propagate job ${job.id}`);
+
+    const { PropagateCommand } = await import('./PropagateCommand.js');
+
+    const onProgress = (percent, message) => {
+      job.updateProgress({ percent, message });
+      job.log(message);
+    };
+
+    try {
+      const options = {
+        api: job.data.apiTarget || 'default',
+        batchSize: job.data.batchSize || 50,
+        showStats: job.data.showStats || false,
+        onProgress,
+      };
+
+      if (job.data.scanConfig) {
+        this.#overrideScanConfig(job.data.scanConfig);
+      }
+
+      const propagateCommand = new PropagateCommand(options);
+      const result = await propagateCommand.execute();
+      return { success: true, ...result };
+    } catch (error) {
+      logger.error(`Propagate job ${job.id} failed: ${error.message}`);
+      throw error;
+    }
+  }
+
+  /**
+   * Process a push job
+   * @private
+   */
+  async #processPushJob(job) {
+    logger.info(`📤 Processing push job ${job.id}`);
+
+    const { PushCommand } = await import('./PushCommand.js');
+    const pushCommand = new PushCommand();
+
+    const onProgress = (percent, message) => {
+      job.updateProgress({ percent, message });
+      job.log(message);
+    };
+
+    try {
+      const options = {
+        api: job.data.apiTarget || 'default',
+        scanApi: job.data.scanApi,
+        pushApi: job.data.pushApi,
+        batchSize: job.data.batchSize || 100,
+        uploadBatchSize: job.data.uploadBatchSize || 10,
+        rfcs: job.data.rfcs,
+        years: job.data.years,
+        folderStructure: job.data.folderStructure,
+        autoOrganize: job.data.autoOrganize !== false,
+        showStats: job.data.showStats || false,
+        onProgress,
+      };
+
+      if (job.data.scanConfig) {
+        this.#overrideScanConfig(job.data.scanConfig);
+      }
+
+      const result = await pushCommand.execute(options);
+      return { success: true, ...result };
+    } catch (error) {
+      logger.error(`Push job ${job.id} failed: ${error.message}`);
+      throw error;
+    }
+  }
+
+  /**
+   * Override scan configuration from job data
+   * @private
+   */
+  #overrideScanConfig(scanConfig) {
+    // Temporarily override environment variables for this job
+    if (scanConfig.companySlug) {
+      process.env.ARELA_COMPANY_SLUG = scanConfig.companySlug;
+    }
+    if (scanConfig.serverId) {
+      process.env.ARELA_SERVER_ID = scanConfig.serverId;
+    }
+    if (scanConfig.basePath) {
+      process.env.UPLOAD_BASE_PATH = scanConfig.basePath;
+      process.env.ARELA_BASE_PATH_LABEL = scanConfig.basePath;
+    }
+    if (scanConfig.directoryLevel !== undefined) {
+      process.env.SCAN_DIRECTORY_LEVEL = String(scanConfig.directoryLevel);
+    }
+
+    // Reload cached config from the updated env vars
+    appConfig.reloadScanConfig();
+  }
+
+  /**
+   * Setup graceful shutdown handlers
+   * @private
+   */
+  #setupShutdownHandlers() {
+    const shutdown = async (signal) => {
+      if (this.isShuttingDown) return;
+      this.isShuttingDown = true;
+
+      logger.info(`\n👋 Received ${signal}. Shutting down workers...`);
+
+      // Close all workers gracefully
+      const closePromises = [];
+      for (const [queueName, worker] of this.workers) {
+        logger.info(`  ⏳ Closing worker: ${queueName}`);
+        closePromises.push(worker.close());
+      }
+
+      await Promise.all(closePromises);
+      logger.info('✅ All workers closed gracefully');
+      process.exit(0);
+    };
+
+    process.on('SIGINT', () => shutdown('SIGINT'));
+    process.on('SIGTERM', () => shutdown('SIGTERM'));
+  }
+
+  /**
+   * Keep the process alive while workers are running
+   * @private
+   */
+  async #keepAlive() {
+    return new Promise((resolve) => {
+      // This promise never resolves, keeping the process alive
+      // The process will exit via the shutdown handlers
+    });
+  }
+}
+
+export default new WorkerCommand();