@arela/uploader 1.1.3 → 1.2.0

package/.claude/worktrees/agent-multi-profile/src/services/PipelineApiService.js

@@ -0,0 +1,274 @@
+import { Agent } from 'http';
+import { Agent as HttpsAgent } from 'https';
+import fetch from 'node-fetch';
+
+import appConfig from '../config/config.js';
+import logger from './LoggingService.js';
+
+/**
+ * Pipeline API Service
+ * Handles HTTP communication with the API for pipeline job polling.
+ * Used by the poll worker mode as an alternative to Redis/BullMQ.
+ */
+export class PipelineApiService {
+  /**
+   * @param {string|null} apiTarget - API target for polling: 'agencia' recommended
+   */
+  constructor(apiTarget = 'agencia') {
+    this.apiTarget = apiTarget;
+    const apiConfig = appConfig.getApiConfig(apiTarget);
+    this.baseUrl = apiConfig.baseUrl;
+    this.token = apiConfig.token;
+
+    // Get API connection settings
+    const maxApiConnections = parseInt(process.env.MAX_API_CONNECTIONS) || 5;
+    const connectionTimeout =
+      parseInt(process.env.API_CONNECTION_TIMEOUT) || 30000;
+
+    // Retry configuration
+    this.maxRetries = parseInt(process.env.API_MAX_RETRIES) || 3;
+    this.retryDelay = parseInt(process.env.API_RETRY_DELAY) || 1000;
+
+    // Initialize HTTP agents for connection pooling
+    this.httpAgent = new Agent({
+      keepAlive: true,
+      keepAliveMsecs: 30000,
+      maxSockets: maxApiConnections,
+      timeout: connectionTimeout,
+    });
+
+    this.httpsAgent = new HttpsAgent({
+      keepAlive: true,
+      keepAliveMsecs: 30000,
+      maxSockets: maxApiConnections,
+      timeout: connectionTimeout,
+    });
+
+    logger.debug(
+      `🔗 Pipeline API Service configured for ${apiTarget} → ${this.baseUrl}`,
+    );
+  }
+
+  /**
+   * Get the appropriate HTTP agent based on URL protocol
+   * @private
+   */
+  #getAgent(url) {
+    return url.startsWith('https://') ? this.httpsAgent : this.httpAgent;
+  }
+
+  /**
+   * Make an HTTP request with retry logic
+   * @private
+   */
+  async #request(method, endpoint, body = null, retries = 0) {
+    const url = `${this.baseUrl}${endpoint}`;
+
+    try {
+      const options = {
+        method,
+        headers: {
+          'Content-Type': 'application/json',
+          'x-api-key': this.token,
+        },
+        agent: this.#getAgent(url),
+      };
+
+      if (body) {
+        options.body = JSON.stringify(body);
+      }
+
+      const response = await fetch(url, options);
+
+      if (!response.ok) {
+        // Handle non-2xx responses
+        if (response.status === 404) {
+          return null; // Resource not found is not an error for polling
+        }
+
+        const errorText = await response.text();
+        throw new Error(
+          `HTTP ${response.status}: ${response.statusText} - ${errorText}`,
+        );
+      }
+
+      // Check if response has content
+      const contentType = response.headers.get('content-type');
+      if (contentType && contentType.includes('application/json')) {
+        return response.json();
+      }
+
+      return { success: true };
+    } catch (error) {
+      // Retry on network errors
+      if (retries < this.maxRetries && this.#isRetryableError(error)) {
+        logger.debug(
+          `⚠️ Retrying request (${retries + 1}/${this.maxRetries}): ${error.message}`,
+        );
+        await this.#sleep(this.retryDelay * (retries + 1));
+        return this.#request(method, endpoint, body, retries + 1);
+      }
+
+      throw error;
+    }
+  }
+
+  /**
+   * Check if error is retryable
+   * @private
+   */
+  #isRetryableError(error) {
+    const retryableCodes = [
+      'ECONNRESET',
+      'ETIMEDOUT',
+      'ECONNREFUSED',
+      'ENOTFOUND',
+      'EAI_AGAIN',
+    ];
+    return (
+      retryableCodes.includes(error.code) ||
+      (error.message && error.message.includes('timeout'))
+    );
+  }
+
+  /**
+   * Sleep for specified milliseconds
+   * @private
+   */
+  #sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
+  // =====================
+  // Public API Methods
+  // =====================
+
+  /**
+   * Get the next available job for this server
+   * @param {string} serverId - Server identifier
+   * @param {Object} [options]
+   * @param {boolean} [options.throwOnError=false] - Rethrow request failures
+   *   instead of swallowing them (agent mode needs this to back off a dead API)
+   * @returns {Promise<Object|null>} Job data or null if no jobs available
+   */
+  async getNextJob(serverId, { throwOnError = false } = {}) {
+    try {
+      const result = await this.#request(
+        'GET',
+        `/api/uploader/pipeline/worker/jobs/next?serverId=${encodeURIComponent(serverId)}`,
+      );
+      return result;
+    } catch (error) {
+      if (throwOnError) {
+        throw error;
+      }
+      logger.error(`❌ Failed to get next job: ${error.message}`);
+      return null;
+    }
+  }
+
+  /**
+   * Update job progress
+   * @param {string} jobId - Job UUID
+   * @param {number} progress - Progress percentage (0-100)
+   * @param {string} message - Progress message
+   * @param {string} currentFile - Current file being processed
+   * @param {string} currentStep - Current pipeline step
+   * @returns {Promise<boolean>} Success status
+   */
+  async updateProgress(jobId, progress, message, currentFile, currentStep) {
+    try {
+      await this.#request(
+        'PATCH',
+        `/api/uploader/pipeline/worker/jobs/${jobId}/progress`,
+        {
+          progress,
+          message,
+          currentFile,
+          currentStep,
+        },
+      );
+      return true;
+    } catch (error) {
+      logger.warn(`⚠️ Failed to update progress: ${error.message}`);
+      return false;
+    }
+  }
+
+  /**
+   * Mark job as completed
+   * @param {string} jobId - Job UUID
+   * @param {Object} result - Result data
+   * @returns {Promise<boolean>} Success status
+   */
+  async completeJob(jobId, result = {}) {
+    try {
+      await this.#request(
+        'PATCH',
+        `/api/uploader/pipeline/worker/jobs/${jobId}/complete`,
+        {
+          status: 'completed',
+          result,
+        },
+      );
+      logger.success(`✅ Job ${jobId} marked as completed`);
+      return true;
+    } catch (error) {
+      logger.error(`❌ Failed to complete job: ${error.message}`);
+      return false;
+    }
+  }
+
+  /**
+   * Mark job as failed
+   * @param {string} jobId - Job UUID
+   * @param {string} errorMessage - Error description
+   * @returns {Promise<boolean>} Success status
+   */
+  async failJob(jobId, errorMessage) {
+    try {
+      await this.#request(
+        'PATCH',
+        `/api/uploader/pipeline/worker/jobs/${jobId}/complete`,
+        {
+          status: 'failed',
+          error: errorMessage,
+        },
+      );
+      logger.error(`❌ Job ${jobId} marked as failed: ${errorMessage}`);
+      return true;
+    } catch (error) {
+      logger.error(`❌ Failed to mark job as failed: ${error.message}`);
+      return false;
+    }
+  }
+
+  /**
+   * Send a heartbeat to keep the worker alive during long-running jobs
+   * @param {string} serverId - Server identifier
+   * @param {'online'|'busy'} status - Worker status
+   * @returns {Promise<boolean>} Success status
+   */
+  async sendHeartbeat(serverId, status = 'busy') {
+    try {
+      await this.#request('POST', '/api/uploader/pipeline/worker/heartbeat', {
+        serverId,
+        status,
+      });
+      return true;
+    } catch (error) {
+      logger.warn(`⚠️ Failed to send heartbeat: ${error.message}`);
+      return false;
+    }
+  }
+
+  /**
+   * Close connections and cleanup
+   */
+  destroy() {
+    this.httpAgent.destroy();
+    this.httpsAgent.destroy();
+  }
+}
+
+export default PipelineApiService;

package/.claude/worktrees/agent-multi-profile/src/services/PipelineJobRunner.js

@@ -0,0 +1,389 @@
+import path from 'path';
+
+import appConfig from '../config/config.js';
+import { PathNormalizer } from '../utils/PathNormalizer.js';
+import logger from './LoggingService.js';
+
+/**
+ * Tuning env vars that can be overridden per job via pipeline_config.extraOptions
+ * (delivered as job.scanConfig). Values not present in the job revert to whatever
+ * the process started with, so overrides never leak between jobs/profiles.
+ */
+const TUNING_ENV_KEYS = [
+  'SCAN_EXCLUDE_PATTERNS',
+  'SCAN_BATCH_SIZE',
+  'PUSH_BATCH_SIZE',
+  'PUSH_UPLOAD_BATCH_SIZE',
+  'MAX_CONCURRENT_SOURCES',
+  'MAX_API_CONNECTIONS',
+  'API_CONNECTION_TIMEOUT',
+  'BATCH_DELAY',
+  'BATCH_SIZE',
+];
+
+/**
+ * Pipeline Job Runner
+ * Executes pipeline jobs (scan/identify/propagate/push/full) fetched from the API.
+ * Extracted from PollWorkerCommand so both `arela worker --poll` (single tenant)
+ * and `arela agent` (multi-profile) share the same job execution logic.
+ *
+ * NOT safe for concurrent jobs: it reconfigures the appConfig singleton and
+ * mutates process.env per job. Callers must run one job at a time per process.
+ */
+export class PipelineJobRunner {
+  /**
+   * @param {import('./PipelineApiService.js').PipelineApiService} pipelineApi - API where progress/completion is reported
+   * @param {string} serverId - Server identifier used for heartbeats and scan registration
+   */
+  constructor(pipelineApi, serverId) {
+    this.pipelineApi = pipelineApi;
+    this.serverId = serverId;
+    this.currentJob = null;
+    // Baseline for tuning overrides: what the process started with (.env or nothing)
+    this.tuningEnvSnapshot = Object.fromEntries(
+      TUNING_ENV_KEYS.map((key) => [key, process.env[key]]),
+    );
+  }
+
+  /**
+   * Process a job end-to-end: heartbeats, progress, completion/failure reporting.
+   * @param {Object} job - Job payload from GET /worker/jobs/next
+   */
+  async processJob(job) {
+    this.currentJob = job;
+
+    // Start heartbeat timer to keep worker alive during long-running jobs
+    const heartbeatInterval = setInterval(async () => {
+      try {
+        await this.pipelineApi.sendHeartbeat(this.serverId, 'busy');
+        logger.debug(`💓 Heartbeat sent for ${this.serverId}`);
+      } catch {
+        // Non-critical — progress updates also refresh heartbeat
+      }
+    }, 20000);
+
+    try {
+      logger.info(`🔄 Processing ${job.type} job ${job.id}`);
+
+      // DEBUG: Delay to observe busy status in UI (remove in production)
+      const debugDelay = parseInt(process.env.DEBUG_JOB_DELAY) || 0;
+      if (debugDelay > 0) {
+        console.log(
+          `⏳ DEBUG: Waiting ${debugDelay / 1000}s before executing job...`,
+        );
+        await this.#sleep(debugDelay);
+      }
+
+      // Create progress callback that reports to API
+      const onProgress = async (percent, message) => {
+        await this.pipelineApi.updateProgress(
+          job.id,
+          percent,
+          message,
+          null,
+          job.type,
+        );
+      };
+
+      // Configure API targets for this job
+      this.#configureApiTargets(job);
+
+      // Execute the appropriate command
+      let result;
+      switch (job.type) {
+        case 'scan':
+          result = await this.#processScanJob(job, onProgress);
+          break;
+        case 'identify':
+          result = await this.#processIdentifyJob(job, onProgress);
+          break;
+        case 'propagate':
+          result = await this.#processPropagateJob(job, onProgress);
+          break;
+        case 'push':
+          result = await this.#processPushJob(job, onProgress);
+          break;
+        case 'full':
+          result = await this.#processFullPipeline(job, onProgress);
+          break;
+        default:
+          throw new Error(`Unknown job type: ${job.type}`);
+      }
+
+      // Mark job as completed
+      await this.pipelineApi.completeJob(job.id, result);
+      logger.success(`✅ Job ${job.id} completed successfully`);
+    } catch (error) {
+      logger.error(`❌ Job ${job.id} failed: ${error.message}`);
+      await this.pipelineApi.failJob(job.id, error.message);
+    } finally {
+      clearInterval(heartbeatInterval);
+      this.currentJob = null;
+      // Reset API configuration
+      this.#resetApiConfig();
+    }
+  }
+
+  /**
+   * Configure API targets for cross-tenant operations
+   * @private
+   */
+  #configureApiTargets(job) {
+    if (job.sourceApi && job.targetApi) {
+      appConfig.setCrossTenantTargets(job.sourceApi, job.targetApi);
+    } else {
+      appConfig.setApiTarget(job.sourceApi || 'agencia');
+    }
+  }
+
+  /**
+   * Reset API configuration after job
+   * @private
+   */
+  #resetApiConfig() {
+    // Clear cross-tenant mode
+    appConfig.api.sourceTarget = null;
+    appConfig.api.targetTarget = null;
+  }
+
+  /**
+   * Override scan configuration from job data
+   * @private
+   */
+  #overrideScanConfig(job) {
+    // The runner's serverId always wins: a stale serverId inside job.scanConfig
+    // must not leak into scan registration (or into other profiles' jobs).
+    process.env.ARELA_SERVER_ID = this.serverId;
+
+    if (job.scanConfig) {
+      if (job.scanConfig.companySlug) {
+        process.env.ARELA_COMPANY_SLUG = job.scanConfig.companySlug;
+      }
+      if (job.scanConfig.basePath) {
+        process.env.UPLOAD_BASE_PATH = job.scanConfig.basePath;
+        process.env.ARELA_BASE_PATH_LABEL = job.scanConfig.basePath;
+      }
+      if (job.scanConfig.directoryLevel !== undefined) {
+        process.env.SCAN_DIRECTORY_LEVEL = String(
+          job.scanConfig.directoryLevel,
+        );
+      }
+    }
+
+    // Per-job tuning from extraOptions; absent keys revert to the process baseline
+    for (const key of TUNING_ENV_KEYS) {
+      const value = job.scanConfig?.[key];
+      if (value !== undefined && value !== null) {
+        process.env[key] = String(value);
+      } else if (this.tuningEnvSnapshot[key] !== undefined) {
+        process.env[key] = this.tuningEnvSnapshot[key];
+      } else {
+        delete process.env[key];
+      }
+    }
+
+    // Override scan directories if provided
+    if (job.scanDirectories && job.scanDirectories.length > 0) {
+      const allAbsolute = job.scanDirectories.every((d) =>
+        PathNormalizer.isAbsolutePath(d),
+      );
+
+      if (allAbsolute) {
+        const ancestor = this.#commonAncestor(job.scanDirectories);
+        // Check if ancestor is meaningful (not just root or a drive letter)
+        const isUseful =
+          ancestor.length > 1 && !/^[a-zA-Z]:[/\\]?$/.test(ancestor);
+
+        if (isUseful) {
+          // Common ancestor found — set as base path, make sources relative
+          process.env.UPLOAD_BASE_PATH = ancestor;
+          process.env.ARELA_BASE_PATH_LABEL = ancestor;
+          const relativeSources = job.scanDirectories.map(
+            (d) => path.relative(ancestor, d) || '.',
+          );
+          process.env.UPLOAD_SOURCES = relativeSources.join('|');
+        } else {
+          // Cross-drive or no common ancestor — wildcard base, absolute sources
+          process.env.UPLOAD_BASE_PATH = '*';
+          process.env.ARELA_BASE_PATH_LABEL = '*';
+          process.env.UPLOAD_SOURCES = job.scanDirectories.join('|');
+        }
+      } else {
+        process.env.UPLOAD_SOURCES = job.scanDirectories.join('|');
+      }
+    }
+
+    // Override file extensions if provided
+    if (job.fileExtensions && job.fileExtensions.length > 0) {
+      process.env.UPLOAD_FILE_EXTENSIONS = job.fileExtensions.join(',');
+    }
+
+    // Reload cached config from the updated env vars
+    appConfig.reloadScanConfig();
+  }
+
+  /**
+   * Compute the longest common ancestor directory of a list of absolute paths.
+   * Uses '/' as separator (PathNormalizer normalizes Windows \\ to /).
+   * @param {string[]} paths
+   * @returns {string}
+   */
+  #commonAncestor(paths) {
+    if (paths.length === 0) return '/';
+    if (paths.length === 1) return paths[0];
+
+    // Normalize separators so O:\exp\... becomes O:/exp/...
+    const normalized = paths.map((p) => PathNormalizer.normalizeSeparators(p));
+    const split = normalized.map((p) => p.split('/').filter(Boolean));
+    const minLen = Math.min(...split.map((s) => s.length));
+    const common = [];
+
+    for (let i = 0; i < minLen; i++) {
+      const seg = split[0][i];
+      if (split.every((s) => s[i] === seg)) {
+        common.push(seg);
+      } else {
+        break;
+      }
+    }
+
+    // Preserve drive letter format (e.g., 'O:' → 'O:/')
+    if (common.length > 0 && /^[a-zA-Z]:$/.test(common[0])) {
+      return common[0] + '/' + common.slice(1).join('/');
+    }
+    return '/' + common.join('/');
+  }
+
+  /**
+   * Process a scan job
+   * @private
+   */
+  async #processScanJob(job, onProgress) {
+    const { ScanCommand } = await import('../commands/ScanCommand.js');
+    const scanCommand = new ScanCommand();
+
+    this.#overrideScanConfig(job);
+
+    const options = {
+      api: job.sourceApi || 'agencia',
+      countFirst: false,
+      stream: true,
+      onProgress,
+    };
+
+    return scanCommand.execute(options);
+  }
+
+  /**
+   * Process an identify job
+   * @private
+   */
+  async #processIdentifyJob(job, onProgress) {
+    const { IdentifyCommand } = await import('../commands/IdentifyCommand.js');
+    const identifyCommand = new IdentifyCommand();
+
+    this.#overrideScanConfig(job);
+
+    const options = {
+      api: job.sourceApi || 'agencia',
+      batchSize: 100,
+      showStats: false,
+      onProgress,
+      ...(job.fileId && { fileId: job.fileId }),
+      ...(job.table && { table: job.table }),
+    };
+
+    return identifyCommand.execute(options);
+  }
+
+  /**
+   * Process a propagate job
+   * @private
+   */
+  async #processPropagateJob(job, onProgress) {
+    const { PropagateCommand } = await import(
+      '../commands/PropagateCommand.js'
+    );
+
+    this.#overrideScanConfig(job);
+
+    const options = {
+      api: job.sourceApi || 'agencia',
+      batchSize: 50,
+      showStats: false,
+      onProgress,
+    };
+
+    const propagateCommand = new PropagateCommand(options);
+    return propagateCommand.execute();
+  }
+
+  /**
+   * Process a push job
+   * @private
+   */
+  async #processPushJob(job, onProgress) {
+    const { PushCommand } = await import('../commands/PushCommand.js');
+    const pushCommand = new PushCommand();
+
+    this.#overrideScanConfig(job);
+
+    const options = {
+      scanApi: job.sourceApi || 'agencia',
+      pushApi: job.targetApi || 'cliente',
+      rfcs: [job.rfc],
+      batchSize: 100,
+      uploadBatchSize: 10,
+      folderStructure: job.folderStructure || 'pedimento',
+      autoOrganize: true,
+      showStats: false,
+      onProgress,
+    };
+
+    return pushCommand.execute(options);
+  }
+
+  /**
+   * Process full pipeline (scan → identify → propagate → push)
+   * @private
+   */
+  async #processFullPipeline(job, onProgress) {
+    const results = {};
+
+    // Step 1: Scan (0-25%)
+    logger.info('📁 Step 1/4: Scanning files...');
+    const wrappedProgress1 = (pct, msg) =>
+      onProgress(pct * 0.25, `[Scan] ${msg}`);
+    results.scan = await this.#processScanJob(job, wrappedProgress1);
+
+    // Step 2: Identify (25-50%)
+    logger.info('🔍 Step 2/4: Identifying documents...');
+    const wrappedProgress2 = (pct, msg) =>
+      onProgress(25 + pct * 0.25, `[Identify] ${msg}`);
+    results.identify = await this.#processIdentifyJob(job, wrappedProgress2);
+
+    // Step 3: Propagate (50-75%)
+    logger.info('🔄 Step 3/4: Propagating paths...');
+    const wrappedProgress3 = (pct, msg) =>
+      onProgress(50 + pct * 0.25, `[Propagate] ${msg}`);
+    results.propagate = await this.#processPropagateJob(job, wrappedProgress3);
+
+    // Step 4: Push (75-100%)
+    logger.info('📤 Step 4/4: Pushing to storage...');
+    const wrappedProgress4 = (pct, msg) =>
+      onProgress(75 + pct * 0.25, `[Push] ${msg}`);
+    results.push = await this.#processPushJob(job, wrappedProgress4);
+
+    return results;
+  }
+
+  /**
+   * Sleep for specified milliseconds
+   * @private
+   */
+  #sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+}
+
+export default PipelineJobRunner;