@arela/uploader 1.1.4 → 1.2.0

package/src/config/config.js

@@ -37,10 +37,10 @@ class Config {
       const __dirname = path.dirname(__filename);
       const packageJsonPath = path.resolve(__dirname, '../../package.json');
       const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
-      return packageJson.version || '1.1.4';
+      return packageJson.version || '1.2.0';
     } catch (error) {
       console.warn('⚠️ Could not read package.json version, using fallback');
-      return '1.1.4';
+      return '1.2.0';
     }
   }
 
@@ -126,7 +126,7 @@ class Config {
    * @param {string} target - API target: 'default', 'agencia', or 'cliente'
    */
   setApiTarget(target) {
-    const validTargets = ['default', 'agencia', 'cliente', 'ktj'];
+    const validTargets = ['default', ...Object.keys(this.api.targets)];
     if (!validTargets.includes(target.toLowerCase())) {
       throw new Error(
         `Invalid API target '${target}'. Must be one of: ${validTargets.join(', ')}`,
@@ -143,7 +143,7 @@ class Config {
    * @param {string} targetTarget - Target API target (for writing data)
    */
   setCrossTenantTargets(sourceTarget, targetTarget) {
-    const validTargets = ['default', 'agencia', 'cliente', 'ktj'];
+    const validTargets = ['default', ...Object.keys(this.api.targets)];
 
     if (!validTargets.includes(sourceTarget.toLowerCase())) {
       throw new Error(
@@ -171,6 +171,41 @@ class Config {
     );
   }
 
+  /**
+   * Upsert an API target in memory (does not touch process.env).
+   * Used by agent mode to register per-profile targets at runtime.
+   * @param {string} name - Target name, e.g. 'cliente', 'agencia'
+   * @param {{baseUrl: string, token: string}} targetConfig
+   */
+  setTargetConfig(name, { baseUrl, token }) {
+    this.api.targets[name.toLowerCase()] = { baseUrl, token };
+  }
+
+  /**
+   * Apply an agent profile: API targets, server id and env overrides.
+   * Idempotent — the agent calls it before every poll/job of the profile so the
+   * singleton always reflects the ACTIVE profile (jobs reference targets by
+   * symbolic name: 'agencia', 'cliente').
+   * @param {Object} profile - Normalized profile from ProfileManager
+   */
+  applyProfile(profile) {
+    this.api.baseUrl = profile.url;
+    this.api.token = profile.token;
+    for (const [name, targetConfig] of Object.entries(profile.targets)) {
+      this.setTargetConfig(name, targetConfig);
+    }
+    if (profile.serverId) {
+      process.env.ARELA_SERVER_ID = profile.serverId;
+    }
+    for (const [key, value] of Object.entries(profile.env || {})) {
+      process.env[key] = String(value);
+    }
+    // Clear cross-tenant state left over from a previous profile's job
+    this.api.sourceTarget = null;
+    this.api.targetTarget = null;
+    this.api.activeTarget = 'default';
+  }
+
   /**
    * Get source API config for cross-tenant operations
    * @returns {Object} Source API configuration
@@ -780,12 +815,15 @@ class Config {
   }
 
   /**
-   * Reload upload and scan config from current process.env values.
-   * Must be called after modifying env vars at runtime (e.g., PollWorkerCommand).
+   * Reload upload, scan, push and performance config from current process.env
+   * values. Must be called after modifying env vars at runtime (e.g.,
+   * PipelineJobRunner applying a job's scanConfig/tuning overrides).
    */
   reloadScanConfig() {
     this.upload = this.#loadUploadConfig();
     this.scan = this.#loadScanConfig();
+    this.push = this.#loadPushConfig();
+    this.performance = this.#loadPerformanceConfig();
   }
 
   /**

package/src/index.js

@@ -1,6 +1,8 @@
 #!/usr/bin/env node
 import { Command } from 'commander';
 
+import agentCommand from './commands/AgentCommand.js';
+import agentInitCommand from './commands/AgentInitCommand.js';
 import datastageCommand from './commands/DatastageCommand.js';
 import gdriveSyncCommand from './commands/GDriveSyncCommand.js';
 import identifyCommand from './commands/IdentifyCommand.js';
@@ -646,6 +648,59 @@ class ArelaUploaderCLI {
         }
       });
 
+    // ============================================================================
+    // AGENT MODE - one process polling N tenant APIs (replaces N worker terminals)
+    // ============================================================================
+
+    const agent = this.program
+      .command('agent')
+      .description(
+        '🤖 Multi-profile polling agent: one process for all tenant APIs (profiles.json)',
+      );
+
+    agent
+      .command('run', { isDefault: true })
+      .description('Run the agent (default: `arela agent` runs this)')
+      .option(
+        '--config <path>',
+        'Path to profiles.json (default: ~/.arela/profiles.json)',
+      )
+      .option('--interval <ms>', 'Base poll interval per idle round')
+      .option(
+        '--profile <names>',
+        'Comma-separated subset of profiles to run (debugging)',
+      )
+      .action(async (options) => {
+        try {
+          await agentCommand.execute(options);
+        } catch (error) {
+          this.errorHandler.handleFatalError(error, { command: 'agent' });
+        }
+      });
+
+    agent
+      .command('init')
+      .description(
+        'Generate profiles.json + pipeline_config seeds from legacy per-folder .env files',
+      )
+      .requiredOption(
+        '--from <dirs...>',
+        'Folders containing the legacy .env files (one per RFC×source)',
+      )
+      .option(
+        '--server-id <id>',
+        'Machine server id (default: ARELA_SERVER_ID from first .env, else hostname)',
+      )
+      .option('--out <dir>', 'Output directory', '~/.arela')
+      .option('--force', 'Overwrite an existing profiles.json')
+      .action(async (options) => {
+        try {
+          await agentInitCommand.execute(options.from, options);
+        } catch (error) {
+          this.errorHandler.handleFatalError(error, { command: 'agent init' });
+        }
+      });
+
     // Version command (already handled by program.version())
 
     // Help command

package/src/services/LoggingService.js

@@ -17,6 +17,7 @@ export class LoggingService {
     this.bufferSize = appConfig.performance.logBufferSize;
     this.flushInterval = appConfig.performance.logFlushInterval;
     this.lastFlushTime = Date.now();
+    this.context = null;
 
     // Event tracking (new in Fase 5)
     this.uploadEvents = [];
@@ -50,6 +51,15 @@ export class LoggingService {
     });
   }
 
+  /**
+   * Prefix a message with the active context (agent profile name), so both the
+   * console echo and the file entry stay grep-able per profile
+   * @private
+   */
+  #withContext(message) {
+    return this.context ? `[${this.context}] ${message}` : message;
+  }
+
   /**
    * Write a log message
    * @param {string} message - Log message
@@ -80,6 +90,7 @@ export class LoggingService {
    * @param {string} message - Message to log
    */
   info(message) {
+    message = this.#withContext(message);
     this.writeLog(message, 'info');
     // Echo to console: run banners, stats and the 🧩 observability lines were
     // file-only, so operators watching the terminal never saw them (the v1
@@ -92,6 +103,7 @@ export class LoggingService {
    * @param {string} message - Message to log
    */
   warn(message) {
+    message = this.#withContext(message);
     this.writeLog(message, 'warn');
     console.warn(`⚠️ ${message}`);
   }
@@ -101,6 +113,7 @@ export class LoggingService {
    * @param {string} message - Message to log
    */
   error(message) {
+    message = this.#withContext(message);
     this.writeLog(message, 'error');
     console.error(`❌ ${message}`);
   }
@@ -111,6 +124,7 @@ export class LoggingService {
    */
   debug(message) {
     if (this.isVerbose) {
+      message = this.#withContext(message);
       this.writeLog(message, 'debug');
       console.log(`🔍 ${message}`);
     }
@@ -122,6 +136,7 @@ export class LoggingService {
    */
   verbose(message) {
     if (this.isVerbose) {
+      message = this.#withContext(message);
       this.writeLog(message, 'verbose');
       console.log(message);
     }
@@ -132,6 +147,7 @@ export class LoggingService {
    * @param {string} message - Message to log
    */
   success(message) {
+    message = this.#withContext(message);
     this.writeLog(message, 'success');
     console.log(`✅ ${message}`);
   }
@@ -173,6 +189,25 @@ export class LoggingService {
     return this.logFilePath;
   }
 
+  /**
+   * Redirect logging to a different file (agent mode uses ~/.arela/arela-agent.log
+   * instead of the cwd-bound arela-upload.log). Flushes pending entries first.
+   * @param {string} filePath - Absolute path to the new log file
+   */
+  setLogFilePath(filePath) {
+    this.flush();
+    this.logFilePath = filePath;
+  }
+
+  /**
+   * Set a context prefix added to every file log entry (e.g. the active agent
+   * profile name, so one shared log stays grep-able per profile).
+   * @param {string|null} prefix - Context name, or null to clear
+   */
+  setContext(prefix) {
+    this.context = prefix || null;
+  }
+
   /**
    * Check if verbose logging is enabled
    * @returns {boolean} True if verbose logging is enabled

package/src/services/PipelineApiService.js

@@ -146,9 +146,12 @@ export class PipelineApiService {
   /**
    * Get the next available job for this server
    * @param {string} serverId - Server identifier
+   * @param {Object} [options]
+   * @param {boolean} [options.throwOnError=false] - Rethrow request failures
+   *   instead of swallowing them (agent mode needs this to back off a dead API)
    * @returns {Promise<Object|null>} Job data or null if no jobs available
    */
-  async getNextJob(serverId) {
+  async getNextJob(serverId, { throwOnError = false } = {}) {
     try {
       const result = await this.#request(
         'GET',
@@ -156,6 +159,9 @@ export class PipelineApiService {
       );
       return result;
     } catch (error) {
+      if (throwOnError) {
+        throw error;
+      }
       logger.error(`❌ Failed to get next job: ${error.message}`);
       return null;
     }

package/src/services/PipelineJobRunner.js

@@ -0,0 +1,389 @@
+import path from 'path';
+
+import appConfig from '../config/config.js';
+import { PathNormalizer } from '../utils/PathNormalizer.js';
+import logger from './LoggingService.js';
+
+/**
+ * Tuning env vars that can be overridden per job via pipeline_config.extraOptions
+ * (delivered as job.scanConfig). Values not present in the job revert to whatever
+ * the process started with, so overrides never leak between jobs/profiles.
+ */
+const TUNING_ENV_KEYS = [
+  'SCAN_EXCLUDE_PATTERNS',
+  'SCAN_BATCH_SIZE',
+  'PUSH_BATCH_SIZE',
+  'PUSH_UPLOAD_BATCH_SIZE',
+  'MAX_CONCURRENT_SOURCES',
+  'MAX_API_CONNECTIONS',
+  'API_CONNECTION_TIMEOUT',
+  'BATCH_DELAY',
+  'BATCH_SIZE',
+];
+
+/**
+ * Pipeline Job Runner
+ * Executes pipeline jobs (scan/identify/propagate/push/full) fetched from the API.
+ * Extracted from PollWorkerCommand so both `arela worker --poll` (single tenant)
+ * and `arela agent` (multi-profile) share the same job execution logic.
+ *
+ * NOT safe for concurrent jobs: it reconfigures the appConfig singleton and
+ * mutates process.env per job. Callers must run one job at a time per process.
+ */
+export class PipelineJobRunner {
+  /**
+   * @param {import('./PipelineApiService.js').PipelineApiService} pipelineApi - API where progress/completion is reported
+   * @param {string} serverId - Server identifier used for heartbeats and scan registration
+   */
+  constructor(pipelineApi, serverId) {
+    this.pipelineApi = pipelineApi;
+    this.serverId = serverId;
+    this.currentJob = null;
+    // Baseline for tuning overrides: what the process started with (.env or nothing)
+    this.tuningEnvSnapshot = Object.fromEntries(
+      TUNING_ENV_KEYS.map((key) => [key, process.env[key]]),
+    );
+  }
+
+  /**
+   * Process a job end-to-end: heartbeats, progress, completion/failure reporting.
+   * @param {Object} job - Job payload from GET /worker/jobs/next
+   */
+  async processJob(job) {
+    this.currentJob = job;
+
+    // Start heartbeat timer to keep worker alive during long-running jobs
+    const heartbeatInterval = setInterval(async () => {
+      try {
+        await this.pipelineApi.sendHeartbeat(this.serverId, 'busy');
+        logger.debug(`💓 Heartbeat sent for ${this.serverId}`);
+      } catch {
+        // Non-critical — progress updates also refresh heartbeat
+      }
+    }, 20000);
+
+    try {
+      logger.info(`🔄 Processing ${job.type} job ${job.id}`);
+
+      // DEBUG: Delay to observe busy status in UI (remove in production)
+      const debugDelay = parseInt(process.env.DEBUG_JOB_DELAY) || 0;
+      if (debugDelay > 0) {
+        console.log(
+          `⏳ DEBUG: Waiting ${debugDelay / 1000}s before executing job...`,
+        );
+        await this.#sleep(debugDelay);
+      }
+
+      // Create progress callback that reports to API
+      const onProgress = async (percent, message) => {
+        await this.pipelineApi.updateProgress(
+          job.id,
+          percent,
+          message,
+          null,
+          job.type,
+        );
+      };
+
+      // Configure API targets for this job
+      this.#configureApiTargets(job);
+
+      // Execute the appropriate command
+      let result;
+      switch (job.type) {
+        case 'scan':
+          result = await this.#processScanJob(job, onProgress);
+          break;
+        case 'identify':
+          result = await this.#processIdentifyJob(job, onProgress);
+          break;
+        case 'propagate':
+          result = await this.#processPropagateJob(job, onProgress);
+          break;
+        case 'push':
+          result = await this.#processPushJob(job, onProgress);
+          break;
+        case 'full':
+          result = await this.#processFullPipeline(job, onProgress);
+          break;
+        default:
+          throw new Error(`Unknown job type: ${job.type}`);
+      }
+
+      // Mark job as completed
+      await this.pipelineApi.completeJob(job.id, result);
+      logger.success(`✅ Job ${job.id} completed successfully`);
+    } catch (error) {
+      logger.error(`❌ Job ${job.id} failed: ${error.message}`);
+      await this.pipelineApi.failJob(job.id, error.message);
+    } finally {
+      clearInterval(heartbeatInterval);
+      this.currentJob = null;
+      // Reset API configuration
+      this.#resetApiConfig();
+    }
+  }
+
+  /**
+   * Configure API targets for cross-tenant operations
+   * @private
+   */
+  #configureApiTargets(job) {
+    if (job.sourceApi && job.targetApi) {
+      appConfig.setCrossTenantTargets(job.sourceApi, job.targetApi);
+    } else {
+      appConfig.setApiTarget(job.sourceApi || 'agencia');
+    }
+  }
+
+  /**
+   * Reset API configuration after job
+   * @private
+   */
+  #resetApiConfig() {
+    // Clear cross-tenant mode
+    appConfig.api.sourceTarget = null;
+    appConfig.api.targetTarget = null;
+  }
+
+  /**
+   * Override scan configuration from job data
+   * @private
+   */
+  #overrideScanConfig(job) {
+    // The runner's serverId always wins: a stale serverId inside job.scanConfig
+    // must not leak into scan registration (or into other profiles' jobs).
+    process.env.ARELA_SERVER_ID = this.serverId;
+
+    if (job.scanConfig) {
+      if (job.scanConfig.companySlug) {
+        process.env.ARELA_COMPANY_SLUG = job.scanConfig.companySlug;
+      }
+      if (job.scanConfig.basePath) {
+        process.env.UPLOAD_BASE_PATH = job.scanConfig.basePath;
+        process.env.ARELA_BASE_PATH_LABEL = job.scanConfig.basePath;
+      }
+      if (job.scanConfig.directoryLevel !== undefined) {
+        process.env.SCAN_DIRECTORY_LEVEL = String(
+          job.scanConfig.directoryLevel,
+        );
+      }
+    }
+
+    // Per-job tuning from extraOptions; absent keys revert to the process baseline
+    for (const key of TUNING_ENV_KEYS) {
+      const value = job.scanConfig?.[key];
+      if (value !== undefined && value !== null) {
+        process.env[key] = String(value);
+      } else if (this.tuningEnvSnapshot[key] !== undefined) {
+        process.env[key] = this.tuningEnvSnapshot[key];
+      } else {
+        delete process.env[key];
+      }
+    }
+
+    // Override scan directories if provided
+    if (job.scanDirectories && job.scanDirectories.length > 0) {
+      const allAbsolute = job.scanDirectories.every((d) =>
+        PathNormalizer.isAbsolutePath(d),
+      );
+
+      if (allAbsolute) {
+        const ancestor = this.#commonAncestor(job.scanDirectories);
+        // Check if ancestor is meaningful (not just root or a drive letter)
+        const isUseful =
+          ancestor.length > 1 && !/^[a-zA-Z]:[/\\]?$/.test(ancestor);
+
+        if (isUseful) {
+          // Common ancestor found — set as base path, make sources relative
+          process.env.UPLOAD_BASE_PATH = ancestor;
+          process.env.ARELA_BASE_PATH_LABEL = ancestor;
+          const relativeSources = job.scanDirectories.map(
+            (d) => path.relative(ancestor, d) || '.',
+          );
+          process.env.UPLOAD_SOURCES = relativeSources.join('|');
+        } else {
+          // Cross-drive or no common ancestor — wildcard base, absolute sources
+          process.env.UPLOAD_BASE_PATH = '*';
+          process.env.ARELA_BASE_PATH_LABEL = '*';
+          process.env.UPLOAD_SOURCES = job.scanDirectories.join('|');
+        }
+      } else {
+        process.env.UPLOAD_SOURCES = job.scanDirectories.join('|');
+      }
+    }
+
+    // Override file extensions if provided
+    if (job.fileExtensions && job.fileExtensions.length > 0) {
+      process.env.UPLOAD_FILE_EXTENSIONS = job.fileExtensions.join(',');
+    }
+
+    // Reload cached config from the updated env vars
+    appConfig.reloadScanConfig();
+  }
+
+  /**
+   * Compute the longest common ancestor directory of a list of absolute paths.
+   * Uses '/' as separator (PathNormalizer normalizes Windows \\ to /).
+   * @param {string[]} paths
+   * @returns {string}
+   */
+  #commonAncestor(paths) {
+    if (paths.length === 0) return '/';
+    if (paths.length === 1) return paths[0];
+
+    // Normalize separators so O:\exp\... becomes O:/exp/...
+    const normalized = paths.map((p) => PathNormalizer.normalizeSeparators(p));
+    const split = normalized.map((p) => p.split('/').filter(Boolean));
+    const minLen = Math.min(...split.map((s) => s.length));
+    const common = [];
+
+    for (let i = 0; i < minLen; i++) {
+      const seg = split[0][i];
+      if (split.every((s) => s[i] === seg)) {
+        common.push(seg);
+      } else {
+        break;
+      }
+    }
+
+    // Preserve drive letter format (e.g., 'O:' → 'O:/')
+    if (common.length > 0 && /^[a-zA-Z]:$/.test(common[0])) {
+      return common[0] + '/' + common.slice(1).join('/');
+    }
+    return '/' + common.join('/');
+  }
+
+  /**
+   * Process a scan job
+   * @private
+   */
+  async #processScanJob(job, onProgress) {
+    const { ScanCommand } = await import('../commands/ScanCommand.js');
+    const scanCommand = new ScanCommand();
+
+    this.#overrideScanConfig(job);
+
+    const options = {
+      api: job.sourceApi || 'agencia',
+      countFirst: false,
+      stream: true,
+      onProgress,
+    };
+
+    return scanCommand.execute(options);
+  }
+
+  /**
+   * Process an identify job
+   * @private
+   */
+  async #processIdentifyJob(job, onProgress) {
+    const { IdentifyCommand } = await import('../commands/IdentifyCommand.js');
+    const identifyCommand = new IdentifyCommand();
+
+    this.#overrideScanConfig(job);
+
+    const options = {
+      api: job.sourceApi || 'agencia',
+      batchSize: 100,
+      showStats: false,
+      onProgress,
+      ...(job.fileId && { fileId: job.fileId }),
+      ...(job.table && { table: job.table }),
+    };
+
+    return identifyCommand.execute(options);
+  }
+
+  /**
+   * Process a propagate job
+   * @private
+   */
+  async #processPropagateJob(job, onProgress) {
+    const { PropagateCommand } = await import(
+      '../commands/PropagateCommand.js'
+    );
+
+    this.#overrideScanConfig(job);
+
+    const options = {
+      api: job.sourceApi || 'agencia',
+      batchSize: 50,
+      showStats: false,
+      onProgress,
+    };
+
+    const propagateCommand = new PropagateCommand(options);
+    return propagateCommand.execute();
+  }
+
+  /**
+   * Process a push job
+   * @private
+   */
+  async #processPushJob(job, onProgress) {
+    const { PushCommand } = await import('../commands/PushCommand.js');
+    const pushCommand = new PushCommand();
+
+    this.#overrideScanConfig(job);
+
+    const options = {
+      scanApi: job.sourceApi || 'agencia',
+      pushApi: job.targetApi || 'cliente',
+      rfcs: [job.rfc],
+      batchSize: 100,
+      uploadBatchSize: 10,
+      folderStructure: job.folderStructure || 'pedimento',
+      autoOrganize: true,
+      showStats: false,
+      onProgress,
+    };
+
+    return pushCommand.execute(options);
+  }
+
+  /**
+   * Process full pipeline (scan → identify → propagate → push)
+   * @private
+   */
+  async #processFullPipeline(job, onProgress) {
+    const results = {};
+
+    // Step 1: Scan (0-25%)
+    logger.info('📁 Step 1/4: Scanning files...');
+    const wrappedProgress1 = (pct, msg) =>
+      onProgress(pct * 0.25, `[Scan] ${msg}`);
+    results.scan = await this.#processScanJob(job, wrappedProgress1);
+
+    // Step 2: Identify (25-50%)
+    logger.info('🔍 Step 2/4: Identifying documents...');
+    const wrappedProgress2 = (pct, msg) =>
+      onProgress(25 + pct * 0.25, `[Identify] ${msg}`);
+    results.identify = await this.#processIdentifyJob(job, wrappedProgress2);
+
+    // Step 3: Propagate (50-75%)
+    logger.info('🔄 Step 3/4: Propagating paths...');
+    const wrappedProgress3 = (pct, msg) =>
+      onProgress(50 + pct * 0.25, `[Propagate] ${msg}`);
+    results.propagate = await this.#processPropagateJob(job, wrappedProgress3);
+
+    // Step 4: Push (75-100%)
+    logger.info('📤 Step 4/4: Pushing to storage...');
+    const wrappedProgress4 = (pct, msg) =>
+      onProgress(75 + pct * 0.25, `[Push] ${msg}`);
+    results.push = await this.#processPushJob(job, wrappedProgress4);
+
+    return results;
+  }
+
+  /**
+   * Sleep for specified milliseconds
+   * @private
+   */
+  #sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+}
+
+export default PipelineJobRunner;