@arela/uploader 1.1.3 → 1.2.0

package/src/services/PipelineJobRunner.js

@@ -0,0 +1,389 @@
+import path from 'path';
+
+import appConfig from '../config/config.js';
+import { PathNormalizer } from '../utils/PathNormalizer.js';
+import logger from './LoggingService.js';
+
+/**
+ * Tuning env vars that can be overridden per job via pipeline_config.extraOptions
+ * (delivered as job.scanConfig). Values not present in the job revert to whatever
+ * the process started with, so overrides never leak between jobs/profiles.
+ */
+const TUNING_ENV_KEYS = [
+  'SCAN_EXCLUDE_PATTERNS',
+  'SCAN_BATCH_SIZE',
+  'PUSH_BATCH_SIZE',
+  'PUSH_UPLOAD_BATCH_SIZE',
+  'MAX_CONCURRENT_SOURCES',
+  'MAX_API_CONNECTIONS',
+  'API_CONNECTION_TIMEOUT',
+  'BATCH_DELAY',
+  'BATCH_SIZE',
+];
+
+/**
+ * Pipeline Job Runner
+ * Executes pipeline jobs (scan/identify/propagate/push/full) fetched from the API.
+ * Extracted from PollWorkerCommand so both `arela worker --poll` (single tenant)
+ * and `arela agent` (multi-profile) share the same job execution logic.
+ *
+ * NOT safe for concurrent jobs: it reconfigures the appConfig singleton and
+ * mutates process.env per job. Callers must run one job at a time per process.
+ */
+export class PipelineJobRunner {
+  /**
+   * @param {import('./PipelineApiService.js').PipelineApiService} pipelineApi - API where progress/completion is reported
+   * @param {string} serverId - Server identifier used for heartbeats and scan registration
+   */
+  constructor(pipelineApi, serverId) {
+    this.pipelineApi = pipelineApi;
+    this.serverId = serverId;
+    this.currentJob = null;
+    // Baseline for tuning overrides: what the process started with (.env or nothing)
+    this.tuningEnvSnapshot = Object.fromEntries(
+      TUNING_ENV_KEYS.map((key) => [key, process.env[key]]),
+    );
+  }
+
+  /**
+   * Process a job end-to-end: heartbeats, progress, completion/failure reporting.
+   * @param {Object} job - Job payload from GET /worker/jobs/next
+   */
+  async processJob(job) {
+    this.currentJob = job;
+
+    // Start heartbeat timer to keep worker alive during long-running jobs
+    const heartbeatInterval = setInterval(async () => {
+      try {
+        await this.pipelineApi.sendHeartbeat(this.serverId, 'busy');
+        logger.debug(`💓 Heartbeat sent for ${this.serverId}`);
+      } catch {
+        // Non-critical — progress updates also refresh heartbeat
+      }
+    }, 20000);
+
+    try {
+      logger.info(`🔄 Processing ${job.type} job ${job.id}`);
+
+      // DEBUG: Delay to observe busy status in UI (remove in production)
+      const debugDelay = parseInt(process.env.DEBUG_JOB_DELAY) || 0;
+      if (debugDelay > 0) {
+        console.log(
+          `⏳ DEBUG: Waiting ${debugDelay / 1000}s before executing job...`,
+        );
+        await this.#sleep(debugDelay);
+      }
+
+      // Create progress callback that reports to API
+      const onProgress = async (percent, message) => {
+        await this.pipelineApi.updateProgress(
+          job.id,
+          percent,
+          message,
+          null,
+          job.type,
+        );
+      };
+
+      // Configure API targets for this job
+      this.#configureApiTargets(job);
+
+      // Execute the appropriate command
+      let result;
+      switch (job.type) {
+        case 'scan':
+          result = await this.#processScanJob(job, onProgress);
+          break;
+        case 'identify':
+          result = await this.#processIdentifyJob(job, onProgress);
+          break;
+        case 'propagate':
+          result = await this.#processPropagateJob(job, onProgress);
+          break;
+        case 'push':
+          result = await this.#processPushJob(job, onProgress);
+          break;
+        case 'full':
+          result = await this.#processFullPipeline(job, onProgress);
+          break;
+        default:
+          throw new Error(`Unknown job type: ${job.type}`);
+      }
+
+      // Mark job as completed
+      await this.pipelineApi.completeJob(job.id, result);
+      logger.success(`✅ Job ${job.id} completed successfully`);
+    } catch (error) {
+      logger.error(`❌ Job ${job.id} failed: ${error.message}`);
+      await this.pipelineApi.failJob(job.id, error.message);
+    } finally {
+      clearInterval(heartbeatInterval);
+      this.currentJob = null;
+      // Reset API configuration
+      this.#resetApiConfig();
+    }
+  }
+
+  /**
+   * Configure API targets for cross-tenant operations
+   * @private
+   */
+  #configureApiTargets(job) {
+    if (job.sourceApi && job.targetApi) {
+      appConfig.setCrossTenantTargets(job.sourceApi, job.targetApi);
+    } else {
+      appConfig.setApiTarget(job.sourceApi || 'agencia');
+    }
+  }
+
+  /**
+   * Reset API configuration after job
+   * @private
+   */
+  #resetApiConfig() {
+    // Clear cross-tenant mode
+    appConfig.api.sourceTarget = null;
+    appConfig.api.targetTarget = null;
+  }
+
+  /**
+   * Override scan configuration from job data
+   * @private
+   */
+  #overrideScanConfig(job) {
+    // The runner's serverId always wins: a stale serverId inside job.scanConfig
+    // must not leak into scan registration (or into other profiles' jobs).
+    process.env.ARELA_SERVER_ID = this.serverId;
+
+    if (job.scanConfig) {
+      if (job.scanConfig.companySlug) {
+        process.env.ARELA_COMPANY_SLUG = job.scanConfig.companySlug;
+      }
+      if (job.scanConfig.basePath) {
+        process.env.UPLOAD_BASE_PATH = job.scanConfig.basePath;
+        process.env.ARELA_BASE_PATH_LABEL = job.scanConfig.basePath;
+      }
+      if (job.scanConfig.directoryLevel !== undefined) {
+        process.env.SCAN_DIRECTORY_LEVEL = String(
+          job.scanConfig.directoryLevel,
+        );
+      }
+    }
+
+    // Per-job tuning from extraOptions; absent keys revert to the process baseline
+    for (const key of TUNING_ENV_KEYS) {
+      const value = job.scanConfig?.[key];
+      if (value !== undefined && value !== null) {
+        process.env[key] = String(value);
+      } else if (this.tuningEnvSnapshot[key] !== undefined) {
+        process.env[key] = this.tuningEnvSnapshot[key];
+      } else {
+        delete process.env[key];
+      }
+    }
+
+    // Override scan directories if provided
+    if (job.scanDirectories && job.scanDirectories.length > 0) {
+      const allAbsolute = job.scanDirectories.every((d) =>
+        PathNormalizer.isAbsolutePath(d),
+      );
+
+      if (allAbsolute) {
+        const ancestor = this.#commonAncestor(job.scanDirectories);
+        // Check if ancestor is meaningful (not just root or a drive letter)
+        const isUseful =
+          ancestor.length > 1 && !/^[a-zA-Z]:[/\\]?$/.test(ancestor);
+
+        if (isUseful) {
+          // Common ancestor found — set as base path, make sources relative
+          process.env.UPLOAD_BASE_PATH = ancestor;
+          process.env.ARELA_BASE_PATH_LABEL = ancestor;
+          const relativeSources = job.scanDirectories.map(
+            (d) => path.relative(ancestor, d) || '.',
+          );
+          process.env.UPLOAD_SOURCES = relativeSources.join('|');
+        } else {
+          // Cross-drive or no common ancestor — wildcard base, absolute sources
+          process.env.UPLOAD_BASE_PATH = '*';
+          process.env.ARELA_BASE_PATH_LABEL = '*';
+          process.env.UPLOAD_SOURCES = job.scanDirectories.join('|');
+        }
+      } else {
+        process.env.UPLOAD_SOURCES = job.scanDirectories.join('|');
+      }
+    }
+
+    // Override file extensions if provided
+    if (job.fileExtensions && job.fileExtensions.length > 0) {
+      process.env.UPLOAD_FILE_EXTENSIONS = job.fileExtensions.join(',');
+    }
+
+    // Reload cached config from the updated env vars
+    appConfig.reloadScanConfig();
+  }
+
+  /**
+   * Compute the longest common ancestor directory of a list of absolute paths.
+   * Uses '/' as separator (PathNormalizer normalizes Windows \\ to /).
+   * @param {string[]} paths
+   * @returns {string}
+   */
+  #commonAncestor(paths) {
+    if (paths.length === 0) return '/';
+    if (paths.length === 1) return paths[0];
+
+    // Normalize separators so O:\exp\... becomes O:/exp/...
+    const normalized = paths.map((p) => PathNormalizer.normalizeSeparators(p));
+    const split = normalized.map((p) => p.split('/').filter(Boolean));
+    const minLen = Math.min(...split.map((s) => s.length));
+    const common = [];
+
+    for (let i = 0; i < minLen; i++) {
+      const seg = split[0][i];
+      if (split.every((s) => s[i] === seg)) {
+        common.push(seg);
+      } else {
+        break;
+      }
+    }
+
+    // Preserve drive letter format (e.g., 'O:' → 'O:/')
+    if (common.length > 0 && /^[a-zA-Z]:$/.test(common[0])) {
+      return common[0] + '/' + common.slice(1).join('/');
+    }
+    return '/' + common.join('/');
+  }
+
+  /**
+   * Process a scan job
+   * @private
+   */
+  async #processScanJob(job, onProgress) {
+    const { ScanCommand } = await import('../commands/ScanCommand.js');
+    const scanCommand = new ScanCommand();
+
+    this.#overrideScanConfig(job);
+
+    const options = {
+      api: job.sourceApi || 'agencia',
+      countFirst: false,
+      stream: true,
+      onProgress,
+    };
+
+    return scanCommand.execute(options);
+  }
+
+  /**
+   * Process an identify job
+   * @private
+   */
+  async #processIdentifyJob(job, onProgress) {
+    const { IdentifyCommand } = await import('../commands/IdentifyCommand.js');
+    const identifyCommand = new IdentifyCommand();
+
+    this.#overrideScanConfig(job);
+
+    const options = {
+      api: job.sourceApi || 'agencia',
+      batchSize: 100,
+      showStats: false,
+      onProgress,
+      ...(job.fileId && { fileId: job.fileId }),
+      ...(job.table && { table: job.table }),
+    };
+
+    return identifyCommand.execute(options);
+  }
+
+  /**
+   * Process a propagate job
+   * @private
+   */
+  async #processPropagateJob(job, onProgress) {
+    const { PropagateCommand } = await import(
+      '../commands/PropagateCommand.js'
+    );
+
+    this.#overrideScanConfig(job);
+
+    const options = {
+      api: job.sourceApi || 'agencia',
+      batchSize: 50,
+      showStats: false,
+      onProgress,
+    };
+
+    const propagateCommand = new PropagateCommand(options);
+    return propagateCommand.execute();
+  }
+
+  /**
+   * Process a push job
+   * @private
+   */
+  async #processPushJob(job, onProgress) {
+    const { PushCommand } = await import('../commands/PushCommand.js');
+    const pushCommand = new PushCommand();
+
+    this.#overrideScanConfig(job);
+
+    const options = {
+      scanApi: job.sourceApi || 'agencia',
+      pushApi: job.targetApi || 'cliente',
+      rfcs: [job.rfc],
+      batchSize: 100,
+      uploadBatchSize: 10,
+      folderStructure: job.folderStructure || 'pedimento',
+      autoOrganize: true,
+      showStats: false,
+      onProgress,
+    };
+
+    return pushCommand.execute(options);
+  }
+
+  /**
+   * Process full pipeline (scan → identify → propagate → push)
+   * @private
+   */
+  async #processFullPipeline(job, onProgress) {
+    const results = {};
+
+    // Step 1: Scan (0-25%)
+    logger.info('📁 Step 1/4: Scanning files...');
+    const wrappedProgress1 = (pct, msg) =>
+      onProgress(pct * 0.25, `[Scan] ${msg}`);
+    results.scan = await this.#processScanJob(job, wrappedProgress1);
+
+    // Step 2: Identify (25-50%)
+    logger.info('🔍 Step 2/4: Identifying documents...');
+    const wrappedProgress2 = (pct, msg) =>
+      onProgress(25 + pct * 0.25, `[Identify] ${msg}`);
+    results.identify = await this.#processIdentifyJob(job, wrappedProgress2);
+
+    // Step 3: Propagate (50-75%)
+    logger.info('🔄 Step 3/4: Propagating paths...');
+    const wrappedProgress3 = (pct, msg) =>
+      onProgress(50 + pct * 0.25, `[Propagate] ${msg}`);
+    results.propagate = await this.#processPropagateJob(job, wrappedProgress3);
+
+    // Step 4: Push (75-100%)
+    logger.info('📤 Step 4/4: Pushing to storage...');
+    const wrappedProgress4 = (pct, msg) =>
+      onProgress(75 + pct * 0.25, `[Push] ${msg}`);
+    results.push = await this.#processPushJob(job, wrappedProgress4);
+
+    return results;
+  }
+
+  /**
+   * Sleep for specified milliseconds
+   * @private
+   */
+  #sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+}
+
+export default PipelineJobRunner;

package/src/services/ProfileManager.js

@@ -0,0 +1,164 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+export const DEFAULT_PROFILES_PATH = '~/.arela/profiles.json';
+export const DEFAULT_AGENT_LOG_PATH = '~/.arela/arela-agent.log';
+const DEFAULT_POLL_INTERVAL_MS = 5000;
+
+/**
+ * Expand a leading '~' to the user's home directory
+ * @param {string} inputPath
+ * @returns {string}
+ */
+export function expandHome(inputPath) {
+  if (inputPath && inputPath.startsWith('~')) {
+    return path.join(os.homedir(), inputPath.slice(1));
+  }
+  return inputPath;
+}
+
+/**
+ * Profile Manager
+ * Loads and validates the agent's profiles.json — the only local configuration
+ * the multi-profile agent needs: one machine serverId plus {name, url, token}
+ * per tenant API. Everything else (scan directories, tuning, schedule) lives
+ * server-side in pipeline_config.
+ *
+ * Minimal schema:
+ * {
+ *   "version": 1,
+ *   "serverId": "PALCO-01",
+ *   "pollIntervalMs": 5000,                  // optional
+ *   "logFile": "~/.arela/arela-agent.log",   // optional
+ *   "profiles": [
+ *     { "name": "ktj", "url": "https://…", "token": "ak_…" },
+ *     {
+ *       "name": "cem", "url": "https://…", "token": "ak_…",
+ *       "targets": { "agencia": { "baseUrl": "https://…", "token": "ak_…" } },
+ *       "env": { "SCAN_DIRECTORY_LEVEL": "2" }
+ *     }
+ *   ]
+ * }
+ */
+export class ProfileManager {
+  /**
+   * Load, validate and normalize the profiles file.
+   * @param {string} [configPath] - Path to profiles.json (supports '~')
+   * @returns {{serverId: string, pollIntervalMs: number, logFile: string, profiles: Object[]}}
+   */
+  static loadProfiles(configPath = DEFAULT_PROFILES_PATH) {
+    const resolvedPath = path.resolve(expandHome(configPath));
+
+    if (!fs.existsSync(resolvedPath)) {
+      throw new Error(
+        `Profiles file not found: ${resolvedPath}. ` +
+          `Run 'arela agent init --from <dir...>' to generate one from existing .env folders.`,
+      );
+    }
+
+    let raw;
+    try {
+      raw = JSON.parse(fs.readFileSync(resolvedPath, 'utf-8'));
+    } catch (error) {
+      throw new Error(`Invalid JSON in ${resolvedPath}: ${error.message}`);
+    }
+
+    if (!raw.serverId || typeof raw.serverId !== 'string') {
+      throw new Error(
+        `${resolvedPath}: 'serverId' is required at the root level (one per machine, e.g. "PALCO-01")`,
+      );
+    }
+
+    if (!Array.isArray(raw.profiles) || raw.profiles.length === 0) {
+      throw new Error(`${resolvedPath}: 'profiles' must be a non-empty array`);
+    }
+
+    const seenNames = new Set();
+    const profiles = [];
+
+    raw.profiles.forEach((profile, index) => {
+      const label = `Profile ${profile?.name ? `'${profile.name}'` : `#${index + 1}`}`;
+
+      if (!profile.name || typeof profile.name !== 'string') {
+        throw new Error(`Profile #${index + 1}: 'name' is required`);
+      }
+      const name = profile.name.toLowerCase();
+      if (seenNames.has(name)) {
+        throw new Error(`${label}: duplicate profile name`);
+      }
+      seenNames.add(name);
+
+      if (profile.enabled === false) return;
+
+      this.#validateEndpoint(label, profile.url, profile.token);
+
+      const targets = {
+        // Same tenant answers under every symbolic name jobs may reference
+        agencia: { baseUrl: profile.url, token: profile.token },
+        cliente: { baseUrl: profile.url, token: profile.token },
+        [name]: { baseUrl: profile.url, token: profile.token },
+      };
+
+      // Explicit targets override symbolic names (real cross-tenant cases)
+      for (const [targetName, targetConfig] of Object.entries(
+        profile.targets || {},
+      )) {
+        this.#validateEndpoint(
+          `${label}: targets.${targetName}`,
+          targetConfig?.baseUrl,
+          targetConfig?.token,
+        );
+        targets[targetName.toLowerCase()] = {
+          baseUrl: targetConfig.baseUrl,
+          token: targetConfig.token,
+        };
+      }
+
+      if (profile.env && typeof profile.env !== 'object') {
+        throw new Error(`${label}: 'env' must be an object of key/value pairs`);
+      }
+
+      profiles.push({
+        name,
+        url: profile.url,
+        token: profile.token,
+        serverId: raw.serverId,
+        targets,
+        env: profile.env || {},
+      });
+    });
+
+    if (profiles.length === 0) {
+      throw new Error(`${resolvedPath}: all profiles are disabled`);
+    }
+
+    return {
+      serverId: raw.serverId,
+      pollIntervalMs:
+        parseInt(raw.pollIntervalMs) > 0
+          ? parseInt(raw.pollIntervalMs)
+          : DEFAULT_POLL_INTERVAL_MS,
+      logFile: path.resolve(expandHome(raw.logFile || DEFAULT_AGENT_LOG_PATH)),
+      profiles,
+    };
+  }
+
+  /**
+   * Validate a {url, token} pair with a per-field error message
+   * @private
+   */
+  static #validateEndpoint(label, url, token) {
+    if (!url || typeof url !== 'string') {
+      throw new Error(`${label}: 'url' is required`);
+    }
+    if (!/^https?:\/\//.test(url)) {
+      throw new Error(`${label}: 'url' must start with http:// or https://`);
+    }
+    if (!token || typeof token !== 'string') {
+      throw new Error(`${label}: 'token' is required`);
+    }
+  }
+}
+
+export default ProfileManager;