@arela/uploader 1.0.21 → 1.0.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arela/uploader",
-  "version": "1.0.21",
+  "version": "1.0.23",
   "description": "CLI to upload files/directories to Arela",
   "bin": {
     "arela": "./src/index.js"

package/src/commands/DatastageCommand.js

@@ -0,0 +1,164 @@
+import fs from 'fs';
+import path from 'path';
+
+import { DatastageApiService } from '../services/DatastageApiService.js';
+import logger from '../services/LoggingService.js';
+
+import appConfig from '../config/config.js';
+import ErrorHandler from '../errors/ErrorHandler.js';
+
+/**
+ * Datastage Command Handler
+ * Uploads monthly Datastage *.zip files from a directory to the API.
+ * Sequential, idempotent via the cli `datastage_uploads` tracking table.
+ */
+export class DatastageCommand {
+  constructor() {
+    this.errorHandler = new ErrorHandler(logger);
+  }
+
+  /**
+   * @param {Object} options
+   * @param {string} options.dir - directory containing *.zip files (required)
+   * @param {string} [options.api] - 'default'|'agencia'|'cliente'
+   * @param {boolean} [options.retryFailed] - re-attempt files in 'failed' status
+   * @param {boolean} [options.showStats] - print final stats from API
+   */
+  async execute(options = {}) {
+    const startTime = Date.now();
+
+    if (!options.dir) {
+      throw new Error('--dir <path> is required');
+    }
+    const sourceDirectory = path.resolve(options.dir);
+    if (!fs.existsSync(sourceDirectory)) {
+      throw new Error(`Directory not found: ${sourceDirectory}`);
+    }
+    const dirStat = fs.statSync(sourceDirectory);
+    if (!dirStat.isDirectory()) {
+      throw new Error(`Not a directory: ${sourceDirectory}`);
+    }
+
+    const apiTarget = options.api || 'default';
+    const api = new DatastageApiService(apiTarget);
+
+    logger.info('📦 Starting arela datastage command');
+    logger.info(`🎯 API Target: ${apiTarget}`);
+    logger.info(`📂 Source: ${sourceDirectory}`);
+
+    // 1. Enumerate *.zip in root directory (non-recursive)
+    const entries = fs.readdirSync(sourceDirectory, { withFileTypes: true });
+    const zipFiles = entries
+      .filter((e) => e.isFile() && /\.zip$/i.test(e.name))
+      .map((e) => path.join(sourceDirectory, e.name));
+
+    if (zipFiles.length === 0) {
+      logger.warn('No *.zip files found in directory. Nothing to do.');
+      return { uploaded: 0, failed: 0, skipped: 0 };
+    }
+    logger.info(`🗂  Found ${zipFiles.length} zip file(s)`);
+
+    // 2. Register each file (idempotent upsert)
+    logger.info('📝 Registering files...');
+    for (const zipPath of zipFiles) {
+      const stats = fs.statSync(zipPath);
+      try {
+        await api.registerUpload({
+          absolutePath: zipPath,
+          fileName: path.basename(zipPath),
+          sizeBytes: stats.size,
+          fileModifiedAt: stats.mtime.toISOString(),
+          sourceDirectory,
+        });
+      } catch (err) {
+        logger.error(
+          `  ✗ register failed for ${path.basename(zipPath)}: ${err.message}`,
+        );
+        throw err;
+      }
+    }
+
+    // 3. Fetch pending list scoped to this directory
+    const pending = await api.getPending(sourceDirectory);
+    const pendingPaths = new Set(pending.map((p) => p.absolutePath));
+
+    const alreadyUploaded = zipFiles.length - pendingPaths.size;
+    if (alreadyUploaded > 0) {
+      logger.info(`⏭  Skipping ${alreadyUploaded} already uploaded file(s)`);
+    }
+
+    if (pending.length === 0) {
+      logger.success('✅ All files already uploaded. Nothing to do.');
+      if (options.showStats) {
+        const s = await api.getStats(sourceDirectory);
+        logger.info(`📊 Stats: ${JSON.stringify(s)}`);
+      }
+      return { uploaded: 0, failed: 0, skipped: alreadyUploaded };
+    }
+
+    // 4. Sequential upload loop
+    logger.info(`🚀 Uploading ${pending.length} file(s) sequentially...`);
+    let uploaded = 0;
+    let failed = 0;
+
+    for (let i = 0; i < pending.length; i++) {
+      const row = pending[i];
+      const localPath = row.absolutePath;
+      const label = `[${i + 1}/${pending.length}] ${row.fileName}`;
+
+      if (!fs.existsSync(localPath)) {
+        const err = `File missing on disk: ${localPath}`;
+        logger.error(`✗ ${label}: ${err}`);
+        try {
+          await api.markFailed(row.id, err);
+        } catch (e) {
+          logger.error(`  mark-failed error: ${e.message}`);
+        }
+        failed++;
+        continue;
+      }
+
+      try {
+        logger.info(`⬆  ${label}: uploading...`);
+        const result = await api.uploadZip(localPath);
+        const datastageId = result?.id || result?.data?.id;
+        const folio = result?.folio || result?.data?.folio;
+        if (!datastageId) {
+          throw new Error(
+            'API returned no datastage id in response: ' +
+              JSON.stringify(result).slice(0, 300),
+          );
+        }
+        await api.markUploaded(row.id, { datastageId, folio });
+        logger.success(
+          `✓ ${label}: folio=${folio || 'n/a'} datastageId=${datastageId}`,
+        );
+        uploaded++;
+      } catch (err) {
+        logger.error(`✗ ${label}: ${err.message}`);
+        try {
+          await api.markFailed(row.id, err.message);
+        } catch (e) {
+          logger.error(`  mark-failed error: ${e.message}`);
+        }
+        failed++;
+      }
+    }
+
+    const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+    logger.info('—'.repeat(60));
+    logger.info(
+      `Done in ${elapsed}s — uploaded=${uploaded} failed=${failed} skipped=${alreadyUploaded}`,
+    );
+
+    if (options.showStats) {
+      const s = await api.getStats(sourceDirectory);
+      logger.info(`📊 Final stats: ${JSON.stringify(s)}`);
+    }
+
+    return { uploaded, failed, skipped: alreadyUploaded };
+  }
+}
+
+const datastageCommand = new DatastageCommand();
+export default datastageCommand;

package/src/commands/IdentifyCommand.js

@@ -10,9 +10,7 @@ import { ConfigurationError } from '../errors/ErrorTypes.js';
 import FileDetectionService from '../file-detection.js';
 
 /**
- * Paid pedimento detected_type values. `pedimento_completo_xml` is included
- * even though the XML matcher is currently disabled in the registry so that
- * re-enabling it requires no changes here.
+ * Paid pedimento detected_type values.
  */
 const DETECTED_PEDIMENTO_TYPES = new Set([
   'pedimento_simplificado',
@@ -74,29 +72,95 @@ export class IdentifyCommand {
       const scanConfig = appConfig.getScanConfig();
       const batchSize = parseInt(options.batchSize) || 100;
 
+      // Parse optional path prefix mapping: "O:/=/Volumes/nas/"
+      const pathPrefixMap = options.pathPrefix
+        ? this.#parsePathPrefix(options.pathPrefix)
+        : null;
+
       logger.info('🔍 Starting arela identify command');
       logger.info(`🎯 API Target: ${apiTarget}`);
       logger.info(`📦 Batch Size: ${batchSize}`);
+      if (options.table) logger.info(`📌 Target table: ${options.table}`);
+      if (options.resetAttempts)
+        logger.info('♻️  Reset detection attempts: ON');
+      if (pathPrefixMap)
+        logger.info(
+          `🗺  Path prefix map: ${pathPrefixMap.from} → ${pathPrefixMap.to}`,
+        );
 
-      // Fetch all tables for this instance
-      logger.info('\n📊 Fetching instance tables...');
-      const tables = await this.scanApiService.getInstanceTables(
-        scanConfig.companySlug,
-        scanConfig.serverId,
-        scanConfig.basePathFull,
-      );
+      // Resolve the list of tables to process
+      let tables;
+      if (options.fileId && options.table) {
+        // Single-file mode — identify exactly one file record
+        logger.info(
+          `\n🎯 Single-file mode: ${options.table} / ${options.fileId}`,
+        );
+        this.#reportProgress(0, `Fetching file record ${options.fileId}...`);
+        const record = await this.scanApiService.getFileRecord(
+          options.table,
+          options.fileId,
+        );
+        const results = await this.#detectFilesLocally(
+          [record],
+          1,
+          pathPrefixMap,
+        );
+        const updates = results.filter((r) => r !== null);
+        if (updates.length > 0) {
+          await this.scanApiService.batchUpdateDetection(
+            options.table,
+            updates,
+          );
+        }
+        this.#reportProgress(100, `Single-file identification complete`);
+        logger.success(`\n✅ Single-file identification complete`);
+        const firstUpdate = updates[0];
+        return {
+          processed: 1,
+          detected: updates.length,
+          proformas: 0,
+          errors: 1 - updates.length,
+          detectedType: firstUpdate?.detectedType ?? null,
+          detectedPedimento: firstUpdate?.detectedPedimento ?? null,
+          rfc: firstUpdate?.rfc ?? null,
+          arelaPath: firstUpdate?.arelaPath ?? null,
+        };
+      } else if (options.table) {
+        // Single-table mode — no need to match instance tables
+        tables = [{ tableName: options.table }];
+        logger.info(`\n📌 Single-table mode: ${options.table}`);
+      } else {
+        logger.info('\n📊 Fetching instance tables...');
+        tables = await this.scanApiService.getInstanceTables(
+          scanConfig.companySlug,
+          scanConfig.serverId,
+          scanConfig.basePathFull,
+        );
+
+        if (tables.length === 0) {
+          throw new ConfigurationError(
+            'No tables found for this instance. Run "arela scan" first.',
+          );
+        }
 
-      if (tables.length === 0) {
-        throw new ConfigurationError(
-          'No tables found for this instance. Run "arela scan" first.',
+        logger.info(
+          `📋 Found ${tables.length} table${tables.length === 1 ? '' : 's'} to process`,
         );
+        for (const table of tables) {
+          logger.info(`   - ${table.tableName}`);
+        }
       }
 
-      logger.info(
-        `📋 Found ${tables.length} table${tables.length === 1 ? '' : 's'} to process`,
-      );
-      for (const table of tables) {
-        logger.info(`   - ${table.tableName}`);
+      // Optionally reset detection attempts so previously-failed files are retried
+      if (options.resetAttempts) {
+        for (const table of tables) {
+          const { reset } = await this.scanApiService.resetDetectionAttempts(
+            table.tableName,
+          );
+          logger.info(
+            `♻️  Reset ${reset} detection attempt(s) in ${table.tableName}`,
+          );
+        }
       }
 
       // Process each table
@@ -127,6 +191,7 @@ export class IdentifyCommand {
           table.tableName,
           batchSize,
           startTime,
+          pathPrefixMap,
         );
 
         totalStats.processed += stats.processed;
@@ -183,7 +248,7 @@ export class IdentifyCommand {
    * @param {number} startTime - Start time for speed calculation
    * @returns {Promise<Object>} Processing statistics
    */
-  async #processTable(tableName, batchSize, startTime) {
+  async #processTable(tableName, batchSize, startTime, pathPrefixMap = null) {
     // Get detection statistics first (allTypes=true to count all supported file types)
     const initialStats = await this.scanApiService.getDetectionStats(
       tableName,
@@ -255,7 +320,11 @@ export class IdentifyCommand {
       const files = response.data;
 
       // Detect files locally with concurrent processing
-      const detectionResults = await this.#detectFilesLocally(files, 10);
+      const detectionResults = await this.#detectFilesLocally(
+        files,
+        10,
+        pathPrefixMap,
+      );
 
       // Batch update to API
       const updateResult = await this.scanApiService.batchUpdateDetection(
@@ -301,7 +370,7 @@ export class IdentifyCommand {
    * @param {number} concurrency - Maximum concurrent detections
    * @returns {Promise<Array>} Detection results
    */
-  async #detectFilesLocally(files, concurrency = 10) {
+  async #detectFilesLocally(files, concurrency = 10, pathPrefixMap = null) {
     const limit = pLimit(concurrency);
     const basePath = appConfig.getBasePath();
 
@@ -309,7 +378,17 @@ export class IdentifyCommand {
       limit(async () => {
         try {
           // Check if file exists on filesystem
-          const absolutePath = file.absolute_path;
+          let absolutePath = file.absolute_path;
+
+          // Apply cross-platform path prefix mapping if configured
+          if (
+            pathPrefixMap &&
+            absolutePath &&
+            absolutePath.startsWith(pathPrefixMap.from)
+          ) {
+            absolutePath =
+              pathPrefixMap.to + absolutePath.slice(pathPrefixMap.from.length);
+          }
 
           if (!fs.existsSync(absolutePath)) {
             return {
@@ -462,13 +541,15 @@ export class IdentifyCommand {
 
     // Check if the text contains any required pedimento marker. This must
     // stay aligned with the `match()` predicates in pedimento-simplificado.js
-    // and pedimento-completo.js.
+    // and pedimento-completo.js (which accept both "DE" and "DEL" in the
+    // title, and treat the colon after "T. OPER" as optional).
     const text = result.text || '';
-    const hasSimplificadoMarker = /FORMA SIMPLIFICADA DE PEDIMENTO/i.test(text);
+    const hasSimplificadoMarker =
+      /FORMA\s+SIMPLIFICADA\s+DEL?\s+PEDIMENTO/i.test(text);
     const hasCompletoMarkers =
       /NUM\.?\s*PEDIMENTO:/i.test(text) &&
       /CVE\.?\s*PEDIMENTO:/i.test(text) &&
-      /T\.?\s*OPER:/i.test(text);
+      /T\.?\s*OPER:?/i.test(text);
 
     return !hasSimplificadoMarker && !hasCompletoMarkers;
   }
@@ -543,6 +624,44 @@ export class IdentifyCommand {
       }
     }
   }
+
+  /**
+   * Parse a path prefix mapping string such as "O:/=/Volumes/nas/" into { from, to }.
+   * Supports both "FROM=TO" and "FROM:TO" separators.
+   * @private
+   * @param {string} mapping
+   * @returns {{ from: string, to: string }}
+   */
+  #parsePathPrefix(mapping) {
+    // Support either "FROM=TO" or "FROM:TO" as separator
+    const eqIdx = mapping.indexOf('=');
+    const colonIdx = mapping.indexOf(':');
+
+    let sep = -1;
+    // "O:/=/Volumes" — the colon inside "O:/" is part of a Windows drive letter; prefer '=' separator
+    if (eqIdx !== -1) {
+      sep = eqIdx;
+    } else if (colonIdx !== -1) {
+      sep = colonIdx;
+    }
+
+    if (sep === -1) {
+      throw new Error(
+        `Invalid --path-prefix format: "${mapping}". Expected "FROM=TO" e.g. "O:/=/Volumes/nas/"`,
+      );
+    }
+
+    const from = mapping.slice(0, sep);
+    const to = mapping.slice(sep + 1);
+
+    if (!from || !to) {
+      throw new Error(
+        `Invalid --path-prefix format: "${mapping}". Both FROM and TO parts must be non-empty.`,
+      );
+    }
+
+    return { from, to };
+  }
 }
 
 // Export singleton instance

package/src/commands/PollWorkerCommand.js

@@ -342,6 +342,8 @@ export class PollWorkerCommand {
       batchSize: 100,
       showStats: false,
       onProgress,
+      ...(job.fileId && { fileId: job.fileId }),
+      ...(job.table && { table: job.table }),
     };
 
     return identifyCommand.execute(options);

package/src/commands/ScanCommand.js

@@ -579,6 +579,9 @@ export class ScanCommand {
    * Normalize file record for database insertion
    * Stores paths with forward slashes for consistency but keeps them absolute
    * Sets likelySimplificado to true if file is a PDF and filename contains 'simp'
+   * Sets likelyInterAgencia to true if filename matches an inter-agency CFDI
+   * pattern (e.g. SICINGR*), so the API forces these XML/PDF through detection
+   * even though they lack the 'simp/pedim/covefact' heuristic.
    * @private
    */
   #normalizeFileRecord(filePath, fileStats, basePath, scanTimestamp) {
@@ -600,6 +603,17 @@ export class ScanCommand {
     const likelySimplificado =
       fileExtension === 'pdf' && /(simp|pedim|covefact)/i.test(fileName);
 
+    // Flag inter-agency CFDIs by filename so detection picks them up.
+    // Patterns are configurable via SCAN_INTER_AGENCIA_PATTERNS env var
+    // (see config.js). Only meaningful for PDF and XML.
+    let likelyInterAgencia = false;
+    if (fileExtension === 'pdf' || fileExtension === 'xml') {
+      const patterns = appConfig.scan.interAgenciaPatterns;
+      if (patterns && patterns.length > 0) {
+        likelyInterAgencia = patterns.some((re) => re.test(fileName));
+      }
+    }
+
     return {
       fileName,
       fileExtension,
@@ -610,6 +624,7 @@ export class ScanCommand {
       modifiedAt: fileStats.mtime.toISOString(),
       scanTimestamp,
       likelySimplificado,
+      likelyInterAgencia,
     };
   }
 

package/src/config/config.js

@@ -37,10 +37,10 @@ class Config {
       const __dirname = path.dirname(__filename);
       const packageJsonPath = path.resolve(__dirname, '../../package.json');
       const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
-      return packageJson.version || '1.0.21';
+      return packageJson.version || '1.0.23';
     } catch (error) {
       console.warn('⚠️ Could not read package.json version, using fallback');
-      return '1.0.21';
+      return '1.0.23';
     }
   }
 
@@ -294,6 +294,31 @@ class Config {
       .map((p) => p.trim())
       .filter(Boolean);
 
+    // Parse inter-agency CFDI filename patterns. Files whose basename matches
+    // any of these regex patterns are flagged at scan time (likelyInterAgencia)
+    // so the API forces them through detection and the factura_inter_agencia
+    // matcher can classify them. The push pipeline then excludes them (see
+    // NON_PUSHABLE_TYPES_SQL in arela-api). Comma-separated regex source list.
+    // Default: ^SICINGR — covers NORCOM's SICINGR70-NNNNNN(...).pdf/.XML files.
+    const defaultInterAgenciaPatterns = '^SICINGR';
+    const interAgenciaPatterns = (
+      process.env.SCAN_INTER_AGENCIA_PATTERNS || defaultInterAgenciaPatterns
+    )
+      .split(',')
+      .map((p) => p.trim())
+      .filter(Boolean)
+      .map((p) => {
+        try {
+          return new RegExp(p, 'i');
+        } catch (err) {
+          console.warn(
+            `⚠️ Invalid SCAN_INTER_AGENCIA_PATTERNS regex "${p}": ${err.message}`,
+          );
+          return null;
+        }
+      })
+      .filter(Boolean);
+
     // Generate table name if all components are available
     // Note: This is just for reference; actual table names are generated dynamically
     // in ScanCommand based on discovered directories and levels
@@ -312,6 +337,7 @@ class Config {
       basePathFull: basePathLabel, // Renamed for consistency
       tableName,
       excludePatterns,
+      interAgenciaPatterns,
       batchSize: parseInt(process.env.SCAN_BATCH_SIZE) || 2000,
       directoryLevel: parseInt(process.env.SCAN_DIRECTORY_LEVEL) || 0,
     };

package/src/document-type-shared.js

@@ -1,10 +1,10 @@
 // Import all document type definitions
 import { dodaPdfDefinition } from './document-types/doda-pdf.js';
 import { dodaXmlDefinition } from './document-types/doda-xml.js';
+import { facturaInterAgenciaDefinition } from './document-types/factura-inter-agencia.js';
 import { facturasComerciales } from './document-types/facturas-comerciales.js';
+import { pedimentoCompletoXmlDefinition } from './document-types/pedimento-completo-xml.js';
 import { pedimentoCompletoDefinition } from './document-types/pedimento-completo.js';
-// TODO: enable XML pedimento detection — implementation ready in pedimento-completo-xml.js
-// import { pedimentoCompletoXmlDefinition } from './document-types/pedimento-completo-xml.js';
 import { pedimentoSimplificadoDefinition } from './document-types/pedimento-simplificado.js';
 import { proformaDefinition } from './document-types/proforma.js';
 import { supportDocumentDefinition } from './document-types/support-document.js';
@@ -45,14 +45,14 @@ export class DocumentTypeDefinition {
 const documentTypes = [
   pedimentoSimplificadoDefinition,
   pedimentoCompletoDefinition,
-  // TODO: enable XML pedimento detection — uncomment the next line and the
-  // matching import at the top of this file. All downstream code
-  // (composeArelaPath, arela-api SQL filters, IdentifyCommand counters)
-  // already accepts `pedimento_completo_xml`.
-  // pedimentoCompletoXmlDefinition,
+  pedimentoCompletoXmlDefinition,
   supportDocumentDefinition,
   dodaPdfDefinition,
   dodaXmlDefinition,
+  // factura_inter_agencia MUST be evaluated BEFORE facturasComerciales
+  // because a NORCOM↔PALCO CFDI would also match the generic commercial
+  // invoice matcher. First match wins (see extractDocumentFields).
+  facturaInterAgenciaDefinition,
   facturasComerciales,
   // Add more document types here as needed
 ];
@@ -114,6 +114,14 @@ export function extractDocumentFields(source, fileExtension, filePath) {
         ? docType.extractPedimentoYear(source, fields, filePath)
         : null;
 
+      // Ensure downstream code (composeArelaPath) sees `numPedimento` as a
+      // field. PDF matchers add it via an explicit extractor; XML matchers
+      // compose it externally via extractNumPedimento. Backfill so both paths
+      // expose the same shape.
+      if (pedimento && !fields.some((f) => f.name === 'numPedimento')) {
+        fields.push(new FieldResult('numPedimento', true, pedimento));
+      }
+
       return [resolvedType, fields, pedimento, year];
     }
   }