@arela/uploader 1.0.6 → 1.0.7

package/.env.template

@@ -77,6 +77,11 @@ PUSH_UPLOAD_BATCH_SIZE=10
 # Examples: "archivos", "documents", "storage"
 PUSH_BUCKET=arela
 
+# Folder structure prefix for uploaded files (optional)
+# This prefix is prepended to the arela_path when uploading files
+# Examples: "agencia/cliente", "2024/docs", "imports/batch1"
+PUSH_FOLDER_STRUCTURE=
+
 # =============================================================================
 # PERFORMANCE OPTIMIZATION FOR MULTIPLE API REPLICAS
 # =============================================================================

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arela/uploader",
-  "version": "1.0.6",
+  "version": "1.0.7",
   "description": "CLI to upload files/directories to Arela",
   "bin": {
     "arela": "./src/index.js"

package/src/commands/IdentifyCommand.js

@@ -273,7 +273,7 @@ export class IdentifyCommand {
               arelaPath: null,
               detectionError:
                 'FILE_NOT_FOUND: File does not exist on filesystem. May have been moved or deleted after scan.',
-              isNotPedimento: false,
+              isPedimento: null,  // Unknown - can't determine
             };
           }
 
@@ -289,7 +289,7 @@ export class IdentifyCommand {
               rfc: null,
               arelaPath: null,
               detectionError: `FILE_TOO_LARGE: File size ${(stats.size / 1024 / 1024).toFixed(2)}MB exceeds ${maxSizeBytes / 1024 / 1024}MB limit.`,
-              isNotPedimento: false,
+              isPedimento: null,  // Unknown - can't determine
             };
           }
 
@@ -306,19 +306,19 @@ export class IdentifyCommand {
               rfc: result.rfc,
               arelaPath: result.arelaPath,
               detectionError: result.error,
-              isNotPedimento: false,
+              isPedimento: true,  // Confirmed pedimento
             };
           }
 
           // If no detection, determine if it's definitely not a pedimento
           // This helps avoid re-processing files we know aren't pedimentos
-          const isNotPedimento = this.#isDefinitelyNotPedimento(result, file);
+          const isDefinitelyNotPedimento = this.#isDefinitelyNotPedimento(result, file);
 
           // Build descriptive error message
           let detectionError = null;
           if (result.error) {
             detectionError = `DETECTION_ERROR: ${result.error}`;
-          } else if (isNotPedimento) {
+          } else if (isDefinitelyNotPedimento) {
             detectionError =
               'NOT_PEDIMENTO: File does not match pedimento-simplificado pattern. Missing key markers: "FORMA SIMPLIFICADA DE PEDIMENTO".';
           } else {
@@ -340,7 +340,7 @@ export class IdentifyCommand {
             rfc: result.rfc,
             arelaPath: result.arelaPath,
             detectionError,
-            isNotPedimento,
+            isPedimento: isDefinitelyNotPedimento ? false : null,  // false = not pedimento, null = unknown
           };
         } catch (error) {
           logger.warn(
@@ -367,7 +367,7 @@ export class IdentifyCommand {
             rfc: null,
             arelaPath: null,
             detectionError: `${errorCategory}: ${error.message}`,
-            isNotPedimento: false,
+            isPedimento: null,  // Unknown - error occurred
           };
         }
       }),

package/src/commands/PushCommand.js

@@ -39,6 +39,15 @@ export class PushCommand {
       const pushConfig = appConfig.getPushConfig();
       const tableName = scanConfig.tableName;
 
+      // Override folderStructure from command option if provided
+      if (options.folderStructure) {
+        // Clean the folder structure: remove leading/trailing slashes
+        pushConfig.folderStructure = options.folderStructure
+          .trim()
+          .replace(/^\/+/, '')
+          .replace(/\/+$/, '');
+      }
+
       // Set API target for scan/push operations
       const scanApiTarget = options.api || options.scanApi || 'default';
       const pushApiTarget = options.pushApi || scanApiTarget;
@@ -57,6 +66,9 @@ export class PushCommand {
       );
       console.log(`📦 Fetch Batch Size: ${options.batchSize}`);
       console.log(`📤 Upload Batch Size: ${options.uploadBatchSize}`);
+      if (pushConfig.folderStructure) {
+        console.log(`📁 Folder Structure Prefix: ${pushConfig.folderStructure}`);
+      }
 
       // Apply filters
       const filters = {
@@ -377,23 +389,29 @@ export class PushCommand {
       // Create form data for CLI upload endpoint
       const form = new FormData();
 
-      // Encode fileId and folderStructure in the filename
-      // Format: [fileId][folderStructure]filename
-      const folderStructure = file.arela_path.endsWith('/')
+      // Build folder structure: optionally prefix with PUSH_FOLDER_STRUCTURE env var
+      // arela_path already contains the logical path (RFC/Year/Patente/Aduana/Pedimento)
+      // folderStructure from config is the bucket prefix (e.g., "documents" or "uploads/2024")
+      let arelaPath = file.arela_path.endsWith('/')
         ? file.arela_path.slice(0, -1)
         : file.arela_path;
 
-      const encodedFilename = `[${file.id}][${folderStructure}]${file.file_name}`;
+      // Prepend folder structure prefix if configured
+      const folderStructure = pushConfig.folderStructure
+        ? `${pushConfig.folderStructure}/${arelaPath}`
+        : arelaPath;
 
-      // Create a read stream with the encoded filename
+      // Create a read stream with the original filename (no encoding needed)
       const fileStream = fs.createReadStream(file.absolute_path);
       form.append('files', fileStream, {
-        filename: encodedFilename,
+        filename: file.file_name,
         contentType: this.#getMimeType(file.file_extension),
       });
 
       // Add required fields for CLI upload
       form.append('tableName', tableName);
+      form.append('fileId', file.id);
+      form.append('folderStructure', folderStructure);
       form.append('rfc', file.rfc);
       form.append('bucket', pushConfig.bucket);
       form.append('autoDetect', 'true');

package/src/config/config.js

@@ -34,10 +34,10 @@ class Config {
       const __dirname = path.dirname(__filename);
       const packageJsonPath = path.resolve(__dirname, '../../package.json');
       const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
-      return packageJson.version || '1.0.6';
+      return packageJson.version || '1.0.7';
     } catch (error) {
       console.warn('⚠️ Could not read package.json version, using fallback');
-      return '1.0.6';
+      return '1.0.7';
     }
   }
 
@@ -313,6 +313,11 @@ class Config {
       .map((s) => parseInt(s.trim(), 10))
       .filter((y) => !isNaN(y));
 
+    // Clean folder structure: remove leading/trailing slashes
+    const folderStructure = process.env.PUSH_FOLDER_STRUCTURE?.trim()
+      .replace(/^\/+/, '')
+      .replace(/\/+$/, '');
+
     return {
       rfcs: pushRfcs,
       years: pushYears,
@@ -320,6 +325,7 @@ class Config {
       uploadBatchSize: parseInt(process.env.PUSH_UPLOAD_BATCH_SIZE) || 10,
       bucket:
         process.env.PUSH_BUCKET || process.env.SUPABASE_BUCKET || 'archivos',
+      folderStructure: folderStructure || '', // Prefix for storage path (e.g., "documents" or "uploads/2024")
     };
   }
 

package/src/index.js

@@ -399,6 +399,10 @@ class ArelaUploaderCLI {
         '--years <years>',
         'Comma-separated years to filter (overrides PUSH_YEARS env var)',
       )
+      .option(
+        '--folder-structure <path>',
+        'Storage path prefix (overrides PUSH_FOLDER_STRUCTURE env var)',
+      )
       .option('--show-stats', 'Show performance statistics')
       .action(async (options) => {
         try {

package/tests/commands/IdentifyCommand.test.js

@@ -539,7 +539,7 @@ describe('IdentifyCommand', () => {
         expect.any(String),
         expect.arrayContaining([
           expect.objectContaining({
-            isNotPedimento: true,
+            isPedimento: false,  // Confirmed NOT a pedimento
           }),
         ])
       );
@@ -561,7 +561,7 @@ describe('IdentifyCommand', () => {
         expect.any(String),
         expect.arrayContaining([
           expect.objectContaining({
-            isNotPedimento: false,
+            isPedimento: null,  // Unknown - might be pedimento but missing fields
           }),
         ])
       );

package/.env.local

@@ -1,316 +0,0 @@
-# ============================================
-# ARELA UPLOADER CONFIGURATION
-# ============================================
-
-# Localhost Arela API Configuration
-ARELA_API_URL=http://localhost:3010
-ARELA_API_TOKEN=555f1d5c1b5020a132002a6fa201e0074e1b057895776bd33619db0cd26b259b
-
-# Localhost Supabase Configuration
-SUPABASE_URL=http://127.0.0.1:54321
-SUPABASE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU
-SUPABASE_BUCKET=arela
-
-# ARELA_API_TARGET=default
-
-# Localhost Upload Configuration
-UPLOAD_BASE_PATH=./sample
-UPLOAD_SOURCES=2023|2024
-UPLOAD_RFCS=DTM090831LF0|AKS151005E46|IMS030409FZ0|RDG1107154L7|SHP031226BV2|CSM9301219B4|LIN960124HT8|LME971009SW4|AKM9707151B6|FEL000822AG2|FDM060802J54|MTM9807279B4|AUM9207011CA|MMJ0810145N1|ACC010328EQ6|PED781129JT6|CAD890407NK7|SME140411IK7|JME1903121C2|EIJ110429NF9|PTJ080414TM6|TME050503BM4
-# UPLOAD_RFCS=FDM060802J54|CSM9301219B4
-UPLOAD_YEARS=2023|2024|2025
-
-# =============================================================================
-# SCAN CONFIGURATION (for arela scan command)
-# =============================================================================
-
-# Company identifier for this CLI instance (required)
-# Use a short, descriptive slug for your company/agency/client
-# Examples: "acme_corp", "cliente_123", "agencia_xyz"
-ARELA_COMPANY_SLUG=palco
-
-# Server identifier (required)
-# Use a unique ID for each server/NAS where arela-cli is installed
-# Examples: "nas01", "server-mx", "storage-01"
-ARELA_SERVER_ID=local
-
-# Base path label (optional, auto-derived from UPLOAD_BASE_PATH if not set)
-# Short label describing the base path being scanned
-# Examples: "data", "documents", "archive"
-ARELA_BASE_PATH_LABEL=
-
-# System file patterns to exclude from scan (comma-separated)
-# These files will be filtered before uploading stats to reduce payload
-SCAN_EXCLUDE_PATTERNS=.DS_Store,Thumbs.db,desktop.ini,__pycache__,.pyc,.tmp,.swp,$RECYCLE.BIN,System Volume Information,~$*
-
-# Batch size for scan operations (default: 2000 records per API call)
-SCAN_BATCH_SIZE=2000
-
-# Directory depth level for creating separate tables (default: 0)
-# 0 = single table for entire base path
-# 1 = one table per first-level subdirectory
-# 2 = one table per second-level subdirectory, etc.
-# Example: with level=1 and base=/data, creates tables for /data/folder1, /data/folder2, etc.
-SCAN_DIRECTORY_LEVEL=0
-
-# =============================================================================
-# PUSH CONFIGURATION (for arela push command)
-# =============================================================================
-
-# Filter files to upload by RFC (pipe-separated, optional)
-# If not set, all files with arela_path will be uploaded
-# Examples: "RFC123456ABC|RFC789012DEF"
-PUSH_RFCS=
-
-# Filter files to upload by year (pipe-separated, optional)
-# If not set, all files with arela_path will be uploaded
-# Examples: "2023|2024|2025"
-PUSH_YEARS=
-
-# Batch size for fetching files from database (default: 100)
-PUSH_BATCH_SIZE=100
-
-# Concurrent upload batch size (default: 10)
-# Number of files to upload simultaneously
-PUSH_UPLOAD_BATCH_SIZE=10
-
-# Storage bucket for uploaded files (optional, defaults to SUPABASE_BUCKET)
-# Examples: "archivos", "documents", "storage"
-PUSH_BUCKET=cli
-
-
-# =============================================================================
-# PERFORMANCE OPTIMIZATION FOR MULTIPLE API REPLICAS
-# =============================================================================
-
-# API Connection Configuration
-# Set this to match your number of API replicas (e.g., if you have 10 API instances, set to 10)
-MAX_API_CONNECTIONS=10
-
-# API Connection Timeout (milliseconds)
-API_CONNECTION_TIMEOUT=60000
-
-# Batch Processing Configuration
-# Files processed concurrently per batch (should be >= MAX_API_CONNECTIONS for best performance)
-BATCH_SIZE=100
-
-# Delay between batches (0 for maximum speed)
-BATCH_DELAY=0
-
-# Source Processing Concurrency
-# Number of upload sources/folders to process simultaneously
-MAX_CONCURRENT_SOURCES=2
-
-# API Retry Configuration
-# Maximum number of retry attempts for failed API requests
-API_MAX_RETRIES=3
-
-# Enable exponential backoff for retries (true/false)
-# When true, retry delays increase: 1s, 2s, 4s, 8s, 16s
-# When false, uses fixed delay (API_RETRY_DELAY)
-API_RETRY_EXPONENTIAL_BACKOFF=true
-
-# Fixed retry delay in milliseconds (only used if exponential backoff is disabled)
-API_RETRY_DELAY=1000
-
-# =============================================================================
-# EXAMPLE CONFIGURATIONS FOR DIFFERENT SCENARIOS
-# =============================================================================
-
-# For 10 API Replicas (High Performance Setup):
-# MAX_API_CONNECTIONS=10
-# BATCH_SIZE=100
-# MAX_CONCURRENT_SOURCES=3
-# BATCH_DELAY=0
-
-# For 5 API Replicas (Medium Performance Setup):
-# MAX_API_CONNECTIONS=5
-# BATCH_SIZE=50
-# MAX_CONCURRENT_SOURCES=2
-# BATCH_DELAY=0
-
-# For 1 API Instance (Single Instance Setup):
-# MAX_API_CONNECTIONS=5
-# BATCH_SIZE=20
-# MAX_CONCURRENT_SOURCES=1
-# BATCH_DELAY=100
-
-# =============================================================================
-# LOGGING AND MONITORING
-# =============================================================================
-
-# Progress bar update frequency
-PROGRESS_UPDATE_INTERVAL=10
-
-# Enable verbose logging (true/false)
-VERBOSE_LOGGING=false
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-# ============================================
-
-# # Cloud Service Arela API Configuration
-# # ARELA_API_URL=https://api.aws.arela.com.mx
-# # ARELA_API_TOKEN=6bd75c5b3699ecf19e6726c10ae88ae0528f0b72d6c10f8b284f92563d3822a7
-
-# # # Cloud Service Supabase Configuration
-# SUPABASE_URL=https://qlospyfsbwvkskivmsgq.supabase.co
-# SUPABASE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InFsb3NweWZzYnd2a3NraXZtc2dxIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NTU5MDg2NjUsImV4cCI6MjA3MTQ4NDY2NX0.BrqcCLxTmpU6Swl7h3gam6TeW4jVf4WssMbRm0sH7l4
-# SUPABASE_BUCKET=zips
-
-# # # Cloud Service Upload Configuration
-# UPLOAD_BASE_PATH=./sample
-# UPLOAD_SOURCES=zips
-# # UPLOAD_RFCS=AKS151005E46|IMS030409FZ0|RDG1107154L7|SHP031226BV2|CSM9301219B4|LIN960124HT8|LME971009SW4|AKM9707151B6|FEL000822AG2|FDM060802J54|MTM9807279B4|AUM9207011CA|MMJ0810145N1|ACC010328EQ6|PED781129JT6|CAD890407NK7|SME140411IK7|JME1903121C2|EIJ110429NF9|PTJ080414TM6|TME050503BM4
-# UPLOAD_RFCS=KTJ931117P55|AUM9207011CA
-# UPLOAD_YEARS=2023|
-
-# # =============================================================================
-# # PERFORMANCE OPTIMIZATION FOR MULTIPLE API REPLICAS
-# # =============================================================================
-
-# # API Connection Configuration
-# # Set this to match your number of API replicas (e.g., if you have 10 API instances, set to 10)
-# MAX_API_CONNECTIONS=10
-
-# # API Connection Timeout (milliseconds)
-# API_CONNECTION_TIMEOUT=60000
-
-# # Batch Processing Configuration
-# # Files processed concurrently per batch (should be >= MAX_API_CONNECTIONS for best performance)
-# BATCH_SIZE=100
-
-# # Delay between batches (0 for maximum speed)
-# BATCH_DELAY=0
-
-# # Source Processing Concurrency
-# # Number of upload sources/folders to process simultaneously
-# MAX_CONCURRENT_SOURCES=2
-
-# # =============================================================================
-# # EXAMPLE CONFIGURATIONS FOR DIFFERENT SCENARIOS
-# # =============================================================================
-
-# # For 10 API Replicas (High Performance Setup):
-# # MAX_API_CONNECTIONS=10
-# # BATCH_SIZE=100
-# # MAX_CONCURRENT_SOURCES=3
-# # BATCH_DELAY=0
-
-# # For 5 API Replicas (Medium Performance Setup):
-# # MAX_API_CONNECTIONS=5
-# # BATCH_SIZE=50
-# # MAX_CONCURRENT_SOURCES=2
-# # BATCH_DELAY=0
-
-# # For 1 API Instance (Single Instance Setup):
-# # MAX_API_CONNECTIONS=5
-# # BATCH_SIZE=20
-# # MAX_CONCURRENT_SOURCES=1
-# # BATCH_DELAY=100
-
-# # =============================================================================
-# # LOGGING AND MONITORING
-# # =============================================================================
-
-# # Progress bar update frequency
-# PROGRESS_UPDATE_INTERVAL=10
-
-# # Enable verbose logging (true/false)
-# VERBOSE_LOGGING=false
-
-# # ============================================
-
-
-
-# =============================================================================
-# WATCH MODE CONFIGURATION
-# =============================================================================
-
-# Habilitar watch mode (true/false)
-WATCH_ENABLED=true
-
-# Configuración de directorios a observar (formato JSON)
-# Cada directorio puede tener su propia folderStructure para organizar en el bucket
-# Formato: {"ruta/directorio1":"estructura-1","ruta/directorio2":"estructura-2"}
-WATCH_DIRECTORY_CONFIGS={"./sample/watcher":"prueba-watcher"}
-
-# Estrategia de upload (opciones: individual|batch|full-structure)
-# - individual: Sube solo el archivo modificado más reciente
-# - batch: Sube un lote de N archivos recientes
-# - full-structure: Sube la estructura completa de carpetas
-WATCH_STRATEGY=batch
-
-# Debouncing en milisegundos (esperar entre eventos antes de procesar)
-WATCH_DEBOUNCE_MS=1000
-
-# Tamaño de batch para strategy batch
-WATCH_BATCH_SIZE=10
-
-# Usar polling en lugar de eventos nativos del filesystem
-# Útil para sistemas de archivos remotos o NFS
-WATCH_USE_POLLING=false
-
-# Interval de polling en milisegundos (solo si WATCH_USE_POLLING=true)
-WATCH_POLL_INTERVAL=100
-
-# Umbral de estabilidad en ms (esperar a que el archivo deje de cambiar)
-WATCH_STABILITY_THRESHOLD=300
-
-# Patrones a ignorar (separados por coma, se usan como regex)
-WATCH_IGNORE_PATTERNS=.tmp,.bak,*.swp
-
-# Detección automática de tipos de documento
-WATCH_AUTO_DETECT=true
-
-# Organización automática de archivos
-WATCH_AUTO_ORGANIZE=true
-
-# =============================================================================
-# WATCH MODE - AUTOMATIC PROCESSING PIPELINE
-# =============================================================================
-# 
-# El pipeline automático ejecuta la siguiente secuencia cuando se detecta un archivo nuevo:
-# 1. Stats Collection   → stats --stats-only (recopila información del archivo)
-# 2. PDF Detection      → detect --detect-pdfs (identifica pedimentos simplificados)
-# 3. Path Propagation   → detect --propagate-arela-path (propaga a documentos relacionados)
-# 4. RFC Upload         → upload --upload-by-rfc --folder-structure (sube con estructura)
-#
-# El pipeline se habilita automáticamente en watch mode y usa la folderStructure
-# definida para cada WATCH_DIRECTORY_CONFIGS
-#
-# Para deshabilitar en CLI, usa: arela watch --no-auto-processing
-
-# =============================================================================
-# LOGGING AND MONITORING
-# =============================================================================
-
-# Progress bar update frequency
-PROGRESS_UPDATE_INTERVAL=10
-
-# Enable verbose logging (true/false)
-VERBOSE_LOGGING=false
-
-# ============================================