@arela/uploader 1.0.2 → 1.0.4

package/.env.local

@@ -0,0 +1,316 @@
+# ============================================
+# ARELA UPLOADER CONFIGURATION
+# ============================================
+
+# Localhost Arela API Configuration
+ARELA_API_URL=http://localhost:3010
+ARELA_API_TOKEN=555f1d5c1b5020a132002a6fa201e0074e1b057895776bd33619db0cd26b259b
+
+# Localhost Supabase Configuration
+SUPABASE_URL=http://127.0.0.1:54321
+SUPABASE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU
+SUPABASE_BUCKET=arela
+
+# ARELA_API_TARGET=default
+
+# Localhost Upload Configuration
+UPLOAD_BASE_PATH=./sample
+UPLOAD_SOURCES=2023|2024
+UPLOAD_RFCS=DTM090831LF0|AKS151005E46|IMS030409FZ0|RDG1107154L7|SHP031226BV2|CSM9301219B4|LIN960124HT8|LME971009SW4|AKM9707151B6|FEL000822AG2|FDM060802J54|MTM9807279B4|AUM9207011CA|MMJ0810145N1|ACC010328EQ6|PED781129JT6|CAD890407NK7|SME140411IK7|JME1903121C2|EIJ110429NF9|PTJ080414TM6|TME050503BM4
+# UPLOAD_RFCS=FDM060802J54|CSM9301219B4
+UPLOAD_YEARS=2023|2024|2025
+
+# =============================================================================
+# SCAN CONFIGURATION (for arela scan command)
+# =============================================================================
+
+# Company identifier for this CLI instance (required)
+# Use a short, descriptive slug for your company/agency/client
+# Examples: "acme_corp", "cliente_123", "agencia_xyz"
+ARELA_COMPANY_SLUG=palco
+
+# Server identifier (required)
+# Use a unique ID for each server/NAS where arela-cli is installed
+# Examples: "nas01", "server-mx", "storage-01"
+ARELA_SERVER_ID=local
+
+# Base path label (optional, auto-derived from UPLOAD_BASE_PATH if not set)
+# Short label describing the base path being scanned
+# Examples: "data", "documents", "archive"
+ARELA_BASE_PATH_LABEL=
+
+# System file patterns to exclude from scan (comma-separated)
+# These files will be filtered before uploading stats to reduce payload
+SCAN_EXCLUDE_PATTERNS=.DS_Store,Thumbs.db,desktop.ini,__pycache__,.pyc,.tmp,.swp,$RECYCLE.BIN,System Volume Information,~$*
+
+# Batch size for scan operations (default: 2000 records per API call)
+SCAN_BATCH_SIZE=2000
+
+# Directory depth level for creating separate tables (default: 0)
+# 0 = single table for entire base path
+# 1 = one table per first-level subdirectory
+# 2 = one table per second-level subdirectory, etc.
+# Example: with level=1 and base=/data, creates tables for /data/folder1, /data/folder2, etc.
+SCAN_DIRECTORY_LEVEL=0
+
+# =============================================================================
+# PUSH CONFIGURATION (for arela push command)
+# =============================================================================
+
+# Filter files to upload by RFC (pipe-separated, optional)
+# If not set, all files with arela_path will be uploaded
+# Examples: "RFC123456ABC|RFC789012DEF"
+PUSH_RFCS=
+
+# Filter files to upload by year (pipe-separated, optional)
+# If not set, all files with arela_path will be uploaded
+# Examples: "2023|2024|2025"
+PUSH_YEARS=
+
+# Batch size for fetching files from database (default: 100)
+PUSH_BATCH_SIZE=100
+
+# Concurrent upload batch size (default: 10)
+# Number of files to upload simultaneously
+PUSH_UPLOAD_BATCH_SIZE=10
+
+# Storage bucket for uploaded files (optional, defaults to SUPABASE_BUCKET)
+# Examples: "archivos", "documents", "storage"
+PUSH_BUCKET=cli
+
+
+# =============================================================================
+# PERFORMANCE OPTIMIZATION FOR MULTIPLE API REPLICAS
+# =============================================================================
+
+# API Connection Configuration
+# Set this to match your number of API replicas (e.g., if you have 10 API instances, set to 10)
+MAX_API_CONNECTIONS=10
+
+# API Connection Timeout (milliseconds)
+API_CONNECTION_TIMEOUT=60000
+
+# Batch Processing Configuration
+# Files processed concurrently per batch (should be >= MAX_API_CONNECTIONS for best performance)
+BATCH_SIZE=100
+
+# Delay between batches (0 for maximum speed)
+BATCH_DELAY=0
+
+# Source Processing Concurrency
+# Number of upload sources/folders to process simultaneously
+MAX_CONCURRENT_SOURCES=2
+
+# API Retry Configuration
+# Maximum number of retry attempts for failed API requests
+API_MAX_RETRIES=3
+
+# Enable exponential backoff for retries (true/false)
+# When true, retry delays increase: 1s, 2s, 4s, 8s, 16s
+# When false, uses fixed delay (API_RETRY_DELAY)
+API_RETRY_EXPONENTIAL_BACKOFF=true
+
+# Fixed retry delay in milliseconds (only used if exponential backoff is disabled)
+API_RETRY_DELAY=1000
+
+# =============================================================================
+# EXAMPLE CONFIGURATIONS FOR DIFFERENT SCENARIOS
+# =============================================================================
+
+# For 10 API Replicas (High Performance Setup):
+# MAX_API_CONNECTIONS=10
+# BATCH_SIZE=100
+# MAX_CONCURRENT_SOURCES=3
+# BATCH_DELAY=0
+
+# For 5 API Replicas (Medium Performance Setup):
+# MAX_API_CONNECTIONS=5
+# BATCH_SIZE=50
+# MAX_CONCURRENT_SOURCES=2
+# BATCH_DELAY=0
+
+# For 1 API Instance (Single Instance Setup):
+# MAX_API_CONNECTIONS=5
+# BATCH_SIZE=20
+# MAX_CONCURRENT_SOURCES=1
+# BATCH_DELAY=100
+
+# =============================================================================
+# LOGGING AND MONITORING
+# =============================================================================
+
+# Progress bar update frequency
+PROGRESS_UPDATE_INTERVAL=10
+
+# Enable verbose logging (true/false)
+VERBOSE_LOGGING=false
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# ============================================
+
+# # Cloud Service Arela API Configuration
+# # ARELA_API_URL=https://api.aws.arela.com.mx
+# # ARELA_API_TOKEN=6bd75c5b3699ecf19e6726c10ae88ae0528f0b72d6c10f8b284f92563d3822a7
+
+# # # Cloud Service Supabase Configuration
+# SUPABASE_URL=https://qlospyfsbwvkskivmsgq.supabase.co
+# SUPABASE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InFsb3NweWZzYnd2a3NraXZtc2dxIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NTU5MDg2NjUsImV4cCI6MjA3MTQ4NDY2NX0.BrqcCLxTmpU6Swl7h3gam6TeW4jVf4WssMbRm0sH7l4
+# SUPABASE_BUCKET=zips
+
+# # # Cloud Service Upload Configuration
+# UPLOAD_BASE_PATH=./sample
+# UPLOAD_SOURCES=zips
+# # UPLOAD_RFCS=AKS151005E46|IMS030409FZ0|RDG1107154L7|SHP031226BV2|CSM9301219B4|LIN960124HT8|LME971009SW4|AKM9707151B6|FEL000822AG2|FDM060802J54|MTM9807279B4|AUM9207011CA|MMJ0810145N1|ACC010328EQ6|PED781129JT6|CAD890407NK7|SME140411IK7|JME1903121C2|EIJ110429NF9|PTJ080414TM6|TME050503BM4
+# UPLOAD_RFCS=KTJ931117P55|AUM9207011CA
+# UPLOAD_YEARS=2023|
+
+# # =============================================================================
+# # PERFORMANCE OPTIMIZATION FOR MULTIPLE API REPLICAS
+# # =============================================================================
+
+# # API Connection Configuration
+# # Set this to match your number of API replicas (e.g., if you have 10 API instances, set to 10)
+# MAX_API_CONNECTIONS=10
+
+# # API Connection Timeout (milliseconds)
+# API_CONNECTION_TIMEOUT=60000
+
+# # Batch Processing Configuration
+# # Files processed concurrently per batch (should be >= MAX_API_CONNECTIONS for best performance)
+# BATCH_SIZE=100
+
+# # Delay between batches (0 for maximum speed)
+# BATCH_DELAY=0
+
+# # Source Processing Concurrency
+# # Number of upload sources/folders to process simultaneously
+# MAX_CONCURRENT_SOURCES=2
+
+# # =============================================================================
+# # EXAMPLE CONFIGURATIONS FOR DIFFERENT SCENARIOS
+# # =============================================================================
+
+# # For 10 API Replicas (High Performance Setup):
+# # MAX_API_CONNECTIONS=10
+# # BATCH_SIZE=100
+# # MAX_CONCURRENT_SOURCES=3
+# # BATCH_DELAY=0
+
+# # For 5 API Replicas (Medium Performance Setup):
+# # MAX_API_CONNECTIONS=5
+# # BATCH_SIZE=50
+# # MAX_CONCURRENT_SOURCES=2
+# # BATCH_DELAY=0
+
+# # For 1 API Instance (Single Instance Setup):
+# # MAX_API_CONNECTIONS=5
+# # BATCH_SIZE=20
+# # MAX_CONCURRENT_SOURCES=1
+# # BATCH_DELAY=100
+
+# # =============================================================================
+# # LOGGING AND MONITORING
+# # =============================================================================
+
+# # Progress bar update frequency
+# PROGRESS_UPDATE_INTERVAL=10
+
+# # Enable verbose logging (true/false)
+# VERBOSE_LOGGING=false
+
+# # ============================================
+
+
+
+# =============================================================================
+# WATCH MODE CONFIGURATION
+# =============================================================================
+
+# Habilitar watch mode (true/false)
+WATCH_ENABLED=true
+
+# Configuración de directorios a observar (formato JSON)
+# Cada directorio puede tener su propia folderStructure para organizar en el bucket
+# Formato: {"ruta/directorio1":"estructura-1","ruta/directorio2":"estructura-2"}
+WATCH_DIRECTORY_CONFIGS={"./sample/watcher":"prueba-watcher"}
+
+# Estrategia de upload (opciones: individual|batch|full-structure)
+# - individual: Sube solo el archivo modificado más reciente
+# - batch: Sube un lote de N archivos recientes
+# - full-structure: Sube la estructura completa de carpetas
+WATCH_STRATEGY=batch
+
+# Debouncing en milisegundos (esperar entre eventos antes de procesar)
+WATCH_DEBOUNCE_MS=1000
+
+# Tamaño de batch para strategy batch
+WATCH_BATCH_SIZE=10
+
+# Usar polling en lugar de eventos nativos del filesystem
+# Útil para sistemas de archivos remotos o NFS
+WATCH_USE_POLLING=false
+
+# Interval de polling en milisegundos (solo si WATCH_USE_POLLING=true)
+WATCH_POLL_INTERVAL=100
+
+# Umbral de estabilidad en ms (esperar a que el archivo deje de cambiar)
+WATCH_STABILITY_THRESHOLD=300
+
+# Patrones a ignorar (separados por coma, se usan como regex)
+WATCH_IGNORE_PATTERNS=.tmp,.bak,*.swp
+
+# Detección automática de tipos de documento
+WATCH_AUTO_DETECT=true
+
+# Organización automática de archivos
+WATCH_AUTO_ORGANIZE=true
+
+# =============================================================================
+# WATCH MODE - AUTOMATIC PROCESSING PIPELINE
+# =============================================================================
+# 
+# El pipeline automático ejecuta la siguiente secuencia cuando se detecta un archivo nuevo:
+# 1. Stats Collection   → stats --stats-only (recopila información del archivo)
+# 2. PDF Detection      → detect --detect-pdfs (identifica pedimentos simplificados)
+# 3. Path Propagation   → detect --propagate-arela-path (propaga a documentos relacionados)
+# 4. RFC Upload         → upload --upload-by-rfc --folder-structure (sube con estructura)
+#
+# El pipeline se habilita automáticamente en watch mode y usa la folderStructure
+# definida para cada WATCH_DIRECTORY_CONFIGS
+#
+# Para deshabilitar en CLI, usa: arela watch --no-auto-processing
+
+# =============================================================================
+# LOGGING AND MONITORING
+# =============================================================================
+
+# Progress bar update frequency
+PROGRESS_UPDATE_INTERVAL=10
+
+# Enable verbose logging (true/false)
+VERBOSE_LOGGING=false
+
+# ============================================

package/.env.template

@@ -19,6 +19,64 @@ UPLOAD_BASE_PATH=/path/to/your/upload/base
 UPLOAD_SOURCES=folder1|folder2|folder3
 UPLOAD_RFCS=rfc1|rfc2|rfc3
 
+# =============================================================================
+# SCAN CONFIGURATION (for arela scan command)
+# =============================================================================
+
+# Company identifier for this CLI instance (required)
+# Use a short, descriptive slug for your company/agency/client
+# Examples: "acme_corp", "cliente_123", "agencia_xyz"
+ARELA_COMPANY_SLUG=
+
+# Server identifier (required)
+# Use a unique ID for each server/NAS where arela-cli is installed
+# Examples: "nas01", "server-mx", "storage-01"
+ARELA_SERVER_ID=
+
+# Base path label (optional, auto-derived from UPLOAD_BASE_PATH if not set)
+# Short label describing the base path being scanned
+# Examples: "data", "documents", "archive"
+ARELA_BASE_PATH_LABEL=
+
+# System file patterns to exclude from scan (comma-separated)
+# These files will be filtered before uploading stats to reduce payload
+SCAN_EXCLUDE_PATTERNS=.DS_Store,Thumbs.db,desktop.ini,__pycache__,.pyc,.tmp,.swp,$RECYCLE.BIN,System Volume Information,~$*
+
+# Batch size for scan operations (default: 2000 records per API call)
+SCAN_BATCH_SIZE=2000
+
+# Directory depth level for creating separate tables (default: 0)
+# 0 = single table for entire base path
+# 1 = one table per first-level subdirectory
+# 2 = one table per second-level subdirectory, etc.
+# Example: with level=1 and base=/data, creates tables for /data/folder1, /data/folder2, etc.
+SCAN_DIRECTORY_LEVEL=0
+
+# =============================================================================
+# PUSH CONFIGURATION (for arela push command)
+# =============================================================================
+
+# Filter files to upload by RFC (pipe-separated, optional)
+# If not set, all files with arela_path will be uploaded
+# Examples: "RFC123456ABC|RFC789012DEF"
+PUSH_RFCS=
+
+# Filter files to upload by year (pipe-separated, optional)
+# If not set, all files with arela_path will be uploaded
+# Examples: "2023|2024|2025"
+PUSH_YEARS=
+
+# Batch size for fetching files from database (default: 100)
+PUSH_BATCH_SIZE=100
+
+# Concurrent upload batch size (default: 10)
+# Number of files to upload simultaneously
+PUSH_UPLOAD_BATCH_SIZE=10
+
+# Storage bucket for uploaded files (optional, defaults to SUPABASE_BUCKET)
+# Examples: "archivos", "documents", "storage"
+PUSH_BUCKET=arela
+
 # =============================================================================
 # PERFORMANCE OPTIMIZATION FOR MULTIPLE API REPLICAS
 # =============================================================================
@@ -30,6 +88,18 @@ MAX_API_CONNECTIONS=10
 # API Connection Timeout (milliseconds)
 API_CONNECTION_TIMEOUT=60000
 
+# API Retry Configuration
+# Maximum number of retry attempts for failed API requests
+API_MAX_RETRIES=3
+
+# Enable exponential backoff for retries (true/false)
+# When true, retry delays increase: 1s, 2s, 4s, 8s, 16s
+# When false, uses fixed delay (API_RETRY_DELAY)
+API_RETRY_EXPONENTIAL_BACKOFF=true
+
+# Fixed retry delay in milliseconds (only used if exponential backoff is disabled)
+API_RETRY_DELAY=1000
+
 # Batch Processing Configuration
 # Files processed concurrently per batch (should be >= MAX_API_CONNECTIONS for best performance)
 BATCH_SIZE=100