@arela/uploader 1.1.3 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/worktrees/agent-multi-profile/.env.template +224 -0
- package/.claude/worktrees/agent-multi-profile/.prettierrc +13 -0
- package/.claude/worktrees/agent-multi-profile/README.md +405 -0
- package/.claude/worktrees/agent-multi-profile/package-lock.json +7096 -0
- package/.claude/worktrees/agent-multi-profile/package.json +78 -0
- package/.claude/worktrees/agent-multi-profile/scripts/cleanup-ds-store.js +109 -0
- package/.claude/worktrees/agent-multi-profile/scripts/cleanup-system-files.js +69 -0
- package/.claude/worktrees/agent-multi-profile/scripts/scoring-compare.js +243 -0
- package/.claude/worktrees/agent-multi-profile/scripts/scoring-phase4-check.js +96 -0
- package/.claude/worktrees/agent-multi-profile/scripts/tests/phase-7-features.test.js +415 -0
- package/.claude/worktrees/agent-multi-profile/scripts/tests/signal-handling.test.js +275 -0
- package/.claude/worktrees/agent-multi-profile/scripts/tests/smart-watch-integration.test.js +554 -0
- package/.claude/worktrees/agent-multi-profile/scripts/tests/watch-service-integration.test.js +584 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/AgentCommand.js +229 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/AgentInitCommand.js +316 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/DatastageCommand.js +164 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/GDriveSyncCommand.js +475 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/IdentifyCommand.js +708 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/PollWorkerCommand.js +169 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/PropagateCommand.js +636 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/PushCommand.js +743 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/ScanCommand.js +722 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/UploadCommand.js +587 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/WatchCommand.js +1342 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/WorkerCommand.js +337 -0
- package/.claude/worktrees/agent-multi-profile/src/config/config.js +862 -0
- package/.claude/worktrees/agent-multi-profile/src/document-type-shared.js +131 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/_pedimento-shared-extractors.js +348 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/doda-pdf.js +121 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/doda-xml.js +118 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/factura-inter-agencia.js +186 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/facturas-comerciales.js +233 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/pedimento-completo-xml.js +372 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/pedimento-completo.js +108 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/pedimento-simplificado.js +76 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/proforma.js +29 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/support-document.js +200 -0
- package/.claude/worktrees/agent-multi-profile/src/errors/ErrorHandler.js +278 -0
- package/.claude/worktrees/agent-multi-profile/src/errors/ErrorTypes.js +104 -0
- package/.claude/worktrees/agent-multi-profile/src/file-detection.js +338 -0
- package/.claude/worktrees/agent-multi-profile/src/index.js +890 -0
- package/.claude/worktrees/agent-multi-profile/src/scoring/db-matcher-adapter.js +98 -0
- package/.claude/worktrees/agent-multi-profile/src/scoring/matchers-seed.js +386 -0
- package/.claude/worktrees/agent-multi-profile/src/scoring/scoring-engine.js +251 -0
- package/.claude/worktrees/agent-multi-profile/src/services/AdvancedFilterService.js +505 -0
- package/.claude/worktrees/agent-multi-profile/src/services/AutoProcessingService.js +749 -0
- package/.claude/worktrees/agent-multi-profile/src/services/BenchmarkingService.js +381 -0
- package/.claude/worktrees/agent-multi-profile/src/services/DatabaseService.js +2173 -0
- package/.claude/worktrees/agent-multi-profile/src/services/DatastageApiService.js +240 -0
- package/.claude/worktrees/agent-multi-profile/src/services/ErrorMonitor.js +275 -0
- package/.claude/worktrees/agent-multi-profile/src/services/GoogleDriveService.js +217 -0
- package/.claude/worktrees/agent-multi-profile/src/services/LoggingService.js +649 -0
- package/.claude/worktrees/agent-multi-profile/src/services/MonitoringService.js +401 -0
- package/.claude/worktrees/agent-multi-profile/src/services/PerformanceOptimizer.js +511 -0
- package/.claude/worktrees/agent-multi-profile/src/services/PipelineApiService.js +274 -0
- package/.claude/worktrees/agent-multi-profile/src/services/PipelineJobRunner.js +389 -0
- package/.claude/worktrees/agent-multi-profile/src/services/ProfileManager.js +164 -0
- package/.claude/worktrees/agent-multi-profile/src/services/ReportingService.js +511 -0
- package/.claude/worktrees/agent-multi-profile/src/services/ScanApiService.js +775 -0
- package/.claude/worktrees/agent-multi-profile/src/services/SignalHandler.js +255 -0
- package/.claude/worktrees/agent-multi-profile/src/services/SmartWatchDatabaseService.js +527 -0
- package/.claude/worktrees/agent-multi-profile/src/services/WatchService.js +783 -0
- package/.claude/worktrees/agent-multi-profile/src/services/upload/ApiUploadService.js +676 -0
- package/.claude/worktrees/agent-multi-profile/src/services/upload/BaseUploadService.js +36 -0
- package/.claude/worktrees/agent-multi-profile/src/services/upload/MultiApiUploadService.js +233 -0
- package/.claude/worktrees/agent-multi-profile/src/services/upload/SupabaseUploadService.js +148 -0
- package/.claude/worktrees/agent-multi-profile/src/services/upload/UploadServiceFactory.js +100 -0
- package/.claude/worktrees/agent-multi-profile/src/utils/CleanupManager.js +262 -0
- package/.claude/worktrees/agent-multi-profile/src/utils/FileOperations.js +192 -0
- package/.claude/worktrees/agent-multi-profile/src/utils/FileSanitizer.js +99 -0
- package/.claude/worktrees/agent-multi-profile/src/utils/PathDetector.js +198 -0
- package/.claude/worktrees/agent-multi-profile/src/utils/PathNormalizer.js +274 -0
- package/.claude/worktrees/agent-multi-profile/src/utils/WatchEventHandler.js +522 -0
- package/.claude/worktrees/agent-multi-profile/supabase/migrations/001_create_initial_schema.sql +366 -0
- package/.claude/worktrees/agent-multi-profile/supabase/migrations/002_align_with_arela_api_schema.sql +145 -0
- package/.claude/worktrees/agent-multi-profile/tests/commands/IdentifyCommand.test.js +570 -0
- package/.claude/worktrees/agent-multi-profile/tests/commands/PropagateCommand.test.js +568 -0
- package/.claude/worktrees/agent-multi-profile/tests/commands/PushCommand.test.js +754 -0
- package/.claude/worktrees/agent-multi-profile/tests/commands/ScanCommand.test.js +382 -0
- package/.claude/worktrees/agent-multi-profile/tests/unit/PathAndTableNameGeneration.test.js +1211 -0
- package/.claude/worktrees/agent-multi-profile/tests/unit/factura-inter-agencia.test.js +218 -0
- package/.claude/worktrees/agent-multi-profile/tests/unit/pedimento-completo-xml-matcher.test.js +271 -0
- package/.claude/worktrees/agent-multi-profile/tests/unit/pedimento-simplificado-matcher.test.js +185 -0
- package/.claude/worktrees/agent-multi-profile/tests/unit/scoring-engine.test.js +221 -0
- package/README.md +65 -0
- package/package.json +1 -1
- package/src/commands/AgentCommand.js +210 -0
- package/src/commands/AgentInitCommand.js +316 -0
- package/src/commands/PollWorkerCommand.js +11 -322
- package/src/config/config.js +44 -6
- package/src/document-type-shared.js +1 -1
- package/src/document-types/pedimento-completo.js +8 -1
- package/src/file-detection.js +9 -0
- package/src/index.js +55 -0
- package/src/scoring/scoring-engine.js +1 -1
- package/src/services/LoggingService.js +35 -0
- package/src/services/PipelineApiService.js +7 -1
- package/src/services/PipelineJobRunner.js +389 -0
- package/src/services/ProfileManager.js +164 -0
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
# Arela Uploader Environment Configuration
|
|
2
|
+
# Copy this to your .env file and adjust values for your setup
|
|
3
|
+
|
|
4
|
+
# =============================================================================
|
|
5
|
+
# BASIC CONFIGURATION
|
|
6
|
+
# =============================================================================
|
|
7
|
+
|
|
8
|
+
# Arela API Configuration
|
|
9
|
+
ARELA_API_URL=https://your-arela-api-url.com
|
|
10
|
+
ARELA_API_TOKEN=your-api-token-here
|
|
11
|
+
|
|
12
|
+
# API Agencia - Configura aquí la URL y Token de la agencia activa
|
|
13
|
+
ARELA_API_AGENCIA_URL=https://agencia-api-example.com
|
|
14
|
+
ARELA_API_AGENCIA_TOKEN=your-agencia-api-token-here
|
|
15
|
+
|
|
16
|
+
# API Cliente - Configura aquí la URL y Token del cliente activo
|
|
17
|
+
ARELA_API_CLIENTE_URL=https://cliente-api-example.com
|
|
18
|
+
ARELA_API_CLIENTE_TOKEN=your-cliente-api-token-here
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
# Supabase Configuration (fallback)
|
|
22
|
+
SUPABASE_URL=https://your-supabase-url.supabase.co
|
|
23
|
+
SUPABASE_KEY=your-supabase-key-here
|
|
24
|
+
SUPABASE_BUCKET=your-bucket-name
|
|
25
|
+
|
|
26
|
+
# Upload Sources (separate with |)
|
|
27
|
+
UPLOAD_BASE_PATH=/path/to/your/upload/base
|
|
28
|
+
UPLOAD_SOURCES=folder1|folder2|folder3
|
|
29
|
+
UPLOAD_RFCS=rfc1|rfc2|rfc3
|
|
30
|
+
|
|
31
|
+
# =============================================================================
|
|
32
|
+
# SCAN CONFIGURATION (for arela scan command)
|
|
33
|
+
# =============================================================================
|
|
34
|
+
|
|
35
|
+
# Company identifier for this CLI instance (required)
|
|
36
|
+
# Use a short, descriptive slug for your company/agency/client
|
|
37
|
+
# Examples: "acme_corp", "cliente_123", "agencia_xyz"
|
|
38
|
+
ARELA_COMPANY_SLUG=
|
|
39
|
+
|
|
40
|
+
# Server identifier (required)
|
|
41
|
+
# Use a unique ID for each server/NAS where arela-cli is installed
|
|
42
|
+
# Examples: "nas01", "server-mx", "storage-01"
|
|
43
|
+
ARELA_SERVER_ID=
|
|
44
|
+
|
|
45
|
+
# Base path label (optional, auto-derived from UPLOAD_BASE_PATH if not set)
|
|
46
|
+
# Short label describing the base path being scanned
|
|
47
|
+
# Examples: "data", "documents", "archive"
|
|
48
|
+
ARELA_BASE_PATH_LABEL=
|
|
49
|
+
|
|
50
|
+
# System file patterns to exclude from scan (comma-separated)
|
|
51
|
+
# These files will be filtered before uploading stats to reduce payload
|
|
52
|
+
SCAN_EXCLUDE_PATTERNS=.DS_Store,Thumbs.db,desktop.ini,__pycache__,.pyc,.tmp,.swp,$RECYCLE.BIN,System Volume Information,~$*
|
|
53
|
+
|
|
54
|
+
# Batch size for scan operations (default: 2000 records per API call)
|
|
55
|
+
SCAN_BATCH_SIZE=2000
|
|
56
|
+
|
|
57
|
+
# Directory depth level for creating separate tables (default: 0)
|
|
58
|
+
# 0 = single table for entire base path
|
|
59
|
+
# 1 = one table per first-level subdirectory
|
|
60
|
+
# 2 = one table per second-level subdirectory, etc.
|
|
61
|
+
# Example: with level=1 and base=/data, creates tables for /data/folder1, /data/folder2, etc.
|
|
62
|
+
SCAN_DIRECTORY_LEVEL=0
|
|
63
|
+
|
|
64
|
+
# =============================================================================
|
|
65
|
+
# PUSH CONFIGURATION (for arela push command)
|
|
66
|
+
# =============================================================================
|
|
67
|
+
|
|
68
|
+
# Filter files to upload by RFC (pipe-separated, optional)
|
|
69
|
+
# If not set, all files with arela_path will be uploaded
|
|
70
|
+
# Examples: "RFC123456ABC|RFC789012DEF"
|
|
71
|
+
PUSH_RFCS=
|
|
72
|
+
|
|
73
|
+
# Filter files to upload by year (pipe-separated, optional)
|
|
74
|
+
# If not set, all files with arela_path will be uploaded
|
|
75
|
+
# Examples: "2023|2024|2025"
|
|
76
|
+
PUSH_YEARS=
|
|
77
|
+
|
|
78
|
+
# Batch size for fetching files from database (default: 100)
|
|
79
|
+
PUSH_BATCH_SIZE=100
|
|
80
|
+
|
|
81
|
+
# Concurrent upload batch size (default: 10)
|
|
82
|
+
# Number of files to upload simultaneously
|
|
83
|
+
PUSH_UPLOAD_BATCH_SIZE=10
|
|
84
|
+
|
|
85
|
+
# Storage bucket for uploaded files (optional, defaults to SUPABASE_BUCKET)
|
|
86
|
+
# Examples: "archivos", "documents", "storage"
|
|
87
|
+
PUSH_BUCKET=arela
|
|
88
|
+
|
|
89
|
+
# Folder structure prefix for uploaded files (optional)
|
|
90
|
+
# This prefix is prepended to the arela_path when uploading files
|
|
91
|
+
# Examples: "agencia/cliente", "2024/docs", "imports/batch1"
|
|
92
|
+
PUSH_FOLDER_STRUCTURE=
|
|
93
|
+
|
|
94
|
+
# =============================================================================
|
|
95
|
+
# PERFORMANCE OPTIMIZATION FOR MULTIPLE API REPLICAS
|
|
96
|
+
# =============================================================================
|
|
97
|
+
|
|
98
|
+
# API Connection Configuration
|
|
99
|
+
# Set this to match your number of API replicas (e.g., if you have 10 API instances, set to 10)
|
|
100
|
+
MAX_API_CONNECTIONS=10
|
|
101
|
+
|
|
102
|
+
# API Connection Timeout (milliseconds)
|
|
103
|
+
API_CONNECTION_TIMEOUT=60000
|
|
104
|
+
|
|
105
|
+
# API Retry Configuration
|
|
106
|
+
# Maximum number of retry attempts for failed API requests
|
|
107
|
+
API_MAX_RETRIES=3
|
|
108
|
+
|
|
109
|
+
# Enable exponential backoff for retries (true/false)
|
|
110
|
+
# When true, retry delays increase: 1s, 2s, 4s, 8s, 16s
|
|
111
|
+
# When false, uses fixed delay (API_RETRY_DELAY)
|
|
112
|
+
API_RETRY_EXPONENTIAL_BACKOFF=true
|
|
113
|
+
|
|
114
|
+
# Fixed retry delay in milliseconds (only used if exponential backoff is disabled)
|
|
115
|
+
API_RETRY_DELAY=1000
|
|
116
|
+
|
|
117
|
+
# Batch Processing Configuration
|
|
118
|
+
# Files processed concurrently per batch (should be >= MAX_API_CONNECTIONS for best performance)
|
|
119
|
+
BATCH_SIZE=100
|
|
120
|
+
|
|
121
|
+
# Delay between batches (0 for maximum speed)
|
|
122
|
+
BATCH_DELAY=0
|
|
123
|
+
|
|
124
|
+
# Source Processing Concurrency
|
|
125
|
+
# Number of upload sources/folders to process simultaneously
|
|
126
|
+
MAX_CONCURRENT_SOURCES=2
|
|
127
|
+
|
|
128
|
+
# =============================================================================
|
|
129
|
+
# EXAMPLE CONFIGURATIONS FOR DIFFERENT SCENARIOS
|
|
130
|
+
# =============================================================================
|
|
131
|
+
|
|
132
|
+
# For 10 API Replicas (High Performance Setup):
|
|
133
|
+
# MAX_API_CONNECTIONS=10
|
|
134
|
+
# BATCH_SIZE=100
|
|
135
|
+
# MAX_CONCURRENT_SOURCES=3
|
|
136
|
+
# BATCH_DELAY=0
|
|
137
|
+
|
|
138
|
+
# For 5 API Replicas (Medium Performance Setup):
|
|
139
|
+
# MAX_API_CONNECTIONS=5
|
|
140
|
+
# BATCH_SIZE=50
|
|
141
|
+
# MAX_CONCURRENT_SOURCES=2
|
|
142
|
+
# BATCH_DELAY=0
|
|
143
|
+
|
|
144
|
+
# For 1 API Instance (Single Instance Setup):
|
|
145
|
+
# MAX_API_CONNECTIONS=5
|
|
146
|
+
# BATCH_SIZE=20
|
|
147
|
+
# MAX_CONCURRENT_SOURCES=1
|
|
148
|
+
# BATCH_DELAY=100
|
|
149
|
+
|
|
150
|
+
# =============================================================================
|
|
151
|
+
# WATCH MODE CONFIGURATION
|
|
152
|
+
# =============================================================================
|
|
153
|
+
|
|
154
|
+
# Habilitar watch mode (true/false)
|
|
155
|
+
WATCH_ENABLED=false
|
|
156
|
+
|
|
157
|
+
# Configuración de directorios a observar (formato JSON)
|
|
158
|
+
# Cada directorio puede tener su propia folderStructure para organizar en el bucket
|
|
159
|
+
# Formato: {"ruta/directorio1":"estructura-1","ruta/directorio2":"estructura-2"}
|
|
160
|
+
#
|
|
161
|
+
# IMPORTANTE: El folderStructure se usa en el comando:
|
|
162
|
+
# upload --upload-by-rfc --folder-structure <estructura>
|
|
163
|
+
#
|
|
164
|
+
# Ejemplo:
|
|
165
|
+
WATCH_DIRECTORY_CONFIGS={"../../Documents/2022":"estructura-2022","../../Documents/2023":"estructura-2023"}
|
|
166
|
+
|
|
167
|
+
# DEPRECATED: Configuración antigua (se mantiene para compatibilidad hacia atrás)
|
|
168
|
+
# WATCH_DIRECTORIES=/ruta/carpeta1,/ruta/carpeta2
|
|
169
|
+
|
|
170
|
+
# Estrategia de upload (opciones: individual|batch|full-structure)
|
|
171
|
+
# - individual: Sube solo el archivo modificado más reciente
|
|
172
|
+
# - batch: Sube un lote de N archivos recientes
|
|
173
|
+
# - full-structure: Sube la estructura completa de carpetas
|
|
174
|
+
WATCH_STRATEGY=batch
|
|
175
|
+
|
|
176
|
+
# Debouncing en milisegundos (esperar entre eventos antes de procesar)
|
|
177
|
+
WATCH_DEBOUNCE_MS=1000
|
|
178
|
+
|
|
179
|
+
# Tamaño de batch para strategy batch
|
|
180
|
+
WATCH_BATCH_SIZE=10
|
|
181
|
+
|
|
182
|
+
# Usar polling en lugar de eventos nativos del filesystem
|
|
183
|
+
# Útil para sistemas de archivos remotos o NFS
|
|
184
|
+
WATCH_USE_POLLING=false
|
|
185
|
+
|
|
186
|
+
# Interval de polling en milisegundos (solo si WATCH_USE_POLLING=true)
|
|
187
|
+
WATCH_POLL_INTERVAL=100
|
|
188
|
+
|
|
189
|
+
# Umbral de estabilidad en ms (esperar a que el archivo deje de cambiar)
|
|
190
|
+
WATCH_STABILITY_THRESHOLD=300
|
|
191
|
+
|
|
192
|
+
# Patrones a ignorar (separados por coma, se usan como regex)
|
|
193
|
+
WATCH_IGNORE_PATTERNS=*.tmp,*.bak,*.swp
|
|
194
|
+
|
|
195
|
+
# Detección automática de tipos de documento
|
|
196
|
+
WATCH_AUTO_DETECT=false
|
|
197
|
+
|
|
198
|
+
# Organización automática de archivos
|
|
199
|
+
WATCH_AUTO_ORGANIZE=false
|
|
200
|
+
|
|
201
|
+
# =============================================================================
|
|
202
|
+
# WATCH MODE - AUTOMATIC PROCESSING PIPELINE
|
|
203
|
+
# =============================================================================
|
|
204
|
+
#
|
|
205
|
+
# El pipeline automático ejecuta la siguiente secuencia cuando se detecta un archivo nuevo:
|
|
206
|
+
# 1. Stats Collection → stats --stats-only (recopila información del archivo)
|
|
207
|
+
# 2. PDF Detection → detect --detect-pdfs (identifica pedimentos simplificados)
|
|
208
|
+
# 3. Path Propagation → detect --propagate-arela-path (propaga a documentos relacionados)
|
|
209
|
+
# 4. RFC Upload → upload --upload-by-rfc --folder-structure (sube con estructura)
|
|
210
|
+
#
|
|
211
|
+
# El pipeline se habilita automáticamente en watch mode y usa la folderStructure
|
|
212
|
+
# definida para cada WATCH_DIRECTORY_CONFIGS
|
|
213
|
+
#
|
|
214
|
+
# Para deshabilitar en CLI, usa: arela watch --no-auto-processing
|
|
215
|
+
|
|
216
|
+
# =============================================================================
|
|
217
|
+
# LOGGING AND MONITORING
|
|
218
|
+
# =============================================================================
|
|
219
|
+
|
|
220
|
+
# Progress bar update frequency
|
|
221
|
+
PROGRESS_UPDATE_INTERVAL=10
|
|
222
|
+
|
|
223
|
+
# Enable verbose logging (true/false)
|
|
224
|
+
VERBOSE_LOGGING=false
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"singleQuote": true,
|
|
3
|
+
"trailingComma": "all",
|
|
4
|
+
"importOrder": ["^\\.\\./services/(.*)$", "^\\.\\./dto/(.*)$", "^[./]"],
|
|
5
|
+
"importOrderSeparation": true,
|
|
6
|
+
"importOrderSortSpecifiers": true,
|
|
7
|
+
"importOrderParserPlugins": [
|
|
8
|
+
"typescript",
|
|
9
|
+
"classProperties",
|
|
10
|
+
"decorators-legacy"
|
|
11
|
+
],
|
|
12
|
+
"plugins": ["@trivago/prettier-plugin-sort-imports"]
|
|
13
|
+
}
|
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
# arela-uploader
|
|
2
|
+
|
|
3
|
+
CLI tool to scan, detect, and upload files to the Arela API with multi-tenant support, automatic document detection, and directory watching.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npm install -g @arela/uploader
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Quick Start
|
|
12
|
+
|
|
13
|
+
The recommended workflow runs 4 phases in sequence:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
arela scan # 1. Discover files and register metadata
|
|
17
|
+
arela identify # 2. Detect document types from PDF content
|
|
18
|
+
arela propagate # 3. Propagate paths to related files
|
|
19
|
+
arela push # 4. Upload files to storage
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## Commands
|
|
23
|
+
|
|
24
|
+
### `arela scan`
|
|
25
|
+
|
|
26
|
+
Scan the filesystem and register file metadata via the API. Supports streaming discovery and multi-level directory partitioning.
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
arela scan # Basic scan
|
|
30
|
+
arela scan --api agencia # Scan using agencia API
|
|
31
|
+
arela scan --count-first # Count files first for progress %
|
|
32
|
+
arela scan --no-stream # Synchronous discovery (no streaming)
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
| Flag | Description | Default |
|
|
36
|
+
|------|-------------|---------|
|
|
37
|
+
| `--api <target>` | API target (`default`, `agencia`, `cliente`) | `default` |
|
|
38
|
+
| `--count-first` | Count files before scanning for progress tracking | — |
|
|
39
|
+
| `--no-stream` | Use synchronous file discovery | — |
|
|
40
|
+
|
|
41
|
+
### `arela identify`
|
|
42
|
+
|
|
43
|
+
Detect document types (e.g. pedimento simplificado) from PDF content using pattern matchers. Runs against existing database records.
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
arela identify # Identify documents
|
|
47
|
+
arela identify --batch-size 200 # Larger batches
|
|
48
|
+
arela identify --show-stats # Show performance stats
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
| Flag | Description | Default |
|
|
52
|
+
|------|-------------|---------|
|
|
53
|
+
| `--api <target>` | API target | `default` |
|
|
54
|
+
| `-b, --batch-size <size>` | Files per batch | `100` |
|
|
55
|
+
| `--show-stats` | Show performance statistics | — |
|
|
56
|
+
|
|
57
|
+
### `arela propagate`
|
|
58
|
+
|
|
59
|
+
Propagate `arela_path` from identified pedimento records to related files in the same directory.
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
arela propagate # Propagate paths
|
|
63
|
+
arela propagate --batch-size 100 # Process 100 pedimentos per batch
|
|
64
|
+
arela propagate --show-stats # Show statistics
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
| Flag | Description | Default |
|
|
68
|
+
|------|-------------|---------|
|
|
69
|
+
| `--api <target>` | API target | `default` |
|
|
70
|
+
| `-b, --batch-size <size>` | Pedimentos per batch | `50` |
|
|
71
|
+
| `--show-stats` | Show performance statistics | — |
|
|
72
|
+
|
|
73
|
+
### `arela push`
|
|
74
|
+
|
|
75
|
+
Upload files to Arela storage, filtered by RFC and/or year. Supports cross-tenant mode.
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
arela push # Upload all files with arela_path
|
|
79
|
+
arela push --rfcs RFC1,RFC2 --years 2023,2024 # Filter by RFC and year
|
|
80
|
+
arela push --source-api agencia --target-api cliente # Cross-tenant upload
|
|
81
|
+
arela push --folder-structure "prefix/path" # Add storage path prefix
|
|
82
|
+
arela push --no-auto-organize # Disable auto-organization
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
| Flag | Description | Default |
|
|
86
|
+
|------|-------------|---------|
|
|
87
|
+
| `--api <target>` | API target | `default` |
|
|
88
|
+
| `--scan-api <target>` | API for reading scan data | `default` |
|
|
89
|
+
| `--push-api <target>` | API for uploading files | — |
|
|
90
|
+
| `--source-api <target>` | Source API (cross-tenant) | — |
|
|
91
|
+
| `--target-api <target>` | Target API (cross-tenant) | — |
|
|
92
|
+
| `-b, --batch-size <size>` | Files to fetch per batch | `100` |
|
|
93
|
+
| `--upload-batch-size <size>` | Concurrent uploads | `10` |
|
|
94
|
+
| `--rfcs <rfcs>` | Comma-separated RFCs (overrides `PUSH_RFCS`) | — |
|
|
95
|
+
| `--years <years>` | Comma-separated years (overrides `PUSH_YEARS`) | — |
|
|
96
|
+
| `--folder-structure <path>` | Storage path prefix | — |
|
|
97
|
+
| `--no-auto-organize` | Disable automatic file organization | — |
|
|
98
|
+
| `--show-stats` | Show performance statistics | — |
|
|
99
|
+
|
|
100
|
+
### `arela upload`
|
|
101
|
+
|
|
102
|
+
Legacy upload command with multiple modes (stats-only, RFC-based, run-all-phases).
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
arela upload --batch-size 10 # Basic upload
|
|
106
|
+
arela upload --upload-by-rfc # Upload by RFC
|
|
107
|
+
arela upload --run-all-phases # Run all phases
|
|
108
|
+
arela upload --force-supabase --prefix "folder" # Direct Supabase upload
|
|
109
|
+
arela upload --folder-structure "2024/pedimentos" # Custom folder structure
|
|
110
|
+
arela upload --auto-detect-structure # Auto-detect year/pedimento from paths
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
| Flag | Description | Default |
|
|
114
|
+
|------|-------------|---------|
|
|
115
|
+
| `--api <target>` | API target | `default` |
|
|
116
|
+
| `--source-api <target>` | Source API (cross-tenant) | — |
|
|
117
|
+
| `--target-api <target>` | Target API (cross-tenant) | — |
|
|
118
|
+
| `-b, --batch-size <size>` | Files per batch | `10` |
|
|
119
|
+
| `-p, --prefix <prefix>` | Prefix for uploaded files | — |
|
|
120
|
+
| `--folder-structure <structure>` | Custom folder structure | — |
|
|
121
|
+
| `--client-path <path>` | Override client path for metadata | — |
|
|
122
|
+
| `--auto-detect-structure` | Auto-detect folder structure from paths | — |
|
|
123
|
+
| `--auto-detect` | Enable document type detection | — |
|
|
124
|
+
| `--auto-organize` | Enable automatic file organization | — |
|
|
125
|
+
| `--force-supabase` | Force direct Supabase upload (skip API) | — |
|
|
126
|
+
| `--skip-processed` | Skip already-processed files | — |
|
|
127
|
+
| `--show-stats` | Show performance statistics | — |
|
|
128
|
+
| `--upload-by-rfc` | Upload based on RFC values from `UPLOAD_RFCS` | — |
|
|
129
|
+
| `--run-all-phases` | Run all processing phases sequentially | — |
|
|
130
|
+
|
|
131
|
+
### `arela watch`
|
|
132
|
+
|
|
133
|
+
Monitor directories for file changes and process them automatically.
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
arela watch -d "/path/to/dir1,/path/to/dir2" # Watch directories
|
|
137
|
+
arela watch --api cliente # Watch with specific API
|
|
138
|
+
arela watch --source-api agencia --target-api cliente # Cross-tenant watch
|
|
139
|
+
arela watch --strategy individual # Upload each file immediately
|
|
140
|
+
arela watch --auto-processing --batch-size 10 # Auto pipeline (scan→identify→propagate→push)
|
|
141
|
+
arela watch --dry-run # Simulate without uploading
|
|
142
|
+
arela watch --poll 5000 # Use polling (for NFS/remote FS)
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
| Flag | Description | Default |
|
|
146
|
+
|------|-------------|---------|
|
|
147
|
+
| `--api <target>` | API target | `default` |
|
|
148
|
+
| `--source-api <target>` | Source API (cross-tenant) | — |
|
|
149
|
+
| `--target-api <target>` | Target API (cross-tenant) | — |
|
|
150
|
+
| `-d, --directories <paths>` | Comma-separated directories to watch | — |
|
|
151
|
+
| `-s, --strategy <strategy>` | `batch`, `individual`, or `full-structure` | `batch` |
|
|
152
|
+
| `--debounce <ms>` | Debounce delay in milliseconds | `1000` |
|
|
153
|
+
| `-b, --batch-size <size>` | Files per batch | `10` |
|
|
154
|
+
| `--poll <ms>` | Use polling with interval in ms | — |
|
|
155
|
+
| `--ignore <patterns>` | Comma-separated patterns to ignore | — |
|
|
156
|
+
| `--auto-detect` | Enable document type detection | — |
|
|
157
|
+
| `--auto-organize` | Enable file organization | — |
|
|
158
|
+
| `--auto-processing` | Enable automatic 4-step pipeline | — |
|
|
159
|
+
| `--dry-run` | Simulate without uploading | — |
|
|
160
|
+
| `--verbose` | Verbose logging | — |
|
|
161
|
+
|
|
162
|
+
**Watch strategies:**
|
|
163
|
+
- **`batch`** (default) — Groups files and uploads periodically
|
|
164
|
+
- **`individual`** — Uploads each file immediately on change
|
|
165
|
+
- **`full-structure`** — Preserves full directory structure during upload
|
|
166
|
+
|
|
167
|
+
### `arela stats`
|
|
168
|
+
|
|
169
|
+
Collect filesystem statistics without uploading (legacy, wraps upload in stats-only mode).
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
arela stats --batch-size 10
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
### `arela detect`
|
|
176
|
+
|
|
177
|
+
Legacy document detection command. Prefer `arela identify` and `arela propagate` instead.
|
|
178
|
+
|
|
179
|
+
```bash
|
|
180
|
+
arela detect --batch-size 10 # PDF detection
|
|
181
|
+
arela detect --propagate-arela-path # Path propagation
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
### `arela query`
|
|
185
|
+
|
|
186
|
+
Query the database for file status.
|
|
187
|
+
|
|
188
|
+
```bash
|
|
189
|
+
arela query --ready-files # Show files ready for upload
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
### `arela config`
|
|
193
|
+
|
|
194
|
+
Display current configuration for all API targets and settings.
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
arela config
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
### `arela agent`
|
|
201
|
+
|
|
202
|
+
Multi-profile polling agent: ONE process (one terminal) serves every tenant API,
|
|
203
|
+
replacing the N `arela worker --poll` terminals (one per RFC×source folder with
|
|
204
|
+
its own `.env`). Scan directories, tuning and scheduling live **server-side** in
|
|
205
|
+
`pipeline_config`; the only local configuration is `profiles.json`.
|
|
206
|
+
|
|
207
|
+
```bash
|
|
208
|
+
arela agent # uses ~/.arela/profiles.json
|
|
209
|
+
arela agent --config ./profiles.json # explicit config path
|
|
210
|
+
arela agent --profile ktj,cem # run a subset (debugging)
|
|
211
|
+
arela agent --interval 10000 # idle poll interval in ms
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
`~/.arela/profiles.json` (keep it private — it contains API keys):
|
|
215
|
+
|
|
216
|
+
```jsonc
|
|
217
|
+
{
|
|
218
|
+
"version": 1,
|
|
219
|
+
"serverId": "PALCO-01", // ONE id per machine; must match pipeline_config.serverId (or leave that null)
|
|
220
|
+
"pollIntervalMs": 5000, // optional
|
|
221
|
+
"logFile": "~/.arela/arela-agent.log", // optional
|
|
222
|
+
"profiles": [
|
|
223
|
+
{ "name": "ktj", "url": "https://ktj-api.example.com", "token": "ak_..." },
|
|
224
|
+
{
|
|
225
|
+
"name": "cem", "url": "https://cem-api.example.com", "token": "ak_...",
|
|
226
|
+
// Only for real cross-tenant jobs (sourceApi != targetApi):
|
|
227
|
+
"targets": { "agencia": { "baseUrl": "https://agencia-api.example.com", "token": "ak_..." } }
|
|
228
|
+
}
|
|
229
|
+
]
|
|
230
|
+
}
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
Behavior notes:
|
|
234
|
+
|
|
235
|
+
- Jobs are processed **one at a time globally** (round-robin between profiles).
|
|
236
|
+
The config singleton and per-job env overrides make in-process parallelism
|
|
237
|
+
unsafe by design; if parallelism is ever needed, run child processes.
|
|
238
|
+
- An unreachable tenant API backs off exponentially (30s → 10min) without
|
|
239
|
+
blocking the other profiles.
|
|
240
|
+
- `Ctrl+C` waits for the in-flight job to finish before exiting, so jobs never
|
|
241
|
+
end up orphaned in RUNNING state.
|
|
242
|
+
- While a long job runs, the OTHER profiles' `worker_status` may show OFFLINE
|
|
243
|
+
(no poll for >30s). Cosmetic: their jobs stay QUEUED and are picked up next.
|
|
244
|
+
- Server-side setup per tenant: seed `pipeline_config` rows (one per RFC×source,
|
|
245
|
+
with `scanDirectories`, `extraOptions` tuning and schedule fields) and enable
|
|
246
|
+
the `PIPELINE_SCHEDULER` cronjob (`PUT /api/cronjobs`).
|
|
247
|
+
|
|
248
|
+
### `arela agent init`
|
|
249
|
+
|
|
250
|
+
Migration helper: reads the legacy per-folder `.env` files and generates
|
|
251
|
+
`profiles.json` plus the `pipeline_config` seed payloads — locally, without
|
|
252
|
+
calling any API (tokens never leave the machine).
|
|
253
|
+
|
|
254
|
+
```bash
|
|
255
|
+
arela agent init --from ../ktj-cli-maya ../ktj-cli-matro ../cem-cli-nas --server-id PALCO-01
|
|
256
|
+
# Generates in ~/.arela/: profiles.json, seed-configs.json, seed-configs.sh
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
Folders pointing at the same tenant API collapse into one profile; each
|
|
260
|
+
folder×RFC becomes a `pipeline_config` payload (`scheduleEnabled: false`
|
|
261
|
+
initially). Review the generated files, run `bash ~/.arela/seed-configs.sh` to
|
|
262
|
+
seed the server, then start `arela agent`. The old `.env` folders keep working
|
|
263
|
+
during the migration — `arela worker --poll` is untouched.
|
|
264
|
+
|
|
265
|
+
### Global Options
|
|
266
|
+
|
|
267
|
+
| Flag | Description |
|
|
268
|
+
|------|-------------|
|
|
269
|
+
| `-v, --verbose` | Enable verbose logging |
|
|
270
|
+
| `--clear-log` | Clear log file before starting |
|
|
271
|
+
| `--version` | Display version number |
|
|
272
|
+
| `-h, --help` | Display help |
|
|
273
|
+
|
|
274
|
+
## Multi-Tenant API
|
|
275
|
+
|
|
276
|
+
Three API targets are available: `default`, `agencia`, and `cliente`. Configure each in your `.env` file.
|
|
277
|
+
|
|
278
|
+
```bash
|
|
279
|
+
# Single-tenant
|
|
280
|
+
arela scan --api cliente
|
|
281
|
+
arela push --api agencia
|
|
282
|
+
|
|
283
|
+
# Cross-tenant (read from one API, upload to another)
|
|
284
|
+
arela push --source-api agencia --target-api cliente
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
| Phase | `--source-api` | `--target-api` |
|
|
288
|
+
|-------|-----------------|-----------------|
|
|
289
|
+
| Scan / Identify / Propagate | ✓ | — |
|
|
290
|
+
| Push / Upload | — | ✓ |
|
|
291
|
+
|
|
292
|
+
## Environment Variables
|
|
293
|
+
|
|
294
|
+
Copy `.env.template` to `.env` and configure:
|
|
295
|
+
|
|
296
|
+
### API Configuration
|
|
297
|
+
|
|
298
|
+
```env
|
|
299
|
+
ARELA_API_URL=https://your-arela-api.example.com
|
|
300
|
+
ARELA_API_TOKEN=your-api-token
|
|
301
|
+
|
|
302
|
+
ARELA_API_AGENCIA_URL=https://agencia-api.example.com
|
|
303
|
+
ARELA_API_AGENCIA_TOKEN=your-agencia-token
|
|
304
|
+
|
|
305
|
+
ARELA_API_CLIENTE_URL=https://cliente-api.example.com
|
|
306
|
+
ARELA_API_CLIENTE_TOKEN=your-cliente-token
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
### Supabase (fallback)
|
|
310
|
+
|
|
311
|
+
```env
|
|
312
|
+
SUPABASE_URL=https://your-project.supabase.co
|
|
313
|
+
SUPABASE_KEY=your-supabase-key
|
|
314
|
+
SUPABASE_BUCKET=your-bucket-name
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
### Upload & File Sources
|
|
318
|
+
|
|
319
|
+
```env
|
|
320
|
+
UPLOAD_BASE_PATH=/path/to/files
|
|
321
|
+
UPLOAD_SOURCES=folder1|folder2|folder3
|
|
322
|
+
UPLOAD_RFCS=RFC1|RFC2|RFC3
|
|
323
|
+
UPLOAD_YEARS=2023|2024
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
### Scan Configuration
|
|
327
|
+
|
|
328
|
+
```env
|
|
329
|
+
ARELA_COMPANY_SLUG=your_company # Company identifier
|
|
330
|
+
ARELA_SERVER_ID=nas01 # Server/NAS identifier
|
|
331
|
+
ARELA_BASE_PATH_LABEL= # Optional (auto-derived from base path)
|
|
332
|
+
SCAN_EXCLUDE_PATTERNS=.DS_Store,Thumbs.db,desktop.ini
|
|
333
|
+
SCAN_BATCH_SIZE=2000 # Records per API call
|
|
334
|
+
SCAN_DIRECTORY_LEVEL=0 # Directory depth for table partitioning
|
|
335
|
+
```
|
|
336
|
+
|
|
337
|
+
### Push Configuration
|
|
338
|
+
|
|
339
|
+
```env
|
|
340
|
+
PUSH_RFCS= # Pipe-separated RFC filter
|
|
341
|
+
PUSH_YEARS= # Pipe-separated year filter
|
|
342
|
+
PUSH_BATCH_SIZE=100 # Fetch batch size
|
|
343
|
+
PUSH_UPLOAD_BATCH_SIZE=10 # Concurrent uploads
|
|
344
|
+
PUSH_BUCKET=arela # Storage bucket
|
|
345
|
+
PUSH_FOLDER_STRUCTURE= # Prefix for storage paths
|
|
346
|
+
```
|
|
347
|
+
|
|
348
|
+
### Performance Tuning
|
|
349
|
+
|
|
350
|
+
```env
|
|
351
|
+
MAX_API_CONNECTIONS=10 # Match your API replica count
|
|
352
|
+
API_CONNECTION_TIMEOUT=60000 # Timeout in ms
|
|
353
|
+
API_MAX_RETRIES=3
|
|
354
|
+
API_RETRY_EXPONENTIAL_BACKOFF=true
|
|
355
|
+
BATCH_SIZE=100
|
|
356
|
+
BATCH_DELAY=0
|
|
357
|
+
MAX_CONCURRENT_SOURCES=2
|
|
358
|
+
```
|
|
359
|
+
|
|
360
|
+
### Watch Mode
|
|
361
|
+
|
|
362
|
+
```env
|
|
363
|
+
WATCH_ENABLED=false
|
|
364
|
+
WATCH_DIRECTORY_CONFIGS={"../docs/2023":"prefix-2023","../docs/2024":"prefix-2024"}
|
|
365
|
+
WATCH_STRATEGY=batch # batch|individual|full-structure
|
|
366
|
+
WATCH_DEBOUNCE_MS=1000
|
|
367
|
+
WATCH_BATCH_SIZE=10
|
|
368
|
+
WATCH_USE_POLLING=false
|
|
369
|
+
WATCH_POLL_INTERVAL=100
|
|
370
|
+
WATCH_STABILITY_THRESHOLD=300
|
|
371
|
+
WATCH_IGNORE_PATTERNS=*.tmp,*.bak,*.swp
|
|
372
|
+
```
|
|
373
|
+
|
|
374
|
+
## Document Detection
|
|
375
|
+
|
|
376
|
+
The tool detects and classifies documents:
|
|
377
|
+
|
|
378
|
+
- **Pedimento Simplificado** — Extracts RFC, pedimento number, patente, aduana, and year from PDF content to compose an `arela_path` (`RFC/Year/Patente/Aduana/Pedimento/`)
|
|
379
|
+
- **Supporting Documents** — XML, TXT, and JSON customs-related documents
|
|
380
|
+
|
|
381
|
+
## File Sanitization
|
|
382
|
+
|
|
383
|
+
Filenames are automatically sanitized before upload:
|
|
384
|
+
|
|
385
|
+
| Original | Sanitized |
|
|
386
|
+
|----------|-----------|
|
|
387
|
+
| `Facturas Importación.pdf` | `Facturas-Importacion.pdf` |
|
|
388
|
+
| `File{with}brackets.pdf` | `File-with-brackets.pdf` |
|
|
389
|
+
| `Document ^& symbols.pdf` | `Document-and-symbols.pdf` |
|
|
390
|
+
| `file with spaces.pdf` | `file-with-spaces.pdf` |
|
|
391
|
+
|
|
392
|
+
Handles accented characters, Korean glyphs, special symbols, and email addresses.
|
|
393
|
+
|
|
394
|
+
## Development
|
|
395
|
+
|
|
396
|
+
```bash
|
|
397
|
+
npm test # Run tests
|
|
398
|
+
npm run test:watch # Run tests in watch mode
|
|
399
|
+
npm run test:coverage # Run tests with coverage
|
|
400
|
+
npm run format # Format code with Prettier
|
|
401
|
+
```
|
|
402
|
+
|
|
403
|
+
## License
|
|
404
|
+
|
|
405
|
+
ISC
|