@arela/uploader 0.2.13 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/.env.template +66 -0
  2. package/README.md +263 -62
  3. package/docs/API_ENDPOINTS_FOR_DETECTION.md +647 -0
  4. package/docs/QUICK_REFERENCE_API_DETECTION.md +264 -0
  5. package/docs/REFACTORING_SUMMARY_DETECT_PEDIMENTOS.md +200 -0
  6. package/package.json +3 -2
  7. package/scripts/cleanup-ds-store.js +109 -0
  8. package/scripts/cleanup-system-files.js +69 -0
  9. package/scripts/tests/phase-7-features.test.js +415 -0
  10. package/scripts/tests/signal-handling.test.js +275 -0
  11. package/scripts/tests/smart-watch-integration.test.js +554 -0
  12. package/scripts/tests/watch-service-integration.test.js +584 -0
  13. package/src/commands/UploadCommand.js +31 -4
  14. package/src/commands/WatchCommand.js +1342 -0
  15. package/src/config/config.js +270 -2
  16. package/src/document-type-shared.js +2 -0
  17. package/src/document-types/support-document.js +200 -0
  18. package/src/file-detection.js +9 -1
  19. package/src/index.js +163 -4
  20. package/src/services/AdvancedFilterService.js +505 -0
  21. package/src/services/AutoProcessingService.js +749 -0
  22. package/src/services/BenchmarkingService.js +381 -0
  23. package/src/services/DatabaseService.js +1019 -539
  24. package/src/services/ErrorMonitor.js +275 -0
  25. package/src/services/LoggingService.js +419 -1
  26. package/src/services/MonitoringService.js +401 -0
  27. package/src/services/PerformanceOptimizer.js +511 -0
  28. package/src/services/ReportingService.js +511 -0
  29. package/src/services/SignalHandler.js +255 -0
  30. package/src/services/SmartWatchDatabaseService.js +527 -0
  31. package/src/services/WatchService.js +783 -0
  32. package/src/services/upload/ApiUploadService.js +447 -3
  33. package/src/services/upload/MultiApiUploadService.js +233 -0
  34. package/src/services/upload/SupabaseUploadService.js +12 -5
  35. package/src/services/upload/UploadServiceFactory.js +24 -0
  36. package/src/utils/CleanupManager.js +262 -0
  37. package/src/utils/FileOperations.js +44 -0
  38. package/src/utils/WatchEventHandler.js +522 -0
  39. package/supabase/migrations/001_create_initial_schema.sql +366 -0
  40. package/supabase/migrations/002_align_with_arela_api_schema.sql +145 -0
  41. package/.envbackup +0 -37
  42. package/SUPABASE_UPLOAD_FIX.md +0 -157
  43. package/commands.md +0 -14
@@ -16,6 +16,7 @@ class Config {
16
16
  this.upload = this.#loadUploadConfig();
17
17
  this.performance = this.#loadPerformanceConfig();
18
18
  this.logging = this.#loadLoggingConfig();
19
+ this.watch = this.#loadWatchConfig();
19
20
  }
20
21
 
21
22
  /**
@@ -49,15 +50,168 @@ class Config {
49
50
 
50
51
  /**
51
52
  * Load API configuration
53
+ * Supports 3 API targets: default, agencia, cliente
54
+ * Configure ARELA_API_CLIENTE_URL/TOKEN for the specific client you want to use
52
55
  * @private
53
56
  */
54
57
  #loadApiConfig() {
55
58
  return {
59
+ // Default/legacy API (backward compatible)
56
60
  baseUrl: process.env.ARELA_API_URL,
57
61
  token: process.env.ARELA_API_TOKEN,
62
+ // Multi-tenant API targets (only 3 options)
63
+ targets: {
64
+ agencia: {
65
+ baseUrl:
66
+ process.env.ARELA_API_AGENCIA_URL || process.env.ARELA_API_URL,
67
+ token:
68
+ process.env.ARELA_API_AGENCIA_TOKEN || process.env.ARELA_API_TOKEN,
69
+ },
70
+ cliente: {
71
+ baseUrl:
72
+ process.env.ARELA_API_CLIENTE_URL || process.env.ARELA_API_URL,
73
+ token:
74
+ process.env.ARELA_API_CLIENTE_TOKEN || process.env.ARELA_API_TOKEN,
75
+ },
76
+ },
77
+ // Current active target (can be changed at runtime)
78
+ activeTarget: process.env.ARELA_API_TARGET || 'default',
58
79
  };
59
80
  }
60
81
 
82
+ /**
83
+ * Get API configuration for a specific target
84
+ * @param {string} target - API target: 'agencia', 'cliente', or 'default'
85
+ * @returns {Object} API configuration with baseUrl and token
86
+ */
87
+ getApiConfig(target = null) {
88
+ const targetName = target || this.api.activeTarget || 'default';
89
+
90
+ if (targetName === 'default') {
91
+ return {
92
+ baseUrl: this.api.baseUrl,
93
+ token: this.api.token,
94
+ };
95
+ }
96
+
97
+ const targetConfig = this.api.targets[targetName];
98
+ if (!targetConfig) {
99
+ console.warn(
100
+ `⚠️ Unknown API target '${targetName}', falling back to default`,
101
+ );
102
+ return {
103
+ baseUrl: this.api.baseUrl,
104
+ token: this.api.token,
105
+ };
106
+ }
107
+
108
+ return targetConfig;
109
+ }
110
+
111
+ /**
112
+ * Set the active API target at runtime
113
+ * @param {string} target - API target: 'default', 'agencia', or 'cliente'
114
+ */
115
+ setApiTarget(target) {
116
+ const validTargets = ['default', 'agencia', 'cliente'];
117
+ if (!validTargets.includes(target.toLowerCase())) {
118
+ throw new Error(
119
+ `Invalid API target '${target}'. Must be one of: ${validTargets.join(', ')}`,
120
+ );
121
+ }
122
+ this.api.activeTarget = target.toLowerCase();
123
+ const config = this.getApiConfig(target.toLowerCase());
124
+ console.log(`🎯 API target set to: ${target} → ${config.baseUrl}`);
125
+ }
126
+
127
+ /**
128
+ * Set source and target APIs for cross-tenant operations
129
+ * @param {string} sourceTarget - Source API target (for reading data)
130
+ * @param {string} targetTarget - Target API target (for writing data)
131
+ */
132
+ setCrossTenantTargets(sourceTarget, targetTarget) {
133
+ const validTargets = ['default', 'agencia', 'cliente'];
134
+
135
+ if (!validTargets.includes(sourceTarget.toLowerCase())) {
136
+ throw new Error(
137
+ `Invalid source API target '${sourceTarget}'. Must be one of: ${validTargets.join(', ')}`,
138
+ );
139
+ }
140
+ if (!validTargets.includes(targetTarget.toLowerCase())) {
141
+ throw new Error(
142
+ `Invalid target API target '${targetTarget}'. Must be one of: ${validTargets.join(', ')}`,
143
+ );
144
+ }
145
+
146
+ this.api.sourceTarget = sourceTarget.toLowerCase();
147
+ this.api.targetTarget = targetTarget.toLowerCase();
148
+
149
+ const sourceConfig = this.getApiConfig(sourceTarget.toLowerCase());
150
+ const targetConfig = this.getApiConfig(targetTarget.toLowerCase());
151
+
152
+ console.log(`🔗 Cross-tenant mode enabled:`);
153
+ console.log(
154
+ ` 📖 Source (read): ${sourceTarget} → ${sourceConfig.baseUrl}`,
155
+ );
156
+ console.log(
157
+ ` 📝 Target (write): ${targetTarget} → ${targetConfig.baseUrl}`,
158
+ );
159
+ }
160
+
161
+ /**
162
+ * Get source API config for cross-tenant operations
163
+ * @returns {Object} Source API configuration
164
+ */
165
+ getSourceApiConfig() {
166
+ if (this.api.sourceTarget) {
167
+ return this.getApiConfig(this.api.sourceTarget);
168
+ }
169
+ return this.getApiConfig();
170
+ }
171
+
172
+ /**
173
+ * Get target API config for cross-tenant operations
174
+ * @returns {Object} Target API configuration
175
+ */
176
+ getTargetApiConfig() {
177
+ if (this.api.targetTarget) {
178
+ return this.getApiConfig(this.api.targetTarget);
179
+ }
180
+ return this.getApiConfig();
181
+ }
182
+
183
+ /**
184
+ * Check if cross-tenant mode is enabled
185
+ * @returns {boolean}
186
+ */
187
+ isCrossTenantMode() {
188
+ return !!(
189
+ this.api.sourceTarget &&
190
+ this.api.targetTarget &&
191
+ this.api.sourceTarget !== this.api.targetTarget
192
+ );
193
+ }
194
+
195
+ /**
196
+ * Get API target by RFC
197
+ * @param {string} rfc - The RFC to find API target for
198
+ * @returns {string|null} The API target name or null if not found
199
+ */
200
+ getApiTargetByRfc(rfc) {
201
+ if (!rfc) return null;
202
+
203
+ const normalizedRfc = rfc.toUpperCase();
204
+ for (const [targetName, targetConfig] of Object.entries(this.api.targets)) {
205
+ if (
206
+ targetConfig.rfc &&
207
+ targetConfig.rfc.toUpperCase() === normalizedRfc
208
+ ) {
209
+ return targetName;
210
+ }
211
+ }
212
+ return null;
213
+ }
214
+
61
215
  /**
62
216
  * Load upload configuration
63
217
  * @private
@@ -114,12 +268,66 @@ class Config {
114
268
  };
115
269
  }
116
270
 
271
+ /**
272
+ * Load watch configuration
273
+ * @private
274
+ */
275
+ #loadWatchConfig() {
276
+ // Parse directory configs from JSON format
277
+ let directoryConfigs = {};
278
+ if (process.env.WATCH_DIRECTORY_CONFIGS) {
279
+ try {
280
+ directoryConfigs = JSON.parse(process.env.WATCH_DIRECTORY_CONFIGS);
281
+ } catch (error) {
282
+ console.warn(
283
+ '⚠️ Invalid JSON in WATCH_DIRECTORY_CONFIGS, using empty config',
284
+ );
285
+ directoryConfigs = {};
286
+ }
287
+ }
288
+
289
+ // Fallback to WATCH_DIRECTORIES for backward compatibility
290
+ let directories = [];
291
+ if (
292
+ Object.keys(directoryConfigs).length === 0 &&
293
+ process.env.WATCH_DIRECTORIES
294
+ ) {
295
+ directories = process.env.WATCH_DIRECTORIES.split(',')
296
+ .map((d) => d.trim())
297
+ .filter(Boolean);
298
+ }
299
+
300
+ // Parse ignore patterns
301
+ const ignorePatterns =
302
+ process.env.WATCH_IGNORE_PATTERNS?.split(',')
303
+ .map((p) => p.trim())
304
+ .filter(Boolean) || [];
305
+
306
+ return {
307
+ enabled: process.env.WATCH_ENABLED === 'true' || false,
308
+ directories,
309
+ directoryConfigs,
310
+ strategy: process.env.WATCH_STRATEGY || 'batch',
311
+ debounceMs: parseInt(process.env.WATCH_DEBOUNCE_MS) || 1000,
312
+ batchSize: parseInt(process.env.WATCH_BATCH_SIZE) || 10,
313
+ usePolling: process.env.WATCH_USE_POLLING === 'true' || false,
314
+ pollInterval: parseInt(process.env.WATCH_POLL_INTERVAL) || 100,
315
+ stabilityThreshold:
316
+ parseInt(process.env.WATCH_STABILITY_THRESHOLD) || 300,
317
+ ignorePatterns,
318
+ autoDetect: process.env.WATCH_AUTO_DETECT === 'true' || false,
319
+ autoOrganize: process.env.WATCH_AUTO_ORGANIZE === 'true' || false,
320
+ };
321
+ }
322
+
117
323
  /**
118
324
  * Check if API mode is available
325
+ * @param {string} target - Optional API target to check
119
326
  * @returns {boolean}
120
327
  */
121
- isApiModeAvailable() {
122
- return !!(this.api.baseUrl && this.api.token);
328
+ isApiModeAvailable(target = null) {
329
+ const config = this.getApiConfig(target);
330
+ return !!(config.baseUrl && config.token);
123
331
  }
124
332
 
125
333
  /**
@@ -181,6 +389,66 @@ class Config {
181
389
  }
182
390
  return this.upload.basePath;
183
391
  }
392
+
393
+ /**
394
+ * Get watch directories with validation
395
+ * @param {string[]} cliDirs - Directories from CLI options
396
+ * @returns {string[]} Validated watch directories
397
+ * @throws {Error} If no directories are configured
398
+ */
399
+ getWatchDirectories(cliDirs = null) {
400
+ // CLI options take precedence over environment variables
401
+ const directories =
402
+ cliDirs && cliDirs.length > 0 ? cliDirs : this.watch.directories;
403
+
404
+ if (!directories || directories.length === 0) {
405
+ throw new Error(
406
+ '⚠️ No watch directories configured. Please use --directories option or set WATCH_DIRECTORIES environment variable.',
407
+ );
408
+ }
409
+
410
+ return directories;
411
+ }
412
+
413
+ /**
414
+ * Get watch configuration with validation
415
+ * @returns {Object} Watch configuration object
416
+ */
417
+ getWatchConfig() {
418
+ return {
419
+ ...this.watch,
420
+ };
421
+ }
422
+
423
+ /**
424
+ * Validate watch configuration
425
+ * @param {string[]} directories - Directories to validate
426
+ * @throws {Error} If configuration is invalid
427
+ */
428
+ validateWatchConfig(directories) {
429
+ if (!directories || directories.length === 0) {
430
+ throw new Error(
431
+ 'At least one directory must be specified for watch mode',
432
+ );
433
+ }
434
+
435
+ // Validate strategy
436
+ const validStrategies = ['individual', 'batch', 'full-structure'];
437
+ if (!validStrategies.includes(this.watch.strategy)) {
438
+ throw new Error(
439
+ `Invalid watch strategy: ${this.watch.strategy}. Must be one of: ${validStrategies.join(', ')}`,
440
+ );
441
+ }
442
+
443
+ // Validate debounce and batch size
444
+ if (this.watch.debounceMs < 0) {
445
+ throw new Error('Debounce time must be >= 0');
446
+ }
447
+
448
+ if (this.watch.batchSize < 1) {
449
+ throw new Error('Batch size must be >= 1');
450
+ }
451
+ }
184
452
  }
185
453
 
186
454
  // Export singleton instance
@@ -1,5 +1,6 @@
1
1
  // Import all document type definitions
2
2
  import { pedimentoSimplificadoDefinition } from './document-types/pedimento-simplificado.js';
3
+ import { supportDocumentDefinition } from './document-types/support-document.js';
3
4
 
4
5
  // Document type definitions and extraction utilities
5
6
  // Ported from TypeScript to JavaScript for Node.js
@@ -33,6 +34,7 @@ export class DocumentTypeDefinition {
33
34
  // Registry of all document types
34
35
  const documentTypes = [
35
36
  pedimentoSimplificadoDefinition,
37
+ supportDocumentDefinition,
36
38
  // Add more document types here as needed
37
39
  ];
38
40
 
@@ -0,0 +1,200 @@
1
+ import { FieldResult } from '../document-type-shared.js';
2
+
3
+ /**
4
+ * Support Document Definition
5
+ * Detects XML and other supporting documents related to customs operations
6
+ * These documents are typically metadata or supporting files that accompany pedimento simplificado
7
+ */
8
+ export const supportDocumentDefinition = {
9
+ type: 'support_document',
10
+ extensions: ['xml', 'txt', 'json'],
11
+ match: (source) => {
12
+ // Detect SOAP/XML structures common in customs systems
13
+ const soapClues = [
14
+ /soapenv:Envelope/i,
15
+ /xmlns:soapenv=/i,
16
+ /solicitarRecibirCoveServicio/i,
17
+ /tipoOperacion/i,
18
+ /patenteAduanal/i,
19
+ ];
20
+
21
+ // Detect customs-related metadata
22
+ const customsClues = [/rfc|RFC/, /patente|aduana|customs|pedimento/i];
23
+
24
+ const soapFound = soapClues.filter((clue) => clue.test(source)).length;
25
+ const customsFound = customsClues.filter((clue) =>
26
+ clue.test(source),
27
+ ).length;
28
+
29
+ // Must have SOAP structure OR customs metadata
30
+ return soapFound >= 2 || customsFound >= 2;
31
+ },
32
+
33
+ extractNumPedimento: (source, fields) => {
34
+ // Try to extract pedimento from various XML/text formats
35
+ const patterns = [
36
+ // SOAP format with tipoOperacion
37
+ /tipoOperacion[^>]*>([A-Z]{4}[^<]*)<\/oxml:tipoOperacion/i,
38
+ // Pedimento number in XML
39
+ /numPedimento[^>]*>(\d{15})<\/\w+:numPedimento/i,
40
+ // Generic pattern
41
+ /pedimento[:\s]*(\d{15})/i,
42
+ ];
43
+
44
+ for (const pattern of patterns) {
45
+ const match = source.match(pattern);
46
+ if (match) {
47
+ return match[1];
48
+ }
49
+ }
50
+
51
+ return null;
52
+ },
53
+
54
+ extractPedimentoYear: (source, fields) => {
55
+ const numPedimento = fields?.find((f) => f.name === 'numPedimento')?.value;
56
+ if (!numPedimento) {
57
+ // Try to extract year from date in XML
58
+ const dateMatch = source.match(/(\d{4})-\d{2}-\d{2}/);
59
+ if (dateMatch) {
60
+ return parseInt(dateMatch[1], 10);
61
+ }
62
+ return null;
63
+ }
64
+
65
+ const year = parseInt(numPedimento.substring(0, 2), 10);
66
+ return year < 50 ? year + 2000 : year + 1900;
67
+ },
68
+
69
+ extractors: [
70
+ // RFC (Registro Federal de Contribuyentes)
71
+ {
72
+ field: 'rfc',
73
+ extract: (source) => {
74
+ const patterns = [
75
+ /rfc[^>]*>([A-ZÑ&]{3,4}\d{6}[A-Z0-9]{3})<\/\w+:rfc/i,
76
+ /rfcConsulta[^>]*>([A-ZÑ&]{3,4}\d{6}[A-Z0-9]{3})<\/\w+:rfcConsulta/i,
77
+ /RFC[:\s]*([A-ZÑ&]{3,4}\d{6}[A-Z0-9]{3})/,
78
+ ];
79
+
80
+ for (const pattern of patterns) {
81
+ const match = source.match(pattern);
82
+ if (match) {
83
+ return new FieldResult('rfc', true, match[1]);
84
+ }
85
+ }
86
+
87
+ return new FieldResult('rfc', false, null);
88
+ },
89
+ },
90
+
91
+ // Patente Aduanal
92
+ {
93
+ field: 'patente',
94
+ extract: (source) => {
95
+ const patterns = [
96
+ /patenteAduanal[^>]*>(\d{4})<\/\w+:patenteAduanal/i,
97
+ /patente[:\s]*(\d{4})/i,
98
+ ];
99
+
100
+ for (const pattern of patterns) {
101
+ const match = source.match(pattern);
102
+ if (match) {
103
+ return new FieldResult('patente', true, match[1]);
104
+ }
105
+ }
106
+
107
+ return new FieldResult('patente', false, null);
108
+ },
109
+ },
110
+
111
+ // Aduana
112
+ {
113
+ field: 'aduanaEntradaSalida',
114
+ extract: (source) => {
115
+ const patterns = [
116
+ /aduanaEntradaSalida[^>]*>(\d{1,2})<\/\w+:aduanaEntradaSalida/i,
117
+ /aduana[:\s]*(\d{1,2})/i,
118
+ ];
119
+
120
+ for (const pattern of patterns) {
121
+ const match = source.match(pattern);
122
+ if (match) {
123
+ return new FieldResult('aduanaEntradaSalida', true, match[1]);
124
+ }
125
+ }
126
+
127
+ return new FieldResult('aduanaEntradaSalida', false, null);
128
+ },
129
+ },
130
+
131
+ // Pedimento Number
132
+ {
133
+ field: 'numPedimento',
134
+ extract: (source) => {
135
+ const patterns = [
136
+ /numPedimento[^>]*>(\d{15})<\/\w+:numPedimento/i,
137
+ /pedimento[:\s]*(\d{15})/i,
138
+ ];
139
+
140
+ for (const pattern of patterns) {
141
+ const match = source.match(pattern);
142
+ if (match) {
143
+ return new FieldResult('numPedimento', true, match[1]);
144
+ }
145
+ }
146
+
147
+ return new FieldResult('numPedimento', false, null);
148
+ },
149
+ },
150
+
151
+ // Email/Contact
152
+ {
153
+ field: 'email',
154
+ extract: (source) => {
155
+ const match = source.match(/[\w.-]+@[\w.-]+\.\w+/);
156
+ return new FieldResult('email', !!match, match ? match[0] : null);
157
+ },
158
+ },
159
+
160
+ // Type of Operation
161
+ {
162
+ field: 'tipoOperacion',
163
+ extract: (source) => {
164
+ const patterns = [
165
+ /tipoOperacion[^>]*>([A-Z]{4}[^<]*)<\/\w+:tipoOperacion/i,
166
+ /tipoOperacion[:\s]*([A-Z]{4})/i,
167
+ ];
168
+
169
+ for (const pattern of patterns) {
170
+ const match = source.match(pattern);
171
+ if (match) {
172
+ return new FieldResult('tipoOperacion', true, match[1]);
173
+ }
174
+ }
175
+
176
+ return new FieldResult('tipoOperacion', false, null);
177
+ },
178
+ },
179
+
180
+ // Document Date
181
+ {
182
+ field: 'documentDate',
183
+ extract: (source) => {
184
+ const patterns = [
185
+ /fechaExpedicion[^>]*>(\d{4}-\d{2}-\d{2})<\/\w+:fechaExpedicion/i,
186
+ /fecha[:\s]*(\d{4}-\d{2}-\d{2})/i,
187
+ ];
188
+
189
+ for (const pattern of patterns) {
190
+ const match = source.match(pattern);
191
+ if (match) {
192
+ return new FieldResult('documentDate', true, match[1]);
193
+ }
194
+ }
195
+
196
+ return new FieldResult('documentDate', false, null);
197
+ },
198
+ },
199
+ ],
200
+ };
@@ -89,6 +89,7 @@ export class FileDetectionService {
89
89
  fields: [],
90
90
  detectedPedimento: null,
91
91
  detectedPedimentoYear: null,
92
+ rfc: null,
92
93
  arelaPath: null,
93
94
  text: '',
94
95
  error: `Unsupported file type: ${fileExtension}`,
@@ -102,6 +103,7 @@ export class FileDetectionService {
102
103
  fields: [],
103
104
  detectedPedimento: null,
104
105
  detectedPedimentoYear: null,
106
+ rfc: null,
105
107
  arelaPath: null,
106
108
  text: '',
107
109
  error: 'No text could be extracted from file',
@@ -112,6 +114,9 @@ export class FileDetectionService {
112
114
  const [detectedType, fields, detectedPedimento, detectedPedimentoYear] =
113
115
  extractDocumentFields(text, fileExtension, filePath);
114
116
 
117
+ // Extract RFC from fields
118
+ const rfc = fields?.find((f) => f.name === 'rfc')?.value ?? null;
119
+
115
120
  // Compose arela_path for pedimento_simplificado documents
116
121
  const arelaPath = composeArelaPath(
117
122
  detectedType,
@@ -125,6 +130,7 @@ export class FileDetectionService {
125
130
  fields,
126
131
  detectedPedimento,
127
132
  detectedPedimentoYear,
133
+ rfc,
128
134
  arelaPath,
129
135
  text,
130
136
  error: null,
@@ -135,6 +141,7 @@ export class FileDetectionService {
135
141
  detectedType: null,
136
142
  fields: [],
137
143
  detectedPedimento: null,
144
+ rfc: null,
138
145
  detectedPedimentoYear: null,
139
146
  arelaPath: null,
140
147
  text: '',
@@ -191,7 +198,8 @@ export class FileDetectionService {
191
198
  */
192
199
  isSupportedFileType(filePath) {
193
200
  const fileExtension = path.extname(filePath).toLowerCase().replace('.', '');
194
- const supportedExtensions = ['pdf'];
201
+ // Support PDF (main documents), XML (metadata/supporting docs), and TXT
202
+ const supportedExtensions = ['pdf', 'xml', 'txt'];
195
203
  return supportedExtensions.includes(fileExtension);
196
204
  }
197
205