@arela/uploader 1.0.6 → 1.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.template +14 -0
- package/package.json +1 -1
- package/src/commands/IdentifyCommand.js +16 -16
- package/src/commands/PropagateCommand.js +4 -2
- package/src/commands/PushCommand.js +64 -25
- package/src/commands/ScanCommand.js +7 -2
- package/src/config/config.js +8 -2
- package/src/index.js +24 -0
- package/src/services/ScanApiService.js +6 -2
- package/tests/commands/IdentifyCommand.test.js +2 -2
- package/.env.local +0 -316
package/.env.template
CHANGED
|
@@ -9,6 +9,15 @@
|
|
|
9
9
|
ARELA_API_URL=https://your-arela-api-url.com
|
|
10
10
|
ARELA_API_TOKEN=your-api-token-here
|
|
11
11
|
|
|
12
|
+
# API Agencia - Configura aquí la URL y Token de la agencia activa
|
|
13
|
+
ARELA_API_AGENCIA_URL=https://agencia-api-example.com
|
|
14
|
+
ARELA_API_AGENCIA_TOKEN=your-agencia-api-token-here
|
|
15
|
+
|
|
16
|
+
# API Cliente - Configura aquí la URL y Token del cliente activo
|
|
17
|
+
ARELA_API_CLIENTE_URL=https://cliente-api-example.com
|
|
18
|
+
ARELA_API_CLIENTE_TOKEN=your-cliente-api-token-here
|
|
19
|
+
|
|
20
|
+
|
|
12
21
|
# Supabase Configuration (fallback)
|
|
13
22
|
SUPABASE_URL=https://your-supabase-url.supabase.co
|
|
14
23
|
SUPABASE_KEY=your-supabase-key-here
|
|
@@ -77,6 +86,11 @@ PUSH_UPLOAD_BATCH_SIZE=10
|
|
|
77
86
|
# Examples: "archivos", "documents", "storage"
|
|
78
87
|
PUSH_BUCKET=arela
|
|
79
88
|
|
|
89
|
+
# Folder structure prefix for uploaded files (optional)
|
|
90
|
+
# This prefix is prepended to the arela_path when uploading files
|
|
91
|
+
# Examples: "agencia/cliente", "2024/docs", "imports/batch1"
|
|
92
|
+
PUSH_FOLDER_STRUCTURE=
|
|
93
|
+
|
|
80
94
|
# =============================================================================
|
|
81
95
|
# PERFORMANCE OPTIMIZATION FOR MULTIPLE API REPLICAS
|
|
82
96
|
# =============================================================================
|
package/package.json
CHANGED
|
@@ -42,23 +42,20 @@ export class IdentifyCommand {
|
|
|
42
42
|
// Validate scan configuration (need same config as scan command)
|
|
43
43
|
appConfig.validateScanConfig();
|
|
44
44
|
|
|
45
|
-
//
|
|
45
|
+
// Determine API target
|
|
46
|
+
const apiTarget = options.api || 'default';
|
|
47
|
+
|
|
48
|
+
// Import ScanApiService dynamically and initialize with target
|
|
46
49
|
const { default: ScanApiService } = await import(
|
|
47
50
|
'../services/ScanApiService.js'
|
|
48
51
|
);
|
|
49
|
-
this.scanApiService = new ScanApiService();
|
|
50
|
-
|
|
51
|
-
// Set API target if specified
|
|
52
|
-
if (options.api) {
|
|
53
|
-
appConfig.setApiTarget(options.api);
|
|
54
|
-
this.scanApiService = new ScanApiService();
|
|
55
|
-
}
|
|
52
|
+
this.scanApiService = new ScanApiService(apiTarget);
|
|
56
53
|
|
|
57
54
|
const scanConfig = appConfig.getScanConfig();
|
|
58
55
|
const batchSize = parseInt(options.batchSize) || 100;
|
|
59
56
|
|
|
60
57
|
logger.info('🔍 Starting arela identify command');
|
|
61
|
-
logger.info(`🎯 API Target: ${
|
|
58
|
+
logger.info(`🎯 API Target: ${apiTarget}`);
|
|
62
59
|
logger.info(`📦 Batch Size: ${batchSize}`);
|
|
63
60
|
|
|
64
61
|
// Fetch all tables for this instance
|
|
@@ -273,7 +270,7 @@ export class IdentifyCommand {
|
|
|
273
270
|
arelaPath: null,
|
|
274
271
|
detectionError:
|
|
275
272
|
'FILE_NOT_FOUND: File does not exist on filesystem. May have been moved or deleted after scan.',
|
|
276
|
-
|
|
273
|
+
isPedimento: null, // Unknown - can't determine
|
|
277
274
|
};
|
|
278
275
|
}
|
|
279
276
|
|
|
@@ -289,7 +286,7 @@ export class IdentifyCommand {
|
|
|
289
286
|
rfc: null,
|
|
290
287
|
arelaPath: null,
|
|
291
288
|
detectionError: `FILE_TOO_LARGE: File size ${(stats.size / 1024 / 1024).toFixed(2)}MB exceeds ${maxSizeBytes / 1024 / 1024}MB limit.`,
|
|
292
|
-
|
|
289
|
+
isPedimento: null, // Unknown - can't determine
|
|
293
290
|
};
|
|
294
291
|
}
|
|
295
292
|
|
|
@@ -306,19 +303,22 @@ export class IdentifyCommand {
|
|
|
306
303
|
rfc: result.rfc,
|
|
307
304
|
arelaPath: result.arelaPath,
|
|
308
305
|
detectionError: result.error,
|
|
309
|
-
|
|
306
|
+
isPedimento: true, // Confirmed pedimento
|
|
310
307
|
};
|
|
311
308
|
}
|
|
312
309
|
|
|
313
310
|
// If no detection, determine if it's definitely not a pedimento
|
|
314
311
|
// This helps avoid re-processing files we know aren't pedimentos
|
|
315
|
-
const
|
|
312
|
+
const isDefinitelyNotPedimento = this.#isDefinitelyNotPedimento(
|
|
313
|
+
result,
|
|
314
|
+
file,
|
|
315
|
+
);
|
|
316
316
|
|
|
317
317
|
// Build descriptive error message
|
|
318
318
|
let detectionError = null;
|
|
319
319
|
if (result.error) {
|
|
320
320
|
detectionError = `DETECTION_ERROR: ${result.error}`;
|
|
321
|
-
} else if (
|
|
321
|
+
} else if (isDefinitelyNotPedimento) {
|
|
322
322
|
detectionError =
|
|
323
323
|
'NOT_PEDIMENTO: File does not match pedimento-simplificado pattern. Missing key markers: "FORMA SIMPLIFICADA DE PEDIMENTO".';
|
|
324
324
|
} else {
|
|
@@ -340,7 +340,7 @@ export class IdentifyCommand {
|
|
|
340
340
|
rfc: result.rfc,
|
|
341
341
|
arelaPath: result.arelaPath,
|
|
342
342
|
detectionError,
|
|
343
|
-
|
|
343
|
+
isPedimento: isDefinitelyNotPedimento ? false : null, // false = not pedimento, null = unknown
|
|
344
344
|
};
|
|
345
345
|
} catch (error) {
|
|
346
346
|
logger.warn(
|
|
@@ -367,7 +367,7 @@ export class IdentifyCommand {
|
|
|
367
367
|
rfc: null,
|
|
368
368
|
arelaPath: null,
|
|
369
369
|
detectionError: `${errorCategory}: ${error.message}`,
|
|
370
|
-
|
|
370
|
+
isPedimento: null, // Unknown - error occurred
|
|
371
371
|
};
|
|
372
372
|
}
|
|
373
373
|
}),
|
|
@@ -39,8 +39,10 @@ export class PropagateCommand {
|
|
|
39
39
|
// Step 1: Validate configuration
|
|
40
40
|
await this.#validateConfiguration();
|
|
41
41
|
|
|
42
|
-
// Step 2: Initialize API service
|
|
43
|
-
this.
|
|
42
|
+
// Step 2: Initialize API service with configured target
|
|
43
|
+
const apiTarget = this.options.api || 'default';
|
|
44
|
+
this.scanApiService = new ScanApiService(apiTarget);
|
|
45
|
+
console.log(`🎯 API Target: ${apiTarget}`);
|
|
44
46
|
|
|
45
47
|
// Step 3: Fetch all tables for this instance
|
|
46
48
|
const scanConfig = appConfig.getScanConfig();
|
|
@@ -15,7 +15,8 @@ import appConfig from '../config/config.js';
|
|
|
15
15
|
*/
|
|
16
16
|
export class PushCommand {
|
|
17
17
|
constructor() {
|
|
18
|
-
|
|
18
|
+
// ScanApiService will be initialized in execute() with proper API target
|
|
19
|
+
this.scanApiService = null;
|
|
19
20
|
}
|
|
20
21
|
|
|
21
22
|
/**
|
|
@@ -37,26 +38,51 @@ export class PushCommand {
|
|
|
37
38
|
// Get configuration
|
|
38
39
|
const scanConfig = appConfig.getScanConfig();
|
|
39
40
|
const pushConfig = appConfig.getPushConfig();
|
|
40
|
-
const tableName = scanConfig.tableName;
|
|
41
41
|
|
|
42
|
-
//
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
42
|
+
// Override folderStructure from command option if provided
|
|
43
|
+
if (options.folderStructure) {
|
|
44
|
+
// Clean the folder structure: remove leading/trailing slashes
|
|
45
|
+
pushConfig.folderStructure = options.folderStructure
|
|
46
|
+
.trim()
|
|
47
|
+
.replace(/^\/+/, '')
|
|
48
|
+
.replace(/\/+$/, '');
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
-
//
|
|
51
|
+
// Determine API targets for scan (read) and push (upload) operations
|
|
52
|
+
// Priority: explicit scan-api/push-api > source-api/target-api > api > default
|
|
53
|
+
const scanApiTarget =
|
|
54
|
+
options.scanApi || options.sourceApi || options.api || 'default';
|
|
55
|
+
const pushApiTarget =
|
|
56
|
+
options.pushApi || options.targetApi || options.api || scanApiTarget;
|
|
57
|
+
|
|
58
|
+
// Initialize ScanApiService with the scan API target
|
|
59
|
+
this.scanApiService = new ScanApiService(scanApiTarget);
|
|
60
|
+
|
|
61
|
+
// Get upload API configuration for the push target
|
|
52
62
|
const uploadApiConfig = appConfig.getApiConfig(pushApiTarget);
|
|
53
63
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
64
|
+
// Display API configuration
|
|
65
|
+
const isCrossTenant = scanApiTarget !== pushApiTarget;
|
|
66
|
+
if (isCrossTenant) {
|
|
67
|
+
console.log('🔗 Cross-tenant mode enabled:');
|
|
68
|
+
console.log(
|
|
69
|
+
` 📖 Scan API (read): ${scanApiTarget} → ${appConfig.getApiConfig(scanApiTarget).baseUrl}`,
|
|
70
|
+
);
|
|
71
|
+
console.log(
|
|
72
|
+
` 📝 Push API (upload): ${pushApiTarget} → ${uploadApiConfig.baseUrl}`,
|
|
73
|
+
);
|
|
74
|
+
} else {
|
|
75
|
+
console.log(
|
|
76
|
+
`🎯 API Target: ${scanApiTarget} → ${uploadApiConfig.baseUrl}`,
|
|
77
|
+
);
|
|
78
|
+
}
|
|
58
79
|
console.log(`📦 Fetch Batch Size: ${options.batchSize}`);
|
|
59
80
|
console.log(`📤 Upload Batch Size: ${options.uploadBatchSize}`);
|
|
81
|
+
if (pushConfig.folderStructure) {
|
|
82
|
+
console.log(
|
|
83
|
+
`📁 Folder Structure Prefix: ${pushConfig.folderStructure}`,
|
|
84
|
+
);
|
|
85
|
+
}
|
|
60
86
|
|
|
61
87
|
// Apply filters
|
|
62
88
|
const filters = {
|
|
@@ -282,7 +308,6 @@ export class PushCommand {
|
|
|
282
308
|
);
|
|
283
309
|
|
|
284
310
|
// Update counters from API response
|
|
285
|
-
// Note: The CLI endpoint now handles updating the scan table directly
|
|
286
311
|
batchResults.forEach((result) => {
|
|
287
312
|
results.processed++;
|
|
288
313
|
if (result.uploaded) {
|
|
@@ -292,6 +317,17 @@ export class PushCommand {
|
|
|
292
317
|
}
|
|
293
318
|
});
|
|
294
319
|
|
|
320
|
+
// Update scan table via scanApi (supports cross-tenant mode)
|
|
321
|
+
// This is called on scanApi, not pushApi, because scan tables live on scanApi
|
|
322
|
+
try {
|
|
323
|
+
await this.scanApiService.batchUpdateUpload(tableName, batchResults);
|
|
324
|
+
} catch (updateError) {
|
|
325
|
+
logger.error(
|
|
326
|
+
`Failed to update scan table for batch: ${updateError.message}`,
|
|
327
|
+
);
|
|
328
|
+
// Don't fail the entire process, just log the error
|
|
329
|
+
}
|
|
330
|
+
|
|
295
331
|
// Update progress bar
|
|
296
332
|
const elapsed = (Date.now() - results.startTime) / 1000;
|
|
297
333
|
const speed =
|
|
@@ -317,7 +353,7 @@ export class PushCommand {
|
|
|
317
353
|
|
|
318
354
|
/**
|
|
319
355
|
* Upload a batch of files using the new CLI upload endpoint
|
|
320
|
-
*
|
|
356
|
+
* Note: Scan table is updated separately via scanApi after this batch completes
|
|
321
357
|
* @private
|
|
322
358
|
*/
|
|
323
359
|
async #uploadBatchViaCli(tableName, files, uploadApiConfig) {
|
|
@@ -356,8 +392,6 @@ export class PushCommand {
|
|
|
356
392
|
if (!fs.existsSync(file.absolute_path)) {
|
|
357
393
|
result.uploadError =
|
|
358
394
|
'FILE_NOT_FOUND: File does not exist on filesystem';
|
|
359
|
-
// Update the scan table with the error
|
|
360
|
-
await this.scanApiService.batchUpdateUpload(tableName, [result]);
|
|
361
395
|
return result;
|
|
362
396
|
}
|
|
363
397
|
|
|
@@ -365,7 +399,6 @@ export class PushCommand {
|
|
|
365
399
|
const stats = fs.statSync(file.absolute_path);
|
|
366
400
|
if (!stats.isFile()) {
|
|
367
401
|
result.uploadError = 'NOT_A_FILE: Path is not a regular file';
|
|
368
|
-
await this.scanApiService.batchUpdateUpload(tableName, [result]);
|
|
369
402
|
return result;
|
|
370
403
|
}
|
|
371
404
|
|
|
@@ -377,23 +410,29 @@ export class PushCommand {
|
|
|
377
410
|
// Create form data for CLI upload endpoint
|
|
378
411
|
const form = new FormData();
|
|
379
412
|
|
|
380
|
-
//
|
|
381
|
-
//
|
|
382
|
-
|
|
413
|
+
// Build folder structure: optionally prefix with PUSH_FOLDER_STRUCTURE env var
|
|
414
|
+
// arela_path already contains the logical path (RFC/Year/Patente/Aduana/Pedimento)
|
|
415
|
+
// folderStructure from config is the bucket prefix (e.g., "documents" or "uploads/2024")
|
|
416
|
+
let arelaPath = file.arela_path.endsWith('/')
|
|
383
417
|
? file.arela_path.slice(0, -1)
|
|
384
418
|
: file.arela_path;
|
|
385
419
|
|
|
386
|
-
|
|
420
|
+
// Prepend folder structure prefix if configured
|
|
421
|
+
const folderStructure = pushConfig.folderStructure
|
|
422
|
+
? `${pushConfig.folderStructure}/${arelaPath}`
|
|
423
|
+
: arelaPath;
|
|
387
424
|
|
|
388
|
-
// Create a read stream with the
|
|
425
|
+
// Create a read stream with the original filename (no encoding needed)
|
|
389
426
|
const fileStream = fs.createReadStream(file.absolute_path);
|
|
390
427
|
form.append('files', fileStream, {
|
|
391
|
-
filename:
|
|
428
|
+
filename: file.file_name,
|
|
392
429
|
contentType: this.#getMimeType(file.file_extension),
|
|
393
430
|
});
|
|
394
431
|
|
|
395
432
|
// Add required fields for CLI upload
|
|
396
433
|
form.append('tableName', tableName);
|
|
434
|
+
form.append('fileId', file.id);
|
|
435
|
+
form.append('folderStructure', folderStructure);
|
|
397
436
|
form.append('rfc', file.rfc);
|
|
398
437
|
form.append('bucket', pushConfig.bucket);
|
|
399
438
|
form.append('autoDetect', 'true');
|
|
@@ -26,6 +26,7 @@ export class ScanCommand {
|
|
|
26
26
|
* Execute the scan command
|
|
27
27
|
* @param {Object} options - Command options
|
|
28
28
|
* @param {boolean} options.countFirst - Count files first for percentage-based progress
|
|
29
|
+
* @param {string} options.api - API target: 'default', 'agencia', or 'cliente'
|
|
29
30
|
*/
|
|
30
31
|
async execute(options = {}) {
|
|
31
32
|
const startTime = Date.now();
|
|
@@ -34,17 +35,21 @@ export class ScanCommand {
|
|
|
34
35
|
// Validate scan configuration
|
|
35
36
|
appConfig.validateScanConfig();
|
|
36
37
|
|
|
37
|
-
//
|
|
38
|
+
// Determine API target
|
|
39
|
+
const apiTarget = options.api || 'default';
|
|
40
|
+
|
|
41
|
+
// Import ScanApiService dynamically and initialize with target
|
|
38
42
|
const { default: ScanApiService } = await import(
|
|
39
43
|
'../services/ScanApiService.js'
|
|
40
44
|
);
|
|
41
|
-
this.scanApiService = new ScanApiService();
|
|
45
|
+
this.scanApiService = new ScanApiService(apiTarget);
|
|
42
46
|
|
|
43
47
|
const scanConfig = appConfig.getScanConfig();
|
|
44
48
|
// Ensure basePath is absolute for scan operations
|
|
45
49
|
const basePath = PathNormalizer.toAbsolutePath(appConfig.getBasePath());
|
|
46
50
|
|
|
47
51
|
logger.info('🔍 Starting arela scan command');
|
|
52
|
+
logger.info(`🎯 API Target: ${apiTarget}`);
|
|
48
53
|
logger.info(`📦 Company: ${scanConfig.companySlug}`);
|
|
49
54
|
logger.info(`🖥️ Server: ${scanConfig.serverId}`);
|
|
50
55
|
logger.info(`📂 Base Path: ${basePath}`);
|
package/src/config/config.js
CHANGED
|
@@ -34,10 +34,10 @@ class Config {
|
|
|
34
34
|
const __dirname = path.dirname(__filename);
|
|
35
35
|
const packageJsonPath = path.resolve(__dirname, '../../package.json');
|
|
36
36
|
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
|
|
37
|
-
return packageJson.version || '1.0.
|
|
37
|
+
return packageJson.version || '1.0.8';
|
|
38
38
|
} catch (error) {
|
|
39
39
|
console.warn('⚠️ Could not read package.json version, using fallback');
|
|
40
|
-
return '1.0.
|
|
40
|
+
return '1.0.8';
|
|
41
41
|
}
|
|
42
42
|
}
|
|
43
43
|
|
|
@@ -313,6 +313,11 @@ class Config {
|
|
|
313
313
|
.map((s) => parseInt(s.trim(), 10))
|
|
314
314
|
.filter((y) => !isNaN(y));
|
|
315
315
|
|
|
316
|
+
// Clean folder structure: remove leading/trailing slashes
|
|
317
|
+
const folderStructure = process.env.PUSH_FOLDER_STRUCTURE?.trim()
|
|
318
|
+
.replace(/^\/+/, '')
|
|
319
|
+
.replace(/\/+$/, '');
|
|
320
|
+
|
|
316
321
|
return {
|
|
317
322
|
rfcs: pushRfcs,
|
|
318
323
|
years: pushYears,
|
|
@@ -320,6 +325,7 @@ class Config {
|
|
|
320
325
|
uploadBatchSize: parseInt(process.env.PUSH_UPLOAD_BATCH_SIZE) || 10,
|
|
321
326
|
bucket:
|
|
322
327
|
process.env.PUSH_BUCKET || process.env.SUPABASE_BUCKET || 'archivos',
|
|
328
|
+
folderStructure: folderStructure || '', // Prefix for storage path (e.g., "documents" or "uploads/2024")
|
|
323
329
|
};
|
|
324
330
|
}
|
|
325
331
|
|
package/src/index.js
CHANGED
|
@@ -381,6 +381,14 @@ class ArelaUploaderCLI {
|
|
|
381
381
|
'--push-api <target>',
|
|
382
382
|
'API for uploading files: default|agencia|cliente',
|
|
383
383
|
)
|
|
384
|
+
.option(
|
|
385
|
+
'--source-api <target>',
|
|
386
|
+
'Source API for reading data (cross-tenant mode): agencia|cliente',
|
|
387
|
+
)
|
|
388
|
+
.option(
|
|
389
|
+
'--target-api <target>',
|
|
390
|
+
'Target API for uploading files (cross-tenant mode): agencia|cliente',
|
|
391
|
+
)
|
|
384
392
|
.option(
|
|
385
393
|
'-b, --batch-size <size>',
|
|
386
394
|
'Number of files to fetch per batch',
|
|
@@ -399,9 +407,25 @@ class ArelaUploaderCLI {
|
|
|
399
407
|
'--years <years>',
|
|
400
408
|
'Comma-separated years to filter (overrides PUSH_YEARS env var)',
|
|
401
409
|
)
|
|
410
|
+
.option(
|
|
411
|
+
'--folder-structure <path>',
|
|
412
|
+
'Storage path prefix (overrides PUSH_FOLDER_STRUCTURE env var)',
|
|
413
|
+
)
|
|
402
414
|
.option('--show-stats', 'Show performance statistics')
|
|
403
415
|
.action(async (options) => {
|
|
404
416
|
try {
|
|
417
|
+
// Handle cross-tenant mode (source and target APIs)
|
|
418
|
+
// Map source-api/target-api to scan-api/push-api for consistency
|
|
419
|
+
if (options.sourceApi && options.targetApi) {
|
|
420
|
+
appConfig.setCrossTenantTargets(
|
|
421
|
+
options.sourceApi,
|
|
422
|
+
options.targetApi,
|
|
423
|
+
);
|
|
424
|
+
// Also set scan-api and push-api for PushCommand compatibility
|
|
425
|
+
options.scanApi = options.sourceApi;
|
|
426
|
+
options.pushApi = options.targetApi;
|
|
427
|
+
}
|
|
428
|
+
|
|
405
429
|
// Parse comma-separated values
|
|
406
430
|
if (options.rfcs) {
|
|
407
431
|
options.rfcs = options.rfcs
|
|
@@ -10,8 +10,12 @@ import logger from './LoggingService.js';
|
|
|
10
10
|
* Handles API communication for the arela scan command
|
|
11
11
|
*/
|
|
12
12
|
export class ScanApiService {
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
/**
|
|
14
|
+
* @param {string|null} apiTarget - API target: 'default', 'agencia', 'cliente', or null (uses active target)
|
|
15
|
+
*/
|
|
16
|
+
constructor(apiTarget = null) {
|
|
17
|
+
this.apiTarget = apiTarget;
|
|
18
|
+
const apiConfig = appConfig.getApiConfig(apiTarget);
|
|
15
19
|
this.baseUrl = apiConfig.baseUrl;
|
|
16
20
|
this.token = apiConfig.token;
|
|
17
21
|
|
|
@@ -539,7 +539,7 @@ describe('IdentifyCommand', () => {
|
|
|
539
539
|
expect.any(String),
|
|
540
540
|
expect.arrayContaining([
|
|
541
541
|
expect.objectContaining({
|
|
542
|
-
|
|
542
|
+
isPedimento: false, // Confirmed NOT a pedimento
|
|
543
543
|
}),
|
|
544
544
|
])
|
|
545
545
|
);
|
|
@@ -561,7 +561,7 @@ describe('IdentifyCommand', () => {
|
|
|
561
561
|
expect.any(String),
|
|
562
562
|
expect.arrayContaining([
|
|
563
563
|
expect.objectContaining({
|
|
564
|
-
|
|
564
|
+
isPedimento: null, // Unknown - might be pedimento but missing fields
|
|
565
565
|
}),
|
|
566
566
|
])
|
|
567
567
|
);
|
package/.env.local
DELETED
|
@@ -1,316 +0,0 @@
|
|
|
1
|
-
# ============================================
|
|
2
|
-
# ARELA UPLOADER CONFIGURATION
|
|
3
|
-
# ============================================
|
|
4
|
-
|
|
5
|
-
# Localhost Arela API Configuration
|
|
6
|
-
ARELA_API_URL=http://localhost:3010
|
|
7
|
-
ARELA_API_TOKEN=555f1d5c1b5020a132002a6fa201e0074e1b057895776bd33619db0cd26b259b
|
|
8
|
-
|
|
9
|
-
# Localhost Supabase Configuration
|
|
10
|
-
SUPABASE_URL=http://127.0.0.1:54321
|
|
11
|
-
SUPABASE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU
|
|
12
|
-
SUPABASE_BUCKET=arela
|
|
13
|
-
|
|
14
|
-
# ARELA_API_TARGET=default
|
|
15
|
-
|
|
16
|
-
# Localhost Upload Configuration
|
|
17
|
-
UPLOAD_BASE_PATH=./sample
|
|
18
|
-
UPLOAD_SOURCES=2023|2024
|
|
19
|
-
UPLOAD_RFCS=DTM090831LF0|AKS151005E46|IMS030409FZ0|RDG1107154L7|SHP031226BV2|CSM9301219B4|LIN960124HT8|LME971009SW4|AKM9707151B6|FEL000822AG2|FDM060802J54|MTM9807279B4|AUM9207011CA|MMJ0810145N1|ACC010328EQ6|PED781129JT6|CAD890407NK7|SME140411IK7|JME1903121C2|EIJ110429NF9|PTJ080414TM6|TME050503BM4
|
|
20
|
-
# UPLOAD_RFCS=FDM060802J54|CSM9301219B4
|
|
21
|
-
UPLOAD_YEARS=2023|2024|2025
|
|
22
|
-
|
|
23
|
-
# =============================================================================
|
|
24
|
-
# SCAN CONFIGURATION (for arela scan command)
|
|
25
|
-
# =============================================================================
|
|
26
|
-
|
|
27
|
-
# Company identifier for this CLI instance (required)
|
|
28
|
-
# Use a short, descriptive slug for your company/agency/client
|
|
29
|
-
# Examples: "acme_corp", "cliente_123", "agencia_xyz"
|
|
30
|
-
ARELA_COMPANY_SLUG=palco
|
|
31
|
-
|
|
32
|
-
# Server identifier (required)
|
|
33
|
-
# Use a unique ID for each server/NAS where arela-cli is installed
|
|
34
|
-
# Examples: "nas01", "server-mx", "storage-01"
|
|
35
|
-
ARELA_SERVER_ID=local
|
|
36
|
-
|
|
37
|
-
# Base path label (optional, auto-derived from UPLOAD_BASE_PATH if not set)
|
|
38
|
-
# Short label describing the base path being scanned
|
|
39
|
-
# Examples: "data", "documents", "archive"
|
|
40
|
-
ARELA_BASE_PATH_LABEL=
|
|
41
|
-
|
|
42
|
-
# System file patterns to exclude from scan (comma-separated)
|
|
43
|
-
# These files will be filtered before uploading stats to reduce payload
|
|
44
|
-
SCAN_EXCLUDE_PATTERNS=.DS_Store,Thumbs.db,desktop.ini,__pycache__,.pyc,.tmp,.swp,$RECYCLE.BIN,System Volume Information,~$*
|
|
45
|
-
|
|
46
|
-
# Batch size for scan operations (default: 2000 records per API call)
|
|
47
|
-
SCAN_BATCH_SIZE=2000
|
|
48
|
-
|
|
49
|
-
# Directory depth level for creating separate tables (default: 0)
|
|
50
|
-
# 0 = single table for entire base path
|
|
51
|
-
# 1 = one table per first-level subdirectory
|
|
52
|
-
# 2 = one table per second-level subdirectory, etc.
|
|
53
|
-
# Example: with level=1 and base=/data, creates tables for /data/folder1, /data/folder2, etc.
|
|
54
|
-
SCAN_DIRECTORY_LEVEL=0
|
|
55
|
-
|
|
56
|
-
# =============================================================================
|
|
57
|
-
# PUSH CONFIGURATION (for arela push command)
|
|
58
|
-
# =============================================================================
|
|
59
|
-
|
|
60
|
-
# Filter files to upload by RFC (pipe-separated, optional)
|
|
61
|
-
# If not set, all files with arela_path will be uploaded
|
|
62
|
-
# Examples: "RFC123456ABC|RFC789012DEF"
|
|
63
|
-
PUSH_RFCS=
|
|
64
|
-
|
|
65
|
-
# Filter files to upload by year (pipe-separated, optional)
|
|
66
|
-
# If not set, all files with arela_path will be uploaded
|
|
67
|
-
# Examples: "2023|2024|2025"
|
|
68
|
-
PUSH_YEARS=
|
|
69
|
-
|
|
70
|
-
# Batch size for fetching files from database (default: 100)
|
|
71
|
-
PUSH_BATCH_SIZE=100
|
|
72
|
-
|
|
73
|
-
# Concurrent upload batch size (default: 10)
|
|
74
|
-
# Number of files to upload simultaneously
|
|
75
|
-
PUSH_UPLOAD_BATCH_SIZE=10
|
|
76
|
-
|
|
77
|
-
# Storage bucket for uploaded files (optional, defaults to SUPABASE_BUCKET)
|
|
78
|
-
# Examples: "archivos", "documents", "storage"
|
|
79
|
-
PUSH_BUCKET=cli
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
# =============================================================================
|
|
83
|
-
# PERFORMANCE OPTIMIZATION FOR MULTIPLE API REPLICAS
|
|
84
|
-
# =============================================================================
|
|
85
|
-
|
|
86
|
-
# API Connection Configuration
|
|
87
|
-
# Set this to match your number of API replicas (e.g., if you have 10 API instances, set to 10)
|
|
88
|
-
MAX_API_CONNECTIONS=10
|
|
89
|
-
|
|
90
|
-
# API Connection Timeout (milliseconds)
|
|
91
|
-
API_CONNECTION_TIMEOUT=60000
|
|
92
|
-
|
|
93
|
-
# Batch Processing Configuration
|
|
94
|
-
# Files processed concurrently per batch (should be >= MAX_API_CONNECTIONS for best performance)
|
|
95
|
-
BATCH_SIZE=100
|
|
96
|
-
|
|
97
|
-
# Delay between batches (0 for maximum speed)
|
|
98
|
-
BATCH_DELAY=0
|
|
99
|
-
|
|
100
|
-
# Source Processing Concurrency
|
|
101
|
-
# Number of upload sources/folders to process simultaneously
|
|
102
|
-
MAX_CONCURRENT_SOURCES=2
|
|
103
|
-
|
|
104
|
-
# API Retry Configuration
|
|
105
|
-
# Maximum number of retry attempts for failed API requests
|
|
106
|
-
API_MAX_RETRIES=3
|
|
107
|
-
|
|
108
|
-
# Enable exponential backoff for retries (true/false)
|
|
109
|
-
# When true, retry delays increase: 1s, 2s, 4s, 8s, 16s
|
|
110
|
-
# When false, uses fixed delay (API_RETRY_DELAY)
|
|
111
|
-
API_RETRY_EXPONENTIAL_BACKOFF=true
|
|
112
|
-
|
|
113
|
-
# Fixed retry delay in milliseconds (only used if exponential backoff is disabled)
|
|
114
|
-
API_RETRY_DELAY=1000
|
|
115
|
-
|
|
116
|
-
# =============================================================================
|
|
117
|
-
# EXAMPLE CONFIGURATIONS FOR DIFFERENT SCENARIOS
|
|
118
|
-
# =============================================================================
|
|
119
|
-
|
|
120
|
-
# For 10 API Replicas (High Performance Setup):
|
|
121
|
-
# MAX_API_CONNECTIONS=10
|
|
122
|
-
# BATCH_SIZE=100
|
|
123
|
-
# MAX_CONCURRENT_SOURCES=3
|
|
124
|
-
# BATCH_DELAY=0
|
|
125
|
-
|
|
126
|
-
# For 5 API Replicas (Medium Performance Setup):
|
|
127
|
-
# MAX_API_CONNECTIONS=5
|
|
128
|
-
# BATCH_SIZE=50
|
|
129
|
-
# MAX_CONCURRENT_SOURCES=2
|
|
130
|
-
# BATCH_DELAY=0
|
|
131
|
-
|
|
132
|
-
# For 1 API Instance (Single Instance Setup):
|
|
133
|
-
# MAX_API_CONNECTIONS=5
|
|
134
|
-
# BATCH_SIZE=20
|
|
135
|
-
# MAX_CONCURRENT_SOURCES=1
|
|
136
|
-
# BATCH_DELAY=100
|
|
137
|
-
|
|
138
|
-
# =============================================================================
|
|
139
|
-
# LOGGING AND MONITORING
|
|
140
|
-
# =============================================================================
|
|
141
|
-
|
|
142
|
-
# Progress bar update frequency
|
|
143
|
-
PROGRESS_UPDATE_INTERVAL=10
|
|
144
|
-
|
|
145
|
-
# Enable verbose logging (true/false)
|
|
146
|
-
VERBOSE_LOGGING=false
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
# ============================================
|
|
173
|
-
|
|
174
|
-
# # Cloud Service Arela API Configuration
|
|
175
|
-
# # ARELA_API_URL=https://api.aws.arela.com.mx
|
|
176
|
-
# # ARELA_API_TOKEN=6bd75c5b3699ecf19e6726c10ae88ae0528f0b72d6c10f8b284f92563d3822a7
|
|
177
|
-
|
|
178
|
-
# # # Cloud Service Supabase Configuration
|
|
179
|
-
# SUPABASE_URL=https://qlospyfsbwvkskivmsgq.supabase.co
|
|
180
|
-
# SUPABASE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InFsb3NweWZzYnd2a3NraXZtc2dxIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NTU5MDg2NjUsImV4cCI6MjA3MTQ4NDY2NX0.BrqcCLxTmpU6Swl7h3gam6TeW4jVf4WssMbRm0sH7l4
|
|
181
|
-
# SUPABASE_BUCKET=zips
|
|
182
|
-
|
|
183
|
-
# # # Cloud Service Upload Configuration
|
|
184
|
-
# UPLOAD_BASE_PATH=./sample
|
|
185
|
-
# UPLOAD_SOURCES=zips
|
|
186
|
-
# # UPLOAD_RFCS=AKS151005E46|IMS030409FZ0|RDG1107154L7|SHP031226BV2|CSM9301219B4|LIN960124HT8|LME971009SW4|AKM9707151B6|FEL000822AG2|FDM060802J54|MTM9807279B4|AUM9207011CA|MMJ0810145N1|ACC010328EQ6|PED781129JT6|CAD890407NK7|SME140411IK7|JME1903121C2|EIJ110429NF9|PTJ080414TM6|TME050503BM4
|
|
187
|
-
# UPLOAD_RFCS=KTJ931117P55|AUM9207011CA
|
|
188
|
-
# UPLOAD_YEARS=2023|
|
|
189
|
-
|
|
190
|
-
# # =============================================================================
|
|
191
|
-
# # PERFORMANCE OPTIMIZATION FOR MULTIPLE API REPLICAS
|
|
192
|
-
# # =============================================================================
|
|
193
|
-
|
|
194
|
-
# # API Connection Configuration
|
|
195
|
-
# # Set this to match your number of API replicas (e.g., if you have 10 API instances, set to 10)
|
|
196
|
-
# MAX_API_CONNECTIONS=10
|
|
197
|
-
|
|
198
|
-
# # API Connection Timeout (milliseconds)
|
|
199
|
-
# API_CONNECTION_TIMEOUT=60000
|
|
200
|
-
|
|
201
|
-
# # Batch Processing Configuration
|
|
202
|
-
# # Files processed concurrently per batch (should be >= MAX_API_CONNECTIONS for best performance)
|
|
203
|
-
# BATCH_SIZE=100
|
|
204
|
-
|
|
205
|
-
# # Delay between batches (0 for maximum speed)
|
|
206
|
-
# BATCH_DELAY=0
|
|
207
|
-
|
|
208
|
-
# # Source Processing Concurrency
|
|
209
|
-
# # Number of upload sources/folders to process simultaneously
|
|
210
|
-
# MAX_CONCURRENT_SOURCES=2
|
|
211
|
-
|
|
212
|
-
# # =============================================================================
|
|
213
|
-
# # EXAMPLE CONFIGURATIONS FOR DIFFERENT SCENARIOS
|
|
214
|
-
# # =============================================================================
|
|
215
|
-
|
|
216
|
-
# # For 10 API Replicas (High Performance Setup):
|
|
217
|
-
# # MAX_API_CONNECTIONS=10
|
|
218
|
-
# # BATCH_SIZE=100
|
|
219
|
-
# # MAX_CONCURRENT_SOURCES=3
|
|
220
|
-
# # BATCH_DELAY=0
|
|
221
|
-
|
|
222
|
-
# # For 5 API Replicas (Medium Performance Setup):
|
|
223
|
-
# # MAX_API_CONNECTIONS=5
|
|
224
|
-
# # BATCH_SIZE=50
|
|
225
|
-
# # MAX_CONCURRENT_SOURCES=2
|
|
226
|
-
# # BATCH_DELAY=0
|
|
227
|
-
|
|
228
|
-
# # For 1 API Instance (Single Instance Setup):
|
|
229
|
-
# # MAX_API_CONNECTIONS=5
|
|
230
|
-
# # BATCH_SIZE=20
|
|
231
|
-
# # MAX_CONCURRENT_SOURCES=1
|
|
232
|
-
# # BATCH_DELAY=100
|
|
233
|
-
|
|
234
|
-
# # =============================================================================
|
|
235
|
-
# # LOGGING AND MONITORING
|
|
236
|
-
# # =============================================================================
|
|
237
|
-
|
|
238
|
-
# # Progress bar update frequency
|
|
239
|
-
# PROGRESS_UPDATE_INTERVAL=10
|
|
240
|
-
|
|
241
|
-
# # Enable verbose logging (true/false)
|
|
242
|
-
# VERBOSE_LOGGING=false
|
|
243
|
-
|
|
244
|
-
# # ============================================
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
# =============================================================================
|
|
249
|
-
# WATCH MODE CONFIGURATION
|
|
250
|
-
# =============================================================================
|
|
251
|
-
|
|
252
|
-
# Habilitar watch mode (true/false)
|
|
253
|
-
WATCH_ENABLED=true
|
|
254
|
-
|
|
255
|
-
# Configuración de directorios a observar (formato JSON)
|
|
256
|
-
# Cada directorio puede tener su propia folderStructure para organizar en el bucket
|
|
257
|
-
# Formato: {"ruta/directorio1":"estructura-1","ruta/directorio2":"estructura-2"}
|
|
258
|
-
WATCH_DIRECTORY_CONFIGS={"./sample/watcher":"prueba-watcher"}
|
|
259
|
-
|
|
260
|
-
# Estrategia de upload (opciones: individual|batch|full-structure)
|
|
261
|
-
# - individual: Sube solo el archivo modificado más reciente
|
|
262
|
-
# - batch: Sube un lote de N archivos recientes
|
|
263
|
-
# - full-structure: Sube la estructura completa de carpetas
|
|
264
|
-
WATCH_STRATEGY=batch
|
|
265
|
-
|
|
266
|
-
# Debouncing en milisegundos (esperar entre eventos antes de procesar)
|
|
267
|
-
WATCH_DEBOUNCE_MS=1000
|
|
268
|
-
|
|
269
|
-
# Tamaño de batch para strategy batch
|
|
270
|
-
WATCH_BATCH_SIZE=10
|
|
271
|
-
|
|
272
|
-
# Usar polling en lugar de eventos nativos del filesystem
|
|
273
|
-
# Útil para sistemas de archivos remotos o NFS
|
|
274
|
-
WATCH_USE_POLLING=false
|
|
275
|
-
|
|
276
|
-
# Interval de polling en milisegundos (solo si WATCH_USE_POLLING=true)
|
|
277
|
-
WATCH_POLL_INTERVAL=100
|
|
278
|
-
|
|
279
|
-
# Umbral de estabilidad en ms (esperar a que el archivo deje de cambiar)
|
|
280
|
-
WATCH_STABILITY_THRESHOLD=300
|
|
281
|
-
|
|
282
|
-
# Patrones a ignorar (separados por coma, se usan como regex)
|
|
283
|
-
WATCH_IGNORE_PATTERNS=.tmp,.bak,*.swp
|
|
284
|
-
|
|
285
|
-
# Detección automática de tipos de documento
|
|
286
|
-
WATCH_AUTO_DETECT=true
|
|
287
|
-
|
|
288
|
-
# Organización automática de archivos
|
|
289
|
-
WATCH_AUTO_ORGANIZE=true
|
|
290
|
-
|
|
291
|
-
# =============================================================================
|
|
292
|
-
# WATCH MODE - AUTOMATIC PROCESSING PIPELINE
|
|
293
|
-
# =============================================================================
|
|
294
|
-
#
|
|
295
|
-
# El pipeline automático ejecuta la siguiente secuencia cuando se detecta un archivo nuevo:
|
|
296
|
-
# 1. Stats Collection → stats --stats-only (recopila información del archivo)
|
|
297
|
-
# 2. PDF Detection → detect --detect-pdfs (identifica pedimentos simplificados)
|
|
298
|
-
# 3. Path Propagation → detect --propagate-arela-path (propaga a documentos relacionados)
|
|
299
|
-
# 4. RFC Upload → upload --upload-by-rfc --folder-structure (sube con estructura)
|
|
300
|
-
#
|
|
301
|
-
# El pipeline se habilita automáticamente en watch mode y usa la folderStructure
|
|
302
|
-
# definida para cada WATCH_DIRECTORY_CONFIGS
|
|
303
|
-
#
|
|
304
|
-
# Para deshabilitar en CLI, usa: arela watch --no-auto-processing
|
|
305
|
-
|
|
306
|
-
# =============================================================================
|
|
307
|
-
# LOGGING AND MONITORING
|
|
308
|
-
# =============================================================================
|
|
309
|
-
|
|
310
|
-
# Progress bar update frequency
|
|
311
|
-
PROGRESS_UPDATE_INTERVAL=10
|
|
312
|
-
|
|
313
|
-
# Enable verbose logging (true/false)
|
|
314
|
-
VERBOSE_LOGGING=false
|
|
315
|
-
|
|
316
|
-
# ============================================
|