@arela/uploader 1.1.4 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/.claude/worktrees/agent-multi-profile/.env.template +224 -0
  2. package/.claude/worktrees/agent-multi-profile/.prettierrc +13 -0
  3. package/.claude/worktrees/agent-multi-profile/README.md +405 -0
  4. package/.claude/worktrees/agent-multi-profile/package-lock.json +7096 -0
  5. package/.claude/worktrees/agent-multi-profile/package.json +78 -0
  6. package/.claude/worktrees/agent-multi-profile/scripts/cleanup-ds-store.js +109 -0
  7. package/.claude/worktrees/agent-multi-profile/scripts/cleanup-system-files.js +69 -0
  8. package/.claude/worktrees/agent-multi-profile/scripts/scoring-compare.js +243 -0
  9. package/.claude/worktrees/agent-multi-profile/scripts/scoring-phase4-check.js +96 -0
  10. package/.claude/worktrees/agent-multi-profile/scripts/tests/phase-7-features.test.js +415 -0
  11. package/.claude/worktrees/agent-multi-profile/scripts/tests/signal-handling.test.js +275 -0
  12. package/.claude/worktrees/agent-multi-profile/scripts/tests/smart-watch-integration.test.js +554 -0
  13. package/.claude/worktrees/agent-multi-profile/scripts/tests/watch-service-integration.test.js +584 -0
  14. package/.claude/worktrees/agent-multi-profile/src/commands/AgentCommand.js +229 -0
  15. package/.claude/worktrees/agent-multi-profile/src/commands/AgentInitCommand.js +316 -0
  16. package/.claude/worktrees/agent-multi-profile/src/commands/DatastageCommand.js +164 -0
  17. package/.claude/worktrees/agent-multi-profile/src/commands/GDriveSyncCommand.js +475 -0
  18. package/.claude/worktrees/agent-multi-profile/src/commands/IdentifyCommand.js +708 -0
  19. package/.claude/worktrees/agent-multi-profile/src/commands/PollWorkerCommand.js +169 -0
  20. package/.claude/worktrees/agent-multi-profile/src/commands/PropagateCommand.js +636 -0
  21. package/.claude/worktrees/agent-multi-profile/src/commands/PushCommand.js +743 -0
  22. package/.claude/worktrees/agent-multi-profile/src/commands/ScanCommand.js +722 -0
  23. package/.claude/worktrees/agent-multi-profile/src/commands/UploadCommand.js +587 -0
  24. package/.claude/worktrees/agent-multi-profile/src/commands/WatchCommand.js +1342 -0
  25. package/.claude/worktrees/agent-multi-profile/src/commands/WorkerCommand.js +337 -0
  26. package/.claude/worktrees/agent-multi-profile/src/config/config.js +862 -0
  27. package/.claude/worktrees/agent-multi-profile/src/document-type-shared.js +131 -0
  28. package/.claude/worktrees/agent-multi-profile/src/document-types/_pedimento-shared-extractors.js +348 -0
  29. package/.claude/worktrees/agent-multi-profile/src/document-types/doda-pdf.js +121 -0
  30. package/.claude/worktrees/agent-multi-profile/src/document-types/doda-xml.js +118 -0
  31. package/.claude/worktrees/agent-multi-profile/src/document-types/factura-inter-agencia.js +186 -0
  32. package/.claude/worktrees/agent-multi-profile/src/document-types/facturas-comerciales.js +233 -0
  33. package/.claude/worktrees/agent-multi-profile/src/document-types/pedimento-completo-xml.js +372 -0
  34. package/.claude/worktrees/agent-multi-profile/src/document-types/pedimento-completo.js +108 -0
  35. package/.claude/worktrees/agent-multi-profile/src/document-types/pedimento-simplificado.js +76 -0
  36. package/.claude/worktrees/agent-multi-profile/src/document-types/proforma.js +29 -0
  37. package/.claude/worktrees/agent-multi-profile/src/document-types/support-document.js +200 -0
  38. package/.claude/worktrees/agent-multi-profile/src/errors/ErrorHandler.js +278 -0
  39. package/.claude/worktrees/agent-multi-profile/src/errors/ErrorTypes.js +104 -0
  40. package/.claude/worktrees/agent-multi-profile/src/file-detection.js +338 -0
  41. package/.claude/worktrees/agent-multi-profile/src/index.js +890 -0
  42. package/.claude/worktrees/agent-multi-profile/src/scoring/db-matcher-adapter.js +98 -0
  43. package/.claude/worktrees/agent-multi-profile/src/scoring/matchers-seed.js +386 -0
  44. package/.claude/worktrees/agent-multi-profile/src/scoring/scoring-engine.js +251 -0
  45. package/.claude/worktrees/agent-multi-profile/src/services/AdvancedFilterService.js +505 -0
  46. package/.claude/worktrees/agent-multi-profile/src/services/AutoProcessingService.js +749 -0
  47. package/.claude/worktrees/agent-multi-profile/src/services/BenchmarkingService.js +381 -0
  48. package/.claude/worktrees/agent-multi-profile/src/services/DatabaseService.js +2173 -0
  49. package/.claude/worktrees/agent-multi-profile/src/services/DatastageApiService.js +240 -0
  50. package/.claude/worktrees/agent-multi-profile/src/services/ErrorMonitor.js +275 -0
  51. package/.claude/worktrees/agent-multi-profile/src/services/GoogleDriveService.js +217 -0
  52. package/.claude/worktrees/agent-multi-profile/src/services/LoggingService.js +649 -0
  53. package/.claude/worktrees/agent-multi-profile/src/services/MonitoringService.js +401 -0
  54. package/.claude/worktrees/agent-multi-profile/src/services/PerformanceOptimizer.js +511 -0
  55. package/.claude/worktrees/agent-multi-profile/src/services/PipelineApiService.js +274 -0
  56. package/.claude/worktrees/agent-multi-profile/src/services/PipelineJobRunner.js +389 -0
  57. package/.claude/worktrees/agent-multi-profile/src/services/ProfileManager.js +164 -0
  58. package/.claude/worktrees/agent-multi-profile/src/services/ReportingService.js +511 -0
  59. package/.claude/worktrees/agent-multi-profile/src/services/ScanApiService.js +775 -0
  60. package/.claude/worktrees/agent-multi-profile/src/services/SignalHandler.js +255 -0
  61. package/.claude/worktrees/agent-multi-profile/src/services/SmartWatchDatabaseService.js +527 -0
  62. package/.claude/worktrees/agent-multi-profile/src/services/WatchService.js +783 -0
  63. package/.claude/worktrees/agent-multi-profile/src/services/upload/ApiUploadService.js +676 -0
  64. package/.claude/worktrees/agent-multi-profile/src/services/upload/BaseUploadService.js +36 -0
  65. package/.claude/worktrees/agent-multi-profile/src/services/upload/MultiApiUploadService.js +233 -0
  66. package/.claude/worktrees/agent-multi-profile/src/services/upload/SupabaseUploadService.js +148 -0
  67. package/.claude/worktrees/agent-multi-profile/src/services/upload/UploadServiceFactory.js +100 -0
  68. package/.claude/worktrees/agent-multi-profile/src/utils/CleanupManager.js +262 -0
  69. package/.claude/worktrees/agent-multi-profile/src/utils/FileOperations.js +192 -0
  70. package/.claude/worktrees/agent-multi-profile/src/utils/FileSanitizer.js +99 -0
  71. package/.claude/worktrees/agent-multi-profile/src/utils/PathDetector.js +198 -0
  72. package/.claude/worktrees/agent-multi-profile/src/utils/PathNormalizer.js +274 -0
  73. package/.claude/worktrees/agent-multi-profile/src/utils/WatchEventHandler.js +522 -0
  74. package/.claude/worktrees/agent-multi-profile/supabase/migrations/001_create_initial_schema.sql +366 -0
  75. package/.claude/worktrees/agent-multi-profile/supabase/migrations/002_align_with_arela_api_schema.sql +145 -0
  76. package/.claude/worktrees/agent-multi-profile/tests/commands/IdentifyCommand.test.js +570 -0
  77. package/.claude/worktrees/agent-multi-profile/tests/commands/PropagateCommand.test.js +568 -0
  78. package/.claude/worktrees/agent-multi-profile/tests/commands/PushCommand.test.js +754 -0
  79. package/.claude/worktrees/agent-multi-profile/tests/commands/ScanCommand.test.js +382 -0
  80. package/.claude/worktrees/agent-multi-profile/tests/unit/PathAndTableNameGeneration.test.js +1211 -0
  81. package/.claude/worktrees/agent-multi-profile/tests/unit/factura-inter-agencia.test.js +218 -0
  82. package/.claude/worktrees/agent-multi-profile/tests/unit/pedimento-completo-xml-matcher.test.js +271 -0
  83. package/.claude/worktrees/agent-multi-profile/tests/unit/pedimento-simplificado-matcher.test.js +185 -0
  84. package/.claude/worktrees/agent-multi-profile/tests/unit/scoring-engine.test.js +221 -0
  85. package/README.md +65 -0
  86. package/package.json +1 -1
  87. package/src/commands/AgentCommand.js +210 -0
  88. package/src/commands/AgentInitCommand.js +316 -0
  89. package/src/commands/PollWorkerCommand.js +11 -322
  90. package/src/config/config.js +44 -6
  91. package/src/index.js +55 -0
  92. package/src/services/LoggingService.js +35 -0
  93. package/src/services/PipelineApiService.js +7 -1
  94. package/src/services/PipelineJobRunner.js +389 -0
  95. package/src/services/ProfileManager.js +164 -0
@@ -0,0 +1,708 @@
1
+ import cliProgress from 'cli-progress';
2
+ import fs from 'fs';
3
+ import pLimit from 'p-limit';
4
+
5
+ import logger from '../services/LoggingService.js';
6
+
7
+ import appConfig from '../config/config.js';
8
+ import ErrorHandler from '../errors/ErrorHandler.js';
9
+ import { ConfigurationError } from '../errors/ErrorTypes.js';
10
+ import FileDetectionService from '../file-detection.js';
11
+ import { adaptDbMatchers } from '../scoring/db-matcher-adapter.js';
12
+ import { scoringMatchers } from '../scoring/matchers-seed.js';
13
+
14
+ /**
15
+ * Paid pedimento detected_type values.
16
+ */
17
+ const DETECTED_PEDIMENTO_TYPES = new Set([
18
+ 'pedimento_simplificado',
19
+ 'pedimento_completo',
20
+ 'pedimento_completo_xml',
21
+ ]);
22
+
23
+ /** Unpaid pedimento detected_type values (proforma variants). */
24
+ const PROFORMA_TYPES = new Set([
25
+ 'proforma',
26
+ 'proforma_completo',
27
+ 'proforma_completo_xml',
28
+ ]);
29
+
30
+ /**
31
+ * Identify Command Handler
32
+ * Optimized replacement for "detect --detect-pdfs"
33
+ * Identifies pedimento-simplificado documents in files scanned by "arela scan"
34
+ *
35
+ * Key improvements over legacy detect command:
36
+ * - Works with dynamic scan_* tables instead of uploader table
37
+ * - Uses configured API instead of direct Supabase access
38
+ * - Detects locally to leverage CLI host resources
39
+ * - Batch processing for efficient API communication
40
+ * - Real-time progress tracking
41
+ */
42
+ export class IdentifyCommand {
43
+ constructor() {
44
+ this.errorHandler = new ErrorHandler(logger);
45
+ this.scanApiService = null; // Will be initialized in execute
46
+ this.detectionService = new FileDetectionService();
47
+ }
48
+
49
+ /**
50
+ * Execute the identify command
51
+ * @param {Object} options - Command options
52
+ * @param {string} options.api - API target (default, agencia, cliente)
53
+ * @param {number} options.batchSize - Batch size for API operations
54
+ * @param {boolean} options.showStats - Show performance statistics
55
+ * @param {Function} options.onProgress - Optional callback for progress updates (percent, message)
56
+ */
57
+ async execute(options = {}) {
58
+ const startTime = Date.now();
59
+ this.onProgress = options.onProgress || null;
60
+
61
+ try {
62
+ // Validate scan configuration (need same config as scan command)
63
+ appConfig.validateScanConfig();
64
+
65
+ // Determine API target
66
+ const apiTarget = options.api || 'default';
67
+
68
+ // Import ScanApiService dynamically and initialize with target
69
+ const { default: ScanApiService } = await import(
70
+ '../services/ScanApiService.js'
71
+ );
72
+ this.scanApiService = new ScanApiService(apiTarget);
73
+
74
+ // Load matchers for best-match classification (phase 4 hybrid). Prefer the
75
+ // DB-resolved set (this RFC + globals); fall back to the validated local
76
+ // seed; set DISABLE_SCORING_MATCHERS=true to force legacy first-match.
77
+ if (process.env.DISABLE_SCORING_MATCHERS === 'true') {
78
+ logger.info('🧩 Scoring matchers disabled — legacy detection');
79
+ } else {
80
+ let matchers = null;
81
+ try {
82
+ const rfc = process.env.MATCHER_RFC || null;
83
+ const dbMatchers = await this.scanApiService.getResolvedMatchers(rfc);
84
+ if (dbMatchers.length) {
85
+ matchers = adaptDbMatchers(dbMatchers);
86
+ logger.info(`🧩 Loaded ${matchers.length} matchers from API`);
87
+ }
88
+ } catch (err) {
89
+ logger.warn(`🧩 Could not load matchers from API: ${err.message}`);
90
+ }
91
+ if (!matchers) {
92
+ matchers = scoringMatchers;
93
+ logger.info(`🧩 Using local seed matchers (${matchers.length})`);
94
+ }
95
+ if (typeof this.detectionService.setMatchers === 'function') {
96
+ this.detectionService.setMatchers(matchers);
97
+ }
98
+ }
99
+
100
+ const scanConfig = appConfig.getScanConfig();
101
+ const batchSize = parseInt(options.batchSize) || 100;
102
+
103
+ // Parse optional path prefix mapping: "O:/=/Volumes/nas/"
104
+ const pathPrefixMap = options.pathPrefix
105
+ ? this.#parsePathPrefix(options.pathPrefix)
106
+ : null;
107
+
108
+ logger.info('šŸ” Starting arela identify command');
109
+ logger.info(`šŸŽÆ API Target: ${apiTarget}`);
110
+ logger.info(`šŸ“¦ Batch Size: ${batchSize}`);
111
+ if (options.table) logger.info(`šŸ“Œ Target table: ${options.table}`);
112
+ if (options.resetAttempts)
113
+ logger.info('ā™»ļø Reset detection attempts: ON');
114
+ if (pathPrefixMap)
115
+ logger.info(
116
+ `šŸ—ŗ Path prefix map: ${pathPrefixMap.from} → ${pathPrefixMap.to}`,
117
+ );
118
+
119
+ // Resolve the list of tables to process
120
+ let tables;
121
+ if (options.fileId && options.table) {
122
+ // Single-file mode — identify exactly one file record
123
+ logger.info(
124
+ `\nšŸŽÆ Single-file mode: ${options.table} / ${options.fileId}`,
125
+ );
126
+ this.#reportProgress(0, `Fetching file record ${options.fileId}...`);
127
+ const record = await this.scanApiService.getFileRecord(
128
+ options.table,
129
+ options.fileId,
130
+ );
131
+ const results = await this.#detectFilesLocally(
132
+ [record],
133
+ 1,
134
+ pathPrefixMap,
135
+ );
136
+ const updates = results.filter((r) => r !== null);
137
+ if (updates.length > 0) {
138
+ await this.scanApiService.batchUpdateDetection(
139
+ options.table,
140
+ updates,
141
+ );
142
+ }
143
+ this.#reportProgress(100, `Single-file identification complete`);
144
+ logger.success(`\nāœ… Single-file identification complete`);
145
+ const firstUpdate = updates[0];
146
+ return {
147
+ processed: 1,
148
+ detected: updates.length,
149
+ proformas: 0,
150
+ errors: 1 - updates.length,
151
+ detectedType: firstUpdate?.detectedType ?? null,
152
+ detectedPedimento: firstUpdate?.detectedPedimento ?? null,
153
+ rfc: firstUpdate?.rfc ?? null,
154
+ arelaPath: firstUpdate?.arelaPath ?? null,
155
+ };
156
+ } else if (options.table) {
157
+ // Single-table mode — no need to match instance tables
158
+ tables = [{ tableName: options.table }];
159
+ logger.info(`\nšŸ“Œ Single-table mode: ${options.table}`);
160
+ } else {
161
+ logger.info('\nšŸ“Š Fetching instance tables...');
162
+ tables = await this.scanApiService.getInstanceTables(
163
+ scanConfig.companySlug,
164
+ scanConfig.serverId,
165
+ scanConfig.basePathFull,
166
+ );
167
+
168
+ if (tables.length === 0) {
169
+ throw new ConfigurationError(
170
+ 'No tables found for this instance. Run "arela scan" first.',
171
+ );
172
+ }
173
+
174
+ logger.info(
175
+ `šŸ“‹ Found ${tables.length} table${tables.length === 1 ? '' : 's'} to process`,
176
+ );
177
+ for (const table of tables) {
178
+ logger.info(` - ${table.tableName}`);
179
+ }
180
+ }
181
+
182
+ // Optionally reset detection attempts so previously-failed files are retried
183
+ if (options.resetAttempts) {
184
+ for (const table of tables) {
185
+ const { reset } = await this.scanApiService.resetDetectionAttempts(
186
+ table.tableName,
187
+ );
188
+ logger.info(
189
+ `ā™»ļø Reset ${reset} detection attempt(s) in ${table.tableName}`,
190
+ );
191
+ }
192
+ }
193
+
194
+ // Process each table
195
+ let totalStats = {
196
+ processed: 0,
197
+ detected: 0,
198
+ proformas: 0,
199
+ errors: 0,
200
+ pending: 0,
201
+ };
202
+
203
+ // Report initial progress
204
+ this.#reportProgress(
205
+ 0,
206
+ `Starting identification on ${tables.length} tables`,
207
+ );
208
+
209
+ for (let i = 0; i < tables.length; i++) {
210
+ const table = tables[i];
211
+ logger.info(`\nšŸ” Processing table: ${table.tableName}`);
212
+ this.#reportProgress(
213
+ Math.round((i / tables.length) * 100),
214
+ `Processing table ${i + 1}/${tables.length}: ${table.tableName}`,
215
+ );
216
+
217
+ // Get detection statistics for this table
218
+ const stats = await this.#processTable(
219
+ table.tableName,
220
+ batchSize,
221
+ startTime,
222
+ pathPrefixMap,
223
+ );
224
+
225
+ totalStats.processed += stats.processed;
226
+ totalStats.detected += stats.detected;
227
+ totalStats.proformas += stats.proformas;
228
+ totalStats.errors += stats.errors;
229
+ }
230
+
231
+ const duration = ((Date.now() - startTime) / 1000).toFixed(2);
232
+ const avgSpeed =
233
+ duration > 0 ? Math.round(totalStats.processed / duration) : 0;
234
+
235
+ // Report completion
236
+ this.#reportProgress(
237
+ 100,
238
+ `Identification completed: ${totalStats.detected} pedimentos detected`,
239
+ );
240
+
241
+ logger.success(`\nāœ… Identification Complete!`);
242
+ logger.info(`\nšŸ“Š Total Results:`);
243
+ logger.info(` Tables Processed: ${tables.length}`);
244
+ logger.info(` Files Processed: ${totalStats.processed}`);
245
+ logger.info(` Pedimentos Detected (paid): ${totalStats.detected}`);
246
+ logger.info(` Proformas Detected (unpaid): ${totalStats.proformas}`);
247
+ logger.info(` Errors: ${totalStats.errors}`);
248
+ logger.info(` Duration: ${duration}s`);
249
+ logger.info(` Speed: ${avgSpeed} files/sec`);
250
+
251
+ // Classification distribution for the run (observability: detect shifts /
252
+ // rising unmatched rate, and which detection mode was used).
253
+ if (
254
+ typeof this.detectionService.logClassificationSummary === 'function'
255
+ ) {
256
+ this.detectionService.logClassificationSummary(logger);
257
+ }
258
+
259
+ if (options.showStats) {
260
+ this.#showDetailedStats(
261
+ startTime,
262
+ totalStats.processed,
263
+ totalStats.detected,
264
+ totalStats.errors,
265
+ );
266
+ }
267
+ } catch (error) {
268
+ logger.error('āŒ Identification failed:', error.message);
269
+
270
+ if (error instanceof ConfigurationError) {
271
+ logger.error('\nConfiguration errors:');
272
+ error.errors.forEach((err) => logger.error(` - ${err}`));
273
+ }
274
+
275
+ throw error;
276
+ }
277
+ }
278
+
279
+ /**
280
+ * Process a single table
281
+ * @private
282
+ * @param {string} tableName - Table name to process
283
+ * @param {number} batchSize - Batch size
284
+ * @param {number} startTime - Start time for speed calculation
285
+ * @returns {Promise<Object>} Processing statistics
286
+ */
287
+ async #processTable(tableName, batchSize, startTime, pathPrefixMap = null) {
288
+ // Get detection statistics first (allTypes=true to count all supported file types)
289
+ const initialStats = await this.scanApiService.getDetectionStats(
290
+ tableName,
291
+ true,
292
+ );
293
+ logger.info(` Total PDFs: ${initialStats.totalPdfs}`);
294
+ logger.info(` Detected: ${initialStats.detected}`);
295
+ logger.info(` Pending: ${initialStats.pending}`);
296
+ logger.info(` Not Pedimento: ${initialStats.notPedimento || 0}`);
297
+ logger.info(
298
+ ` Max Attempts Reached: ${initialStats.maxAttemptsReached || 0}`,
299
+ );
300
+ logger.info(` Errors: ${initialStats.errors}`);
301
+
302
+ if (initialStats.pending === 0) {
303
+ logger.info(' āœ… All PDFs processed. Skipping.');
304
+
305
+ if (initialStats.maxAttemptsReached > 0) {
306
+ logger.info(
307
+ ` āš ļø ${initialStats.maxAttemptsReached} PDFs reached max attempts.`,
308
+ );
309
+ }
310
+
311
+ return {
312
+ processed: 0,
313
+ detected: 0,
314
+ // Must include every field totalStats accumulates — omitting proformas
315
+ // made `totalStats.proformas += undefined` = NaN in the final summary.
316
+ proformas: 0,
317
+ errors: 0,
318
+ };
319
+ }
320
+
321
+ logger.info(` šŸš€ Processing ${initialStats.pending} pending PDFs...`);
322
+
323
+ logger.info(` šŸš€ Processing ${initialStats.pending} pending PDFs...`);
324
+
325
+ // Setup progress bar
326
+ const progressBar = new cliProgress.SingleBar({
327
+ format:
328
+ ' šŸ“„ |{bar}| {percentage}% | {value}/{total} files | {speed} files/sec',
329
+ barCompleteChar: '\u2588',
330
+ barIncompleteChar: '\u2591',
331
+ hideCursor: true,
332
+ });
333
+
334
+ progressBar.start(initialStats.pending, 0, { speed: 0 });
335
+
336
+ let processedCount = 0;
337
+ let detectedCount = 0;
338
+ let proformaCount = 0;
339
+ let errorCount = 0;
340
+ let hasMore = true;
341
+ let batchNumber = 0;
342
+
343
+ // Process in batches
344
+ while (hasMore) {
345
+ batchNumber++;
346
+
347
+ // Fetch from offset 0 since processed files are filtered out
348
+ const response = await this.scanApiService.fetchPdfsForDetection(
349
+ tableName,
350
+ 0,
351
+ batchSize,
352
+ true, // allTypes: fetch all supported file types, not just likely-simplificado PDFs
353
+ );
354
+
355
+ if (!response.data || response.data.length === 0) {
356
+ break;
357
+ }
358
+
359
+ const files = response.data;
360
+
361
+ // Detect files locally with concurrent processing
362
+ const detectionResults = await this.#detectFilesLocally(
363
+ files,
364
+ 10,
365
+ pathPrefixMap,
366
+ );
367
+
368
+ // Batch update to API
369
+ const updateResult = await this.scanApiService.batchUpdateDetection(
370
+ tableName,
371
+ detectionResults,
372
+ );
373
+
374
+ // Update statistics
375
+ processedCount += files.length;
376
+ // "Detected" counts paid pedimentos of any flavour (simplificado,
377
+ // completo, completo_xml when enabled).
378
+ detectedCount += detectionResults.filter((r) =>
379
+ DETECTED_PEDIMENTO_TYPES.has(r.detectedType),
380
+ ).length;
381
+ proformaCount += detectionResults.filter((r) =>
382
+ PROFORMA_TYPES.has(r.detectedType),
383
+ ).length;
384
+ errorCount += detectionResults.filter((r) => r.detectionError).length;
385
+
386
+ // Update progress bar
387
+ const elapsed = (Date.now() - startTime) / 1000;
388
+ const speed = elapsed > 0 ? Math.round(processedCount / elapsed) : 0;
389
+ progressBar.update(processedCount, { speed });
390
+
391
+ // Check if there are more files
392
+ hasMore = response.hasMore;
393
+ }
394
+
395
+ progressBar.stop();
396
+
397
+ return {
398
+ processed: processedCount,
399
+ detected: detectedCount,
400
+ proformas: proformaCount,
401
+ errors: errorCount,
402
+ };
403
+ }
404
+
405
+ /**
406
+ * Detect files locally using FileDetectionService
407
+ * @private
408
+ * @param {Array} files - Files to detect
409
+ * @param {number} concurrency - Maximum concurrent detections
410
+ * @returns {Promise<Array>} Detection results
411
+ */
412
+ async #detectFilesLocally(files, concurrency = 10, pathPrefixMap = null) {
413
+ const limit = pLimit(concurrency);
414
+ const basePath = appConfig.getBasePath();
415
+
416
+ const detectionPromises = files.map((file) =>
417
+ limit(async () => {
418
+ try {
419
+ // Check if file exists on filesystem
420
+ let absolutePath = file.absolute_path;
421
+
422
+ // Apply cross-platform path prefix mapping if configured
423
+ if (
424
+ pathPrefixMap &&
425
+ absolutePath &&
426
+ absolutePath.startsWith(pathPrefixMap.from)
427
+ ) {
428
+ absolutePath =
429
+ pathPrefixMap.to + absolutePath.slice(pathPrefixMap.from.length);
430
+ }
431
+
432
+ if (!fs.existsSync(absolutePath)) {
433
+ return {
434
+ id: file.id,
435
+ detectedType: null,
436
+ detectedPedimento: null,
437
+ detectedPedimentoYear: null,
438
+ rfc: null,
439
+ arelaPath: null,
440
+ detectionError:
441
+ 'FILE_NOT_FOUND: File does not exist on filesystem. May have been moved or deleted after scan.',
442
+ isPedimento: null, // Unknown - can't determine
443
+ };
444
+ }
445
+
446
+ // Check file size - skip very large files
447
+ const stats = fs.statSync(absolutePath);
448
+ const maxSizeBytes = 50 * 1024 * 1024; // 50MB
449
+ if (stats.size > maxSizeBytes) {
450
+ return {
451
+ id: file.id,
452
+ detectedType: null,
453
+ detectedPedimento: null,
454
+ detectedPedimentoYear: null,
455
+ rfc: null,
456
+ arelaPath: null,
457
+ detectionError: `FILE_TOO_LARGE: File size ${(stats.size / 1024 / 1024).toFixed(2)}MB exceeds ${maxSizeBytes / 1024 / 1024}MB limit.`,
458
+ isPedimento: null, // Unknown - can't determine
459
+ };
460
+ }
461
+
462
+ // Detect using existing FileDetectionService
463
+ const result = await this.detectionService.detectFile(absolutePath);
464
+
465
+ // If detection succeeded and found a paid pedimento (any variant)
466
+ if (DETECTED_PEDIMENTO_TYPES.has(result.detectedType)) {
467
+ return {
468
+ id: file.id,
469
+ detectedType: result.detectedType,
470
+ detectedPedimento: result.detectedPedimento,
471
+ detectedPedimentoYear: result.detectedPedimentoYear,
472
+ rfc: result.rfc,
473
+ arelaPath: result.arelaPath,
474
+ detectionError: result.error,
475
+ isPedimento: true, // Confirmed paid pedimento
476
+ };
477
+ }
478
+
479
+ // If detection succeeded and found a proforma (any variant)
480
+ if (PROFORMA_TYPES.has(result.detectedType)) {
481
+ return {
482
+ id: file.id,
483
+ detectedType: result.detectedType,
484
+ detectedPedimento: result.detectedPedimento,
485
+ detectedPedimentoYear: result.detectedPedimentoYear,
486
+ rfc: result.rfc,
487
+ arelaPath: null, // Proformas don't get arela_path (not paid yet)
488
+ detectionError: result.error,
489
+ isPedimento: false, // Not a paid pedimento — proforma only
490
+ };
491
+ }
492
+
493
+ // If no detection, determine if it's definitely not a pedimento
494
+ // This helps avoid re-processing files we know aren't pedimentos
495
+ const isDefinitelyNotPedimento = this.#isDefinitelyNotPedimento(
496
+ result,
497
+ file,
498
+ );
499
+
500
+ // Build descriptive error message
501
+ let detectionError = null;
502
+ if (result.error) {
503
+ detectionError = `DETECTION_ERROR: ${result.error}`;
504
+ } else if (isDefinitelyNotPedimento) {
505
+ detectionError =
506
+ 'NOT_PEDIMENTO: File does not match any pedimento pattern. Missing key markers (e.g. "FORMA SIMPLIFICADA DE PEDIMENTO" or "NUM. PEDIMENTO:" + copy markers).';
507
+ } else {
508
+ // Partial match - might be a pedimento with missing fields
509
+ const missingFields = this.#getMissingFields(result);
510
+ if (missingFields.length > 0) {
511
+ detectionError = `INCOMPLETE_PEDIMENTO: Detected as potential pedimento but missing fields: ${missingFields.join(', ')}. Matcher may need improvement.`;
512
+ } else {
513
+ detectionError =
514
+ 'UNKNOWN_ERROR: Detection completed but no pedimento found. Check file content and matcher patterns.';
515
+ }
516
+ }
517
+
518
+ return {
519
+ id: file.id,
520
+ detectedType: result.detectedType,
521
+ detectedPedimento: result.detectedPedimento,
522
+ detectedPedimentoYear: result.detectedPedimentoYear,
523
+ rfc: result.rfc,
524
+ arelaPath: result.arelaPath,
525
+ detectionError,
526
+ isPedimento: isDefinitelyNotPedimento ? false : null, // false = not pedimento, null = unknown
527
+ };
528
+ } catch (error) {
529
+ logger.warn(
530
+ `Failed to detect ${file.relative_path}: ${error.message}`,
531
+ );
532
+
533
+ // Categorize the error
534
+ let errorCategory = 'UNKNOWN_ERROR';
535
+ if (error.message.includes('ENOENT')) {
536
+ errorCategory = 'FILE_NOT_FOUND';
537
+ } else if (error.message.includes('timeout')) {
538
+ errorCategory = 'TIMEOUT';
539
+ } else if (error.message.includes('PDF')) {
540
+ errorCategory = 'PDF_PARSE_ERROR';
541
+ } else if (error.message.includes('extract')) {
542
+ errorCategory = 'TEXT_EXTRACTION_ERROR';
543
+ }
544
+
545
+ return {
546
+ id: file.id,
547
+ detectedType: null,
548
+ detectedPedimento: null,
549
+ detectedPedimentoYear: null,
550
+ rfc: null,
551
+ arelaPath: null,
552
+ detectionError: `${errorCategory}: ${error.message}`,
553
+ isPedimento: null, // Unknown - error occurred
554
+ };
555
+ }
556
+ }),
557
+ );
558
+
559
+ return Promise.all(detectionPromises);
560
+ }
561
+
562
+ /**
563
+ * Determine if a file is definitely not a pedimento/proforma document
564
+ * @private
565
+ * @param {Object} result - Detection result
566
+ * @param {Object} file - File metadata
567
+ * @returns {boolean} True if definitely not a pedimento/proforma
568
+ */
569
+ #isDefinitelyNotPedimento(result, file) {
570
+ // If we got any pedimento-related fields, it might be a pedimento or proforma
571
+ if (result.detectedPedimento || result.rfc || result.arelaPath) {
572
+ return false;
573
+ }
574
+
575
+ // If it was detected as a proforma (any variant), it's related to a
576
+ // pedimento structure — not "definitely not".
577
+ if (PROFORMA_TYPES.has(result.detectedType)) {
578
+ return false;
579
+ }
580
+
581
+ // Check if the text contains any required pedimento marker. This must
582
+ // stay aligned with the `match()` predicates in pedimento-simplificado.js
583
+ // and pedimento-completo.js (which accept both "DE" and "DEL" in the
584
+ // title, and treat the colon after "T. OPER" as optional).
585
+ const text = result.text || '';
586
+ const hasSimplificadoMarker =
587
+ /FORMA\s+SIMPLIFICADA\s+DEL?\s+PEDIMENTO/i.test(text);
588
+ const hasCompletoMarkers =
589
+ /NUM\.?\s*PEDIMENTO:/i.test(text) &&
590
+ /CVE\.?\s*PEDIMENTO:/i.test(text) &&
591
+ /T\.?\s*OPER:?/i.test(text);
592
+
593
+ return !hasSimplificadoMarker && !hasCompletoMarkers;
594
+ }
595
+
596
+ /**
597
+ * Get list of missing required fields for pedimento
598
+ * @private
599
+ * @param {Object} result - Detection result
600
+ * @returns {Array<string>} Missing field names
601
+ */
602
+ #getMissingFields(result) {
603
+ const requiredFields = [
604
+ { key: 'detectedPedimento', name: 'numPedimento' },
605
+ { key: 'rfc', name: 'rfc' },
606
+ { key: 'detectedPedimentoYear', name: 'year' },
607
+ ];
608
+
609
+ const missing = [];
610
+ for (const field of requiredFields) {
611
+ if (!result[field.key]) {
612
+ missing.push(field.name);
613
+ }
614
+ }
615
+
616
+ return missing;
617
+ }
618
+
619
+ /**
620
+ * Show detailed performance statistics
621
+ * @private
622
+ */
623
+ #showDetailedStats(startTime, processedCount, detectedCount, errorCount) {
624
+ const duration = (Date.now() - startTime) / 1000;
625
+ const avgSpeed = duration > 0 ? (processedCount / duration).toFixed(2) : 0;
626
+ const detectionRate =
627
+ processedCount > 0
628
+ ? ((detectedCount / processedCount) * 100).toFixed(1)
629
+ : 0;
630
+
631
+ logger.info('\nšŸ“ˆ Detailed Statistics:');
632
+ logger.info(` Total Processing Time: ${duration.toFixed(2)}s`);
633
+ logger.info(` Average Speed: ${avgSpeed} files/sec`);
634
+ logger.info(` Detection Rate: ${detectionRate}%`);
635
+ logger.info(
636
+ ` Error Rate: ${errorCount > 0 ? ((errorCount / processedCount) * 100).toFixed(1) : 0}%`,
637
+ );
638
+
639
+ // Memory usage
640
+ const memUsage = process.memoryUsage();
641
+ logger.info('\nšŸ’¾ Memory Usage:');
642
+ logger.info(` RSS: ${(memUsage.rss / 1024 / 1024).toFixed(2)} MB`);
643
+ logger.info(
644
+ ` Heap Used: ${(memUsage.heapUsed / 1024 / 1024).toFixed(2)} MB`,
645
+ );
646
+ logger.info(
647
+ ` Heap Total: ${(memUsage.heapTotal / 1024 / 1024).toFixed(2)} MB`,
648
+ );
649
+ }
650
+
651
+ /**
652
+ * Report progress to callback if available (for worker mode)
653
+ * @private
654
+ * @param {number} percent - Progress percentage (0-100)
655
+ * @param {string} message - Progress message
656
+ */
657
+ #reportProgress(percent, message) {
658
+ if (this.onProgress && typeof this.onProgress === 'function') {
659
+ try {
660
+ this.onProgress(percent, message);
661
+ } catch (error) {
662
+ logger.debug(`Progress callback error: ${error.message}`);
663
+ }
664
+ }
665
+ }
666
+
667
+ /**
668
+ * Parse a path prefix mapping string such as "O:/=/Volumes/nas/" into { from, to }.
669
+ * Supports both "FROM=TO" and "FROM:TO" separators.
670
+ * @private
671
+ * @param {string} mapping
672
+ * @returns {{ from: string, to: string }}
673
+ */
674
+ #parsePathPrefix(mapping) {
675
+ // Support either "FROM=TO" or "FROM:TO" as separator
676
+ const eqIdx = mapping.indexOf('=');
677
+ const colonIdx = mapping.indexOf(':');
678
+
679
+ let sep = -1;
680
+ // "O:/=/Volumes" — the colon inside "O:/" is part of a Windows drive letter; prefer '=' separator
681
+ if (eqIdx !== -1) {
682
+ sep = eqIdx;
683
+ } else if (colonIdx !== -1) {
684
+ sep = colonIdx;
685
+ }
686
+
687
+ if (sep === -1) {
688
+ throw new Error(
689
+ `Invalid --path-prefix format: "${mapping}". Expected "FROM=TO" e.g. "O:/=/Volumes/nas/"`,
690
+ );
691
+ }
692
+
693
+ const from = mapping.slice(0, sep);
694
+ const to = mapping.slice(sep + 1);
695
+
696
+ if (!from || !to) {
697
+ throw new Error(
698
+ `Invalid --path-prefix format: "${mapping}". Both FROM and TO parts must be non-empty.`,
699
+ );
700
+ }
701
+
702
+ return { from, to };
703
+ }
704
+ }
705
+
706
+ // Export singleton instance
707
+ const identifyCommand = new IdentifyCommand();
708
+ export default identifyCommand;