@arela/uploader 1.1.4 ā 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/worktrees/agent-multi-profile/.env.template +224 -0
- package/.claude/worktrees/agent-multi-profile/.prettierrc +13 -0
- package/.claude/worktrees/agent-multi-profile/README.md +405 -0
- package/.claude/worktrees/agent-multi-profile/package-lock.json +7096 -0
- package/.claude/worktrees/agent-multi-profile/package.json +78 -0
- package/.claude/worktrees/agent-multi-profile/scripts/cleanup-ds-store.js +109 -0
- package/.claude/worktrees/agent-multi-profile/scripts/cleanup-system-files.js +69 -0
- package/.claude/worktrees/agent-multi-profile/scripts/scoring-compare.js +243 -0
- package/.claude/worktrees/agent-multi-profile/scripts/scoring-phase4-check.js +96 -0
- package/.claude/worktrees/agent-multi-profile/scripts/tests/phase-7-features.test.js +415 -0
- package/.claude/worktrees/agent-multi-profile/scripts/tests/signal-handling.test.js +275 -0
- package/.claude/worktrees/agent-multi-profile/scripts/tests/smart-watch-integration.test.js +554 -0
- package/.claude/worktrees/agent-multi-profile/scripts/tests/watch-service-integration.test.js +584 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/AgentCommand.js +229 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/AgentInitCommand.js +316 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/DatastageCommand.js +164 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/GDriveSyncCommand.js +475 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/IdentifyCommand.js +708 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/PollWorkerCommand.js +169 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/PropagateCommand.js +636 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/PushCommand.js +743 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/ScanCommand.js +722 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/UploadCommand.js +587 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/WatchCommand.js +1342 -0
- package/.claude/worktrees/agent-multi-profile/src/commands/WorkerCommand.js +337 -0
- package/.claude/worktrees/agent-multi-profile/src/config/config.js +862 -0
- package/.claude/worktrees/agent-multi-profile/src/document-type-shared.js +131 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/_pedimento-shared-extractors.js +348 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/doda-pdf.js +121 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/doda-xml.js +118 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/factura-inter-agencia.js +186 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/facturas-comerciales.js +233 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/pedimento-completo-xml.js +372 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/pedimento-completo.js +108 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/pedimento-simplificado.js +76 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/proforma.js +29 -0
- package/.claude/worktrees/agent-multi-profile/src/document-types/support-document.js +200 -0
- package/.claude/worktrees/agent-multi-profile/src/errors/ErrorHandler.js +278 -0
- package/.claude/worktrees/agent-multi-profile/src/errors/ErrorTypes.js +104 -0
- package/.claude/worktrees/agent-multi-profile/src/file-detection.js +338 -0
- package/.claude/worktrees/agent-multi-profile/src/index.js +890 -0
- package/.claude/worktrees/agent-multi-profile/src/scoring/db-matcher-adapter.js +98 -0
- package/.claude/worktrees/agent-multi-profile/src/scoring/matchers-seed.js +386 -0
- package/.claude/worktrees/agent-multi-profile/src/scoring/scoring-engine.js +251 -0
- package/.claude/worktrees/agent-multi-profile/src/services/AdvancedFilterService.js +505 -0
- package/.claude/worktrees/agent-multi-profile/src/services/AutoProcessingService.js +749 -0
- package/.claude/worktrees/agent-multi-profile/src/services/BenchmarkingService.js +381 -0
- package/.claude/worktrees/agent-multi-profile/src/services/DatabaseService.js +2173 -0
- package/.claude/worktrees/agent-multi-profile/src/services/DatastageApiService.js +240 -0
- package/.claude/worktrees/agent-multi-profile/src/services/ErrorMonitor.js +275 -0
- package/.claude/worktrees/agent-multi-profile/src/services/GoogleDriveService.js +217 -0
- package/.claude/worktrees/agent-multi-profile/src/services/LoggingService.js +649 -0
- package/.claude/worktrees/agent-multi-profile/src/services/MonitoringService.js +401 -0
- package/.claude/worktrees/agent-multi-profile/src/services/PerformanceOptimizer.js +511 -0
- package/.claude/worktrees/agent-multi-profile/src/services/PipelineApiService.js +274 -0
- package/.claude/worktrees/agent-multi-profile/src/services/PipelineJobRunner.js +389 -0
- package/.claude/worktrees/agent-multi-profile/src/services/ProfileManager.js +164 -0
- package/.claude/worktrees/agent-multi-profile/src/services/ReportingService.js +511 -0
- package/.claude/worktrees/agent-multi-profile/src/services/ScanApiService.js +775 -0
- package/.claude/worktrees/agent-multi-profile/src/services/SignalHandler.js +255 -0
- package/.claude/worktrees/agent-multi-profile/src/services/SmartWatchDatabaseService.js +527 -0
- package/.claude/worktrees/agent-multi-profile/src/services/WatchService.js +783 -0
- package/.claude/worktrees/agent-multi-profile/src/services/upload/ApiUploadService.js +676 -0
- package/.claude/worktrees/agent-multi-profile/src/services/upload/BaseUploadService.js +36 -0
- package/.claude/worktrees/agent-multi-profile/src/services/upload/MultiApiUploadService.js +233 -0
- package/.claude/worktrees/agent-multi-profile/src/services/upload/SupabaseUploadService.js +148 -0
- package/.claude/worktrees/agent-multi-profile/src/services/upload/UploadServiceFactory.js +100 -0
- package/.claude/worktrees/agent-multi-profile/src/utils/CleanupManager.js +262 -0
- package/.claude/worktrees/agent-multi-profile/src/utils/FileOperations.js +192 -0
- package/.claude/worktrees/agent-multi-profile/src/utils/FileSanitizer.js +99 -0
- package/.claude/worktrees/agent-multi-profile/src/utils/PathDetector.js +198 -0
- package/.claude/worktrees/agent-multi-profile/src/utils/PathNormalizer.js +274 -0
- package/.claude/worktrees/agent-multi-profile/src/utils/WatchEventHandler.js +522 -0
- package/.claude/worktrees/agent-multi-profile/supabase/migrations/001_create_initial_schema.sql +366 -0
- package/.claude/worktrees/agent-multi-profile/supabase/migrations/002_align_with_arela_api_schema.sql +145 -0
- package/.claude/worktrees/agent-multi-profile/tests/commands/IdentifyCommand.test.js +570 -0
- package/.claude/worktrees/agent-multi-profile/tests/commands/PropagateCommand.test.js +568 -0
- package/.claude/worktrees/agent-multi-profile/tests/commands/PushCommand.test.js +754 -0
- package/.claude/worktrees/agent-multi-profile/tests/commands/ScanCommand.test.js +382 -0
- package/.claude/worktrees/agent-multi-profile/tests/unit/PathAndTableNameGeneration.test.js +1211 -0
- package/.claude/worktrees/agent-multi-profile/tests/unit/factura-inter-agencia.test.js +218 -0
- package/.claude/worktrees/agent-multi-profile/tests/unit/pedimento-completo-xml-matcher.test.js +271 -0
- package/.claude/worktrees/agent-multi-profile/tests/unit/pedimento-simplificado-matcher.test.js +185 -0
- package/.claude/worktrees/agent-multi-profile/tests/unit/scoring-engine.test.js +221 -0
- package/README.md +65 -0
- package/package.json +1 -1
- package/src/commands/AgentCommand.js +210 -0
- package/src/commands/AgentInitCommand.js +316 -0
- package/src/commands/PollWorkerCommand.js +11 -322
- package/src/config/config.js +44 -6
- package/src/index.js +55 -0
- package/src/services/LoggingService.js +35 -0
- package/src/services/PipelineApiService.js +7 -1
- package/src/services/PipelineJobRunner.js +389 -0
- package/src/services/ProfileManager.js +164 -0
|
@@ -0,0 +1,708 @@
|
|
|
1
|
+
import cliProgress from 'cli-progress';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import pLimit from 'p-limit';
|
|
4
|
+
|
|
5
|
+
import logger from '../services/LoggingService.js';
|
|
6
|
+
|
|
7
|
+
import appConfig from '../config/config.js';
|
|
8
|
+
import ErrorHandler from '../errors/ErrorHandler.js';
|
|
9
|
+
import { ConfigurationError } from '../errors/ErrorTypes.js';
|
|
10
|
+
import FileDetectionService from '../file-detection.js';
|
|
11
|
+
import { adaptDbMatchers } from '../scoring/db-matcher-adapter.js';
|
|
12
|
+
import { scoringMatchers } from '../scoring/matchers-seed.js';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Paid pedimento detected_type values.
|
|
16
|
+
*/
|
|
17
|
+
const DETECTED_PEDIMENTO_TYPES = new Set([
|
|
18
|
+
'pedimento_simplificado',
|
|
19
|
+
'pedimento_completo',
|
|
20
|
+
'pedimento_completo_xml',
|
|
21
|
+
]);
|
|
22
|
+
|
|
23
|
+
/** Unpaid pedimento detected_type values (proforma variants). */
|
|
24
|
+
const PROFORMA_TYPES = new Set([
|
|
25
|
+
'proforma',
|
|
26
|
+
'proforma_completo',
|
|
27
|
+
'proforma_completo_xml',
|
|
28
|
+
]);
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Identify Command Handler
|
|
32
|
+
* Optimized replacement for "detect --detect-pdfs"
|
|
33
|
+
* Identifies pedimento-simplificado documents in files scanned by "arela scan"
|
|
34
|
+
*
|
|
35
|
+
* Key improvements over legacy detect command:
|
|
36
|
+
* - Works with dynamic scan_* tables instead of uploader table
|
|
37
|
+
* - Uses configured API instead of direct Supabase access
|
|
38
|
+
* - Detects locally to leverage CLI host resources
|
|
39
|
+
* - Batch processing for efficient API communication
|
|
40
|
+
* - Real-time progress tracking
|
|
41
|
+
*/
|
|
42
|
+
export class IdentifyCommand {
|
|
43
|
+
constructor() {
|
|
44
|
+
this.errorHandler = new ErrorHandler(logger);
|
|
45
|
+
this.scanApiService = null; // Will be initialized in execute
|
|
46
|
+
this.detectionService = new FileDetectionService();
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Execute the identify command
|
|
51
|
+
* @param {Object} options - Command options
|
|
52
|
+
* @param {string} options.api - API target (default, agencia, cliente)
|
|
53
|
+
* @param {number} options.batchSize - Batch size for API operations
|
|
54
|
+
* @param {boolean} options.showStats - Show performance statistics
|
|
55
|
+
* @param {Function} options.onProgress - Optional callback for progress updates (percent, message)
|
|
56
|
+
*/
|
|
57
|
+
async execute(options = {}) {
|
|
58
|
+
const startTime = Date.now();
|
|
59
|
+
this.onProgress = options.onProgress || null;
|
|
60
|
+
|
|
61
|
+
try {
|
|
62
|
+
// Validate scan configuration (need same config as scan command)
|
|
63
|
+
appConfig.validateScanConfig();
|
|
64
|
+
|
|
65
|
+
// Determine API target
|
|
66
|
+
const apiTarget = options.api || 'default';
|
|
67
|
+
|
|
68
|
+
// Import ScanApiService dynamically and initialize with target
|
|
69
|
+
const { default: ScanApiService } = await import(
|
|
70
|
+
'../services/ScanApiService.js'
|
|
71
|
+
);
|
|
72
|
+
this.scanApiService = new ScanApiService(apiTarget);
|
|
73
|
+
|
|
74
|
+
// Load matchers for best-match classification (phase 4 hybrid). Prefer the
|
|
75
|
+
// DB-resolved set (this RFC + globals); fall back to the validated local
|
|
76
|
+
// seed; set DISABLE_SCORING_MATCHERS=true to force legacy first-match.
|
|
77
|
+
if (process.env.DISABLE_SCORING_MATCHERS === 'true') {
|
|
78
|
+
logger.info('š§© Scoring matchers disabled ā legacy detection');
|
|
79
|
+
} else {
|
|
80
|
+
let matchers = null;
|
|
81
|
+
try {
|
|
82
|
+
const rfc = process.env.MATCHER_RFC || null;
|
|
83
|
+
const dbMatchers = await this.scanApiService.getResolvedMatchers(rfc);
|
|
84
|
+
if (dbMatchers.length) {
|
|
85
|
+
matchers = adaptDbMatchers(dbMatchers);
|
|
86
|
+
logger.info(`š§© Loaded ${matchers.length} matchers from API`);
|
|
87
|
+
}
|
|
88
|
+
} catch (err) {
|
|
89
|
+
logger.warn(`š§© Could not load matchers from API: ${err.message}`);
|
|
90
|
+
}
|
|
91
|
+
if (!matchers) {
|
|
92
|
+
matchers = scoringMatchers;
|
|
93
|
+
logger.info(`š§© Using local seed matchers (${matchers.length})`);
|
|
94
|
+
}
|
|
95
|
+
if (typeof this.detectionService.setMatchers === 'function') {
|
|
96
|
+
this.detectionService.setMatchers(matchers);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const scanConfig = appConfig.getScanConfig();
|
|
101
|
+
const batchSize = parseInt(options.batchSize) || 100;
|
|
102
|
+
|
|
103
|
+
// Parse optional path prefix mapping: "O:/=/Volumes/nas/"
|
|
104
|
+
const pathPrefixMap = options.pathPrefix
|
|
105
|
+
? this.#parsePathPrefix(options.pathPrefix)
|
|
106
|
+
: null;
|
|
107
|
+
|
|
108
|
+
logger.info('š Starting arela identify command');
|
|
109
|
+
logger.info(`šÆ API Target: ${apiTarget}`);
|
|
110
|
+
logger.info(`š¦ Batch Size: ${batchSize}`);
|
|
111
|
+
if (options.table) logger.info(`š Target table: ${options.table}`);
|
|
112
|
+
if (options.resetAttempts)
|
|
113
|
+
logger.info('ā»ļø Reset detection attempts: ON');
|
|
114
|
+
if (pathPrefixMap)
|
|
115
|
+
logger.info(
|
|
116
|
+
`šŗ Path prefix map: ${pathPrefixMap.from} ā ${pathPrefixMap.to}`,
|
|
117
|
+
);
|
|
118
|
+
|
|
119
|
+
// Resolve the list of tables to process
|
|
120
|
+
let tables;
|
|
121
|
+
if (options.fileId && options.table) {
|
|
122
|
+
// Single-file mode ā identify exactly one file record
|
|
123
|
+
logger.info(
|
|
124
|
+
`\nšÆ Single-file mode: ${options.table} / ${options.fileId}`,
|
|
125
|
+
);
|
|
126
|
+
this.#reportProgress(0, `Fetching file record ${options.fileId}...`);
|
|
127
|
+
const record = await this.scanApiService.getFileRecord(
|
|
128
|
+
options.table,
|
|
129
|
+
options.fileId,
|
|
130
|
+
);
|
|
131
|
+
const results = await this.#detectFilesLocally(
|
|
132
|
+
[record],
|
|
133
|
+
1,
|
|
134
|
+
pathPrefixMap,
|
|
135
|
+
);
|
|
136
|
+
const updates = results.filter((r) => r !== null);
|
|
137
|
+
if (updates.length > 0) {
|
|
138
|
+
await this.scanApiService.batchUpdateDetection(
|
|
139
|
+
options.table,
|
|
140
|
+
updates,
|
|
141
|
+
);
|
|
142
|
+
}
|
|
143
|
+
this.#reportProgress(100, `Single-file identification complete`);
|
|
144
|
+
logger.success(`\nā
Single-file identification complete`);
|
|
145
|
+
const firstUpdate = updates[0];
|
|
146
|
+
return {
|
|
147
|
+
processed: 1,
|
|
148
|
+
detected: updates.length,
|
|
149
|
+
proformas: 0,
|
|
150
|
+
errors: 1 - updates.length,
|
|
151
|
+
detectedType: firstUpdate?.detectedType ?? null,
|
|
152
|
+
detectedPedimento: firstUpdate?.detectedPedimento ?? null,
|
|
153
|
+
rfc: firstUpdate?.rfc ?? null,
|
|
154
|
+
arelaPath: firstUpdate?.arelaPath ?? null,
|
|
155
|
+
};
|
|
156
|
+
} else if (options.table) {
|
|
157
|
+
// Single-table mode ā no need to match instance tables
|
|
158
|
+
tables = [{ tableName: options.table }];
|
|
159
|
+
logger.info(`\nš Single-table mode: ${options.table}`);
|
|
160
|
+
} else {
|
|
161
|
+
logger.info('\nš Fetching instance tables...');
|
|
162
|
+
tables = await this.scanApiService.getInstanceTables(
|
|
163
|
+
scanConfig.companySlug,
|
|
164
|
+
scanConfig.serverId,
|
|
165
|
+
scanConfig.basePathFull,
|
|
166
|
+
);
|
|
167
|
+
|
|
168
|
+
if (tables.length === 0) {
|
|
169
|
+
throw new ConfigurationError(
|
|
170
|
+
'No tables found for this instance. Run "arela scan" first.',
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
logger.info(
|
|
175
|
+
`š Found ${tables.length} table${tables.length === 1 ? '' : 's'} to process`,
|
|
176
|
+
);
|
|
177
|
+
for (const table of tables) {
|
|
178
|
+
logger.info(` - ${table.tableName}`);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// Optionally reset detection attempts so previously-failed files are retried
|
|
183
|
+
if (options.resetAttempts) {
|
|
184
|
+
for (const table of tables) {
|
|
185
|
+
const { reset } = await this.scanApiService.resetDetectionAttempts(
|
|
186
|
+
table.tableName,
|
|
187
|
+
);
|
|
188
|
+
logger.info(
|
|
189
|
+
`ā»ļø Reset ${reset} detection attempt(s) in ${table.tableName}`,
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// Process each table
|
|
195
|
+
let totalStats = {
|
|
196
|
+
processed: 0,
|
|
197
|
+
detected: 0,
|
|
198
|
+
proformas: 0,
|
|
199
|
+
errors: 0,
|
|
200
|
+
pending: 0,
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
// Report initial progress
|
|
204
|
+
this.#reportProgress(
|
|
205
|
+
0,
|
|
206
|
+
`Starting identification on ${tables.length} tables`,
|
|
207
|
+
);
|
|
208
|
+
|
|
209
|
+
for (let i = 0; i < tables.length; i++) {
|
|
210
|
+
const table = tables[i];
|
|
211
|
+
logger.info(`\nš Processing table: ${table.tableName}`);
|
|
212
|
+
this.#reportProgress(
|
|
213
|
+
Math.round((i / tables.length) * 100),
|
|
214
|
+
`Processing table ${i + 1}/${tables.length}: ${table.tableName}`,
|
|
215
|
+
);
|
|
216
|
+
|
|
217
|
+
// Get detection statistics for this table
|
|
218
|
+
const stats = await this.#processTable(
|
|
219
|
+
table.tableName,
|
|
220
|
+
batchSize,
|
|
221
|
+
startTime,
|
|
222
|
+
pathPrefixMap,
|
|
223
|
+
);
|
|
224
|
+
|
|
225
|
+
totalStats.processed += stats.processed;
|
|
226
|
+
totalStats.detected += stats.detected;
|
|
227
|
+
totalStats.proformas += stats.proformas;
|
|
228
|
+
totalStats.errors += stats.errors;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
const duration = ((Date.now() - startTime) / 1000).toFixed(2);
|
|
232
|
+
const avgSpeed =
|
|
233
|
+
duration > 0 ? Math.round(totalStats.processed / duration) : 0;
|
|
234
|
+
|
|
235
|
+
// Report completion
|
|
236
|
+
this.#reportProgress(
|
|
237
|
+
100,
|
|
238
|
+
`Identification completed: ${totalStats.detected} pedimentos detected`,
|
|
239
|
+
);
|
|
240
|
+
|
|
241
|
+
logger.success(`\nā
Identification Complete!`);
|
|
242
|
+
logger.info(`\nš Total Results:`);
|
|
243
|
+
logger.info(` Tables Processed: ${tables.length}`);
|
|
244
|
+
logger.info(` Files Processed: ${totalStats.processed}`);
|
|
245
|
+
logger.info(` Pedimentos Detected (paid): ${totalStats.detected}`);
|
|
246
|
+
logger.info(` Proformas Detected (unpaid): ${totalStats.proformas}`);
|
|
247
|
+
logger.info(` Errors: ${totalStats.errors}`);
|
|
248
|
+
logger.info(` Duration: ${duration}s`);
|
|
249
|
+
logger.info(` Speed: ${avgSpeed} files/sec`);
|
|
250
|
+
|
|
251
|
+
// Classification distribution for the run (observability: detect shifts /
|
|
252
|
+
// rising unmatched rate, and which detection mode was used).
|
|
253
|
+
if (
|
|
254
|
+
typeof this.detectionService.logClassificationSummary === 'function'
|
|
255
|
+
) {
|
|
256
|
+
this.detectionService.logClassificationSummary(logger);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
if (options.showStats) {
|
|
260
|
+
this.#showDetailedStats(
|
|
261
|
+
startTime,
|
|
262
|
+
totalStats.processed,
|
|
263
|
+
totalStats.detected,
|
|
264
|
+
totalStats.errors,
|
|
265
|
+
);
|
|
266
|
+
}
|
|
267
|
+
} catch (error) {
|
|
268
|
+
logger.error('ā Identification failed:', error.message);
|
|
269
|
+
|
|
270
|
+
if (error instanceof ConfigurationError) {
|
|
271
|
+
logger.error('\nConfiguration errors:');
|
|
272
|
+
error.errors.forEach((err) => logger.error(` - ${err}`));
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
throw error;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Process a single table
|
|
281
|
+
* @private
|
|
282
|
+
* @param {string} tableName - Table name to process
|
|
283
|
+
* @param {number} batchSize - Batch size
|
|
284
|
+
* @param {number} startTime - Start time for speed calculation
|
|
285
|
+
* @returns {Promise<Object>} Processing statistics
|
|
286
|
+
*/
|
|
287
|
+
async #processTable(tableName, batchSize, startTime, pathPrefixMap = null) {
|
|
288
|
+
// Get detection statistics first (allTypes=true to count all supported file types)
|
|
289
|
+
const initialStats = await this.scanApiService.getDetectionStats(
|
|
290
|
+
tableName,
|
|
291
|
+
true,
|
|
292
|
+
);
|
|
293
|
+
logger.info(` Total PDFs: ${initialStats.totalPdfs}`);
|
|
294
|
+
logger.info(` Detected: ${initialStats.detected}`);
|
|
295
|
+
logger.info(` Pending: ${initialStats.pending}`);
|
|
296
|
+
logger.info(` Not Pedimento: ${initialStats.notPedimento || 0}`);
|
|
297
|
+
logger.info(
|
|
298
|
+
` Max Attempts Reached: ${initialStats.maxAttemptsReached || 0}`,
|
|
299
|
+
);
|
|
300
|
+
logger.info(` Errors: ${initialStats.errors}`);
|
|
301
|
+
|
|
302
|
+
if (initialStats.pending === 0) {
|
|
303
|
+
logger.info(' ā
All PDFs processed. Skipping.');
|
|
304
|
+
|
|
305
|
+
if (initialStats.maxAttemptsReached > 0) {
|
|
306
|
+
logger.info(
|
|
307
|
+
` ā ļø ${initialStats.maxAttemptsReached} PDFs reached max attempts.`,
|
|
308
|
+
);
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
return {
|
|
312
|
+
processed: 0,
|
|
313
|
+
detected: 0,
|
|
314
|
+
// Must include every field totalStats accumulates ā omitting proformas
|
|
315
|
+
// made `totalStats.proformas += undefined` = NaN in the final summary.
|
|
316
|
+
proformas: 0,
|
|
317
|
+
errors: 0,
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
logger.info(` š Processing ${initialStats.pending} pending PDFs...`);
|
|
322
|
+
|
|
323
|
+
logger.info(` š Processing ${initialStats.pending} pending PDFs...`);
|
|
324
|
+
|
|
325
|
+
// Setup progress bar
|
|
326
|
+
const progressBar = new cliProgress.SingleBar({
|
|
327
|
+
format:
|
|
328
|
+
' š |{bar}| {percentage}% | {value}/{total} files | {speed} files/sec',
|
|
329
|
+
barCompleteChar: '\u2588',
|
|
330
|
+
barIncompleteChar: '\u2591',
|
|
331
|
+
hideCursor: true,
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
progressBar.start(initialStats.pending, 0, { speed: 0 });
|
|
335
|
+
|
|
336
|
+
let processedCount = 0;
|
|
337
|
+
let detectedCount = 0;
|
|
338
|
+
let proformaCount = 0;
|
|
339
|
+
let errorCount = 0;
|
|
340
|
+
let hasMore = true;
|
|
341
|
+
let batchNumber = 0;
|
|
342
|
+
|
|
343
|
+
// Process in batches
|
|
344
|
+
while (hasMore) {
|
|
345
|
+
batchNumber++;
|
|
346
|
+
|
|
347
|
+
// Fetch from offset 0 since processed files are filtered out
|
|
348
|
+
const response = await this.scanApiService.fetchPdfsForDetection(
|
|
349
|
+
tableName,
|
|
350
|
+
0,
|
|
351
|
+
batchSize,
|
|
352
|
+
true, // allTypes: fetch all supported file types, not just likely-simplificado PDFs
|
|
353
|
+
);
|
|
354
|
+
|
|
355
|
+
if (!response.data || response.data.length === 0) {
|
|
356
|
+
break;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
const files = response.data;
|
|
360
|
+
|
|
361
|
+
// Detect files locally with concurrent processing
|
|
362
|
+
const detectionResults = await this.#detectFilesLocally(
|
|
363
|
+
files,
|
|
364
|
+
10,
|
|
365
|
+
pathPrefixMap,
|
|
366
|
+
);
|
|
367
|
+
|
|
368
|
+
// Batch update to API
|
|
369
|
+
const updateResult = await this.scanApiService.batchUpdateDetection(
|
|
370
|
+
tableName,
|
|
371
|
+
detectionResults,
|
|
372
|
+
);
|
|
373
|
+
|
|
374
|
+
// Update statistics
|
|
375
|
+
processedCount += files.length;
|
|
376
|
+
// "Detected" counts paid pedimentos of any flavour (simplificado,
|
|
377
|
+
// completo, completo_xml when enabled).
|
|
378
|
+
detectedCount += detectionResults.filter((r) =>
|
|
379
|
+
DETECTED_PEDIMENTO_TYPES.has(r.detectedType),
|
|
380
|
+
).length;
|
|
381
|
+
proformaCount += detectionResults.filter((r) =>
|
|
382
|
+
PROFORMA_TYPES.has(r.detectedType),
|
|
383
|
+
).length;
|
|
384
|
+
errorCount += detectionResults.filter((r) => r.detectionError).length;
|
|
385
|
+
|
|
386
|
+
// Update progress bar
|
|
387
|
+
const elapsed = (Date.now() - startTime) / 1000;
|
|
388
|
+
const speed = elapsed > 0 ? Math.round(processedCount / elapsed) : 0;
|
|
389
|
+
progressBar.update(processedCount, { speed });
|
|
390
|
+
|
|
391
|
+
// Check if there are more files
|
|
392
|
+
hasMore = response.hasMore;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
progressBar.stop();
|
|
396
|
+
|
|
397
|
+
return {
|
|
398
|
+
processed: processedCount,
|
|
399
|
+
detected: detectedCount,
|
|
400
|
+
proformas: proformaCount,
|
|
401
|
+
errors: errorCount,
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/**
|
|
406
|
+
* Detect files locally using FileDetectionService
|
|
407
|
+
* @private
|
|
408
|
+
* @param {Array} files - Files to detect
|
|
409
|
+
* @param {number} concurrency - Maximum concurrent detections
|
|
410
|
+
* @returns {Promise<Array>} Detection results
|
|
411
|
+
*/
|
|
412
|
+
async #detectFilesLocally(files, concurrency = 10, pathPrefixMap = null) {
|
|
413
|
+
const limit = pLimit(concurrency);
|
|
414
|
+
const basePath = appConfig.getBasePath();
|
|
415
|
+
|
|
416
|
+
const detectionPromises = files.map((file) =>
|
|
417
|
+
limit(async () => {
|
|
418
|
+
try {
|
|
419
|
+
// Check if file exists on filesystem
|
|
420
|
+
let absolutePath = file.absolute_path;
|
|
421
|
+
|
|
422
|
+
// Apply cross-platform path prefix mapping if configured
|
|
423
|
+
if (
|
|
424
|
+
pathPrefixMap &&
|
|
425
|
+
absolutePath &&
|
|
426
|
+
absolutePath.startsWith(pathPrefixMap.from)
|
|
427
|
+
) {
|
|
428
|
+
absolutePath =
|
|
429
|
+
pathPrefixMap.to + absolutePath.slice(pathPrefixMap.from.length);
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
if (!fs.existsSync(absolutePath)) {
|
|
433
|
+
return {
|
|
434
|
+
id: file.id,
|
|
435
|
+
detectedType: null,
|
|
436
|
+
detectedPedimento: null,
|
|
437
|
+
detectedPedimentoYear: null,
|
|
438
|
+
rfc: null,
|
|
439
|
+
arelaPath: null,
|
|
440
|
+
detectionError:
|
|
441
|
+
'FILE_NOT_FOUND: File does not exist on filesystem. May have been moved or deleted after scan.',
|
|
442
|
+
isPedimento: null, // Unknown - can't determine
|
|
443
|
+
};
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
// Check file size - skip very large files
|
|
447
|
+
const stats = fs.statSync(absolutePath);
|
|
448
|
+
const maxSizeBytes = 50 * 1024 * 1024; // 50MB
|
|
449
|
+
if (stats.size > maxSizeBytes) {
|
|
450
|
+
return {
|
|
451
|
+
id: file.id,
|
|
452
|
+
detectedType: null,
|
|
453
|
+
detectedPedimento: null,
|
|
454
|
+
detectedPedimentoYear: null,
|
|
455
|
+
rfc: null,
|
|
456
|
+
arelaPath: null,
|
|
457
|
+
detectionError: `FILE_TOO_LARGE: File size ${(stats.size / 1024 / 1024).toFixed(2)}MB exceeds ${maxSizeBytes / 1024 / 1024}MB limit.`,
|
|
458
|
+
isPedimento: null, // Unknown - can't determine
|
|
459
|
+
};
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
// Detect using existing FileDetectionService
|
|
463
|
+
const result = await this.detectionService.detectFile(absolutePath);
|
|
464
|
+
|
|
465
|
+
// If detection succeeded and found a paid pedimento (any variant)
|
|
466
|
+
if (DETECTED_PEDIMENTO_TYPES.has(result.detectedType)) {
|
|
467
|
+
return {
|
|
468
|
+
id: file.id,
|
|
469
|
+
detectedType: result.detectedType,
|
|
470
|
+
detectedPedimento: result.detectedPedimento,
|
|
471
|
+
detectedPedimentoYear: result.detectedPedimentoYear,
|
|
472
|
+
rfc: result.rfc,
|
|
473
|
+
arelaPath: result.arelaPath,
|
|
474
|
+
detectionError: result.error,
|
|
475
|
+
isPedimento: true, // Confirmed paid pedimento
|
|
476
|
+
};
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
// If detection succeeded and found a proforma (any variant)
|
|
480
|
+
if (PROFORMA_TYPES.has(result.detectedType)) {
|
|
481
|
+
return {
|
|
482
|
+
id: file.id,
|
|
483
|
+
detectedType: result.detectedType,
|
|
484
|
+
detectedPedimento: result.detectedPedimento,
|
|
485
|
+
detectedPedimentoYear: result.detectedPedimentoYear,
|
|
486
|
+
rfc: result.rfc,
|
|
487
|
+
arelaPath: null, // Proformas don't get arela_path (not paid yet)
|
|
488
|
+
detectionError: result.error,
|
|
489
|
+
isPedimento: false, // Not a paid pedimento ā proforma only
|
|
490
|
+
};
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
// If no detection, determine if it's definitely not a pedimento
|
|
494
|
+
// This helps avoid re-processing files we know aren't pedimentos
|
|
495
|
+
const isDefinitelyNotPedimento = this.#isDefinitelyNotPedimento(
|
|
496
|
+
result,
|
|
497
|
+
file,
|
|
498
|
+
);
|
|
499
|
+
|
|
500
|
+
// Build descriptive error message
|
|
501
|
+
let detectionError = null;
|
|
502
|
+
if (result.error) {
|
|
503
|
+
detectionError = `DETECTION_ERROR: ${result.error}`;
|
|
504
|
+
} else if (isDefinitelyNotPedimento) {
|
|
505
|
+
detectionError =
|
|
506
|
+
'NOT_PEDIMENTO: File does not match any pedimento pattern. Missing key markers (e.g. "FORMA SIMPLIFICADA DE PEDIMENTO" or "NUM. PEDIMENTO:" + copy markers).';
|
|
507
|
+
} else {
|
|
508
|
+
// Partial match - might be a pedimento with missing fields
|
|
509
|
+
const missingFields = this.#getMissingFields(result);
|
|
510
|
+
if (missingFields.length > 0) {
|
|
511
|
+
detectionError = `INCOMPLETE_PEDIMENTO: Detected as potential pedimento but missing fields: ${missingFields.join(', ')}. Matcher may need improvement.`;
|
|
512
|
+
} else {
|
|
513
|
+
detectionError =
|
|
514
|
+
'UNKNOWN_ERROR: Detection completed but no pedimento found. Check file content and matcher patterns.';
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
return {
|
|
519
|
+
id: file.id,
|
|
520
|
+
detectedType: result.detectedType,
|
|
521
|
+
detectedPedimento: result.detectedPedimento,
|
|
522
|
+
detectedPedimentoYear: result.detectedPedimentoYear,
|
|
523
|
+
rfc: result.rfc,
|
|
524
|
+
arelaPath: result.arelaPath,
|
|
525
|
+
detectionError,
|
|
526
|
+
isPedimento: isDefinitelyNotPedimento ? false : null, // false = not pedimento, null = unknown
|
|
527
|
+
};
|
|
528
|
+
} catch (error) {
|
|
529
|
+
logger.warn(
|
|
530
|
+
`Failed to detect ${file.relative_path}: ${error.message}`,
|
|
531
|
+
);
|
|
532
|
+
|
|
533
|
+
// Categorize the error
|
|
534
|
+
let errorCategory = 'UNKNOWN_ERROR';
|
|
535
|
+
if (error.message.includes('ENOENT')) {
|
|
536
|
+
errorCategory = 'FILE_NOT_FOUND';
|
|
537
|
+
} else if (error.message.includes('timeout')) {
|
|
538
|
+
errorCategory = 'TIMEOUT';
|
|
539
|
+
} else if (error.message.includes('PDF')) {
|
|
540
|
+
errorCategory = 'PDF_PARSE_ERROR';
|
|
541
|
+
} else if (error.message.includes('extract')) {
|
|
542
|
+
errorCategory = 'TEXT_EXTRACTION_ERROR';
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
return {
|
|
546
|
+
id: file.id,
|
|
547
|
+
detectedType: null,
|
|
548
|
+
detectedPedimento: null,
|
|
549
|
+
detectedPedimentoYear: null,
|
|
550
|
+
rfc: null,
|
|
551
|
+
arelaPath: null,
|
|
552
|
+
detectionError: `${errorCategory}: ${error.message}`,
|
|
553
|
+
isPedimento: null, // Unknown - error occurred
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
}),
|
|
557
|
+
);
|
|
558
|
+
|
|
559
|
+
return Promise.all(detectionPromises);
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* Determine if a file is definitely not a pedimento/proforma document
|
|
564
|
+
* @private
|
|
565
|
+
* @param {Object} result - Detection result
|
|
566
|
+
* @param {Object} file - File metadata
|
|
567
|
+
* @returns {boolean} True if definitely not a pedimento/proforma
|
|
568
|
+
*/
|
|
569
|
+
#isDefinitelyNotPedimento(result, file) {
|
|
570
|
+
// If we got any pedimento-related fields, it might be a pedimento or proforma
|
|
571
|
+
if (result.detectedPedimento || result.rfc || result.arelaPath) {
|
|
572
|
+
return false;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
// If it was detected as a proforma (any variant), it's related to a
|
|
576
|
+
// pedimento structure ā not "definitely not".
|
|
577
|
+
if (PROFORMA_TYPES.has(result.detectedType)) {
|
|
578
|
+
return false;
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
// Check if the text contains any required pedimento marker. This must
|
|
582
|
+
// stay aligned with the `match()` predicates in pedimento-simplificado.js
|
|
583
|
+
// and pedimento-completo.js (which accept both "DE" and "DEL" in the
|
|
584
|
+
// title, and treat the colon after "T. OPER" as optional).
|
|
585
|
+
const text = result.text || '';
|
|
586
|
+
const hasSimplificadoMarker =
|
|
587
|
+
/FORMA\s+SIMPLIFICADA\s+DEL?\s+PEDIMENTO/i.test(text);
|
|
588
|
+
const hasCompletoMarkers =
|
|
589
|
+
/NUM\.?\s*PEDIMENTO:/i.test(text) &&
|
|
590
|
+
/CVE\.?\s*PEDIMENTO:/i.test(text) &&
|
|
591
|
+
/T\.?\s*OPER:?/i.test(text);
|
|
592
|
+
|
|
593
|
+
return !hasSimplificadoMarker && !hasCompletoMarkers;
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
/**
|
|
597
|
+
* Get list of missing required fields for pedimento
|
|
598
|
+
* @private
|
|
599
|
+
* @param {Object} result - Detection result
|
|
600
|
+
* @returns {Array<string>} Missing field names
|
|
601
|
+
*/
|
|
602
|
+
#getMissingFields(result) {
|
|
603
|
+
const requiredFields = [
|
|
604
|
+
{ key: 'detectedPedimento', name: 'numPedimento' },
|
|
605
|
+
{ key: 'rfc', name: 'rfc' },
|
|
606
|
+
{ key: 'detectedPedimentoYear', name: 'year' },
|
|
607
|
+
];
|
|
608
|
+
|
|
609
|
+
const missing = [];
|
|
610
|
+
for (const field of requiredFields) {
|
|
611
|
+
if (!result[field.key]) {
|
|
612
|
+
missing.push(field.name);
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
return missing;
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
/**
|
|
620
|
+
* Show detailed performance statistics
|
|
621
|
+
* @private
|
|
622
|
+
*/
|
|
623
|
+
#showDetailedStats(startTime, processedCount, detectedCount, errorCount) {
|
|
624
|
+
const duration = (Date.now() - startTime) / 1000;
|
|
625
|
+
const avgSpeed = duration > 0 ? (processedCount / duration).toFixed(2) : 0;
|
|
626
|
+
const detectionRate =
|
|
627
|
+
processedCount > 0
|
|
628
|
+
? ((detectedCount / processedCount) * 100).toFixed(1)
|
|
629
|
+
: 0;
|
|
630
|
+
|
|
631
|
+
logger.info('\nš Detailed Statistics:');
|
|
632
|
+
logger.info(` Total Processing Time: ${duration.toFixed(2)}s`);
|
|
633
|
+
logger.info(` Average Speed: ${avgSpeed} files/sec`);
|
|
634
|
+
logger.info(` Detection Rate: ${detectionRate}%`);
|
|
635
|
+
logger.info(
|
|
636
|
+
` Error Rate: ${errorCount > 0 ? ((errorCount / processedCount) * 100).toFixed(1) : 0}%`,
|
|
637
|
+
);
|
|
638
|
+
|
|
639
|
+
// Memory usage
|
|
640
|
+
const memUsage = process.memoryUsage();
|
|
641
|
+
logger.info('\nš¾ Memory Usage:');
|
|
642
|
+
logger.info(` RSS: ${(memUsage.rss / 1024 / 1024).toFixed(2)} MB`);
|
|
643
|
+
logger.info(
|
|
644
|
+
` Heap Used: ${(memUsage.heapUsed / 1024 / 1024).toFixed(2)} MB`,
|
|
645
|
+
);
|
|
646
|
+
logger.info(
|
|
647
|
+
` Heap Total: ${(memUsage.heapTotal / 1024 / 1024).toFixed(2)} MB`,
|
|
648
|
+
);
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
/**
|
|
652
|
+
* Report progress to callback if available (for worker mode)
|
|
653
|
+
* @private
|
|
654
|
+
* @param {number} percent - Progress percentage (0-100)
|
|
655
|
+
* @param {string} message - Progress message
|
|
656
|
+
*/
|
|
657
|
+
#reportProgress(percent, message) {
|
|
658
|
+
if (this.onProgress && typeof this.onProgress === 'function') {
|
|
659
|
+
try {
|
|
660
|
+
this.onProgress(percent, message);
|
|
661
|
+
} catch (error) {
|
|
662
|
+
logger.debug(`Progress callback error: ${error.message}`);
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
/**
|
|
668
|
+
* Parse a path prefix mapping string such as "O:/=/Volumes/nas/" into { from, to }.
|
|
669
|
+
* Supports both "FROM=TO" and "FROM:TO" separators.
|
|
670
|
+
* @private
|
|
671
|
+
* @param {string} mapping
|
|
672
|
+
* @returns {{ from: string, to: string }}
|
|
673
|
+
*/
|
|
674
|
+
#parsePathPrefix(mapping) {
|
|
675
|
+
// Support either "FROM=TO" or "FROM:TO" as separator
|
|
676
|
+
const eqIdx = mapping.indexOf('=');
|
|
677
|
+
const colonIdx = mapping.indexOf(':');
|
|
678
|
+
|
|
679
|
+
let sep = -1;
|
|
680
|
+
// "O:/=/Volumes" ā the colon inside "O:/" is part of a Windows drive letter; prefer '=' separator
|
|
681
|
+
if (eqIdx !== -1) {
|
|
682
|
+
sep = eqIdx;
|
|
683
|
+
} else if (colonIdx !== -1) {
|
|
684
|
+
sep = colonIdx;
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
if (sep === -1) {
|
|
688
|
+
throw new Error(
|
|
689
|
+
`Invalid --path-prefix format: "${mapping}". Expected "FROM=TO" e.g. "O:/=/Volumes/nas/"`,
|
|
690
|
+
);
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
const from = mapping.slice(0, sep);
|
|
694
|
+
const to = mapping.slice(sep + 1);
|
|
695
|
+
|
|
696
|
+
if (!from || !to) {
|
|
697
|
+
throw new Error(
|
|
698
|
+
`Invalid --path-prefix format: "${mapping}". Both FROM and TO parts must be non-empty.`,
|
|
699
|
+
);
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
return { from, to };
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
// Export singleton instance
|
|
707
|
+
const identifyCommand = new IdentifyCommand();
|
|
708
|
+
export default identifyCommand;
|