@arela/uploader 1.1.3 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/.claude/worktrees/agent-multi-profile/.env.template +224 -0
  2. package/.claude/worktrees/agent-multi-profile/.prettierrc +13 -0
  3. package/.claude/worktrees/agent-multi-profile/README.md +405 -0
  4. package/.claude/worktrees/agent-multi-profile/package-lock.json +7096 -0
  5. package/.claude/worktrees/agent-multi-profile/package.json +78 -0
  6. package/.claude/worktrees/agent-multi-profile/scripts/cleanup-ds-store.js +109 -0
  7. package/.claude/worktrees/agent-multi-profile/scripts/cleanup-system-files.js +69 -0
  8. package/.claude/worktrees/agent-multi-profile/scripts/scoring-compare.js +243 -0
  9. package/.claude/worktrees/agent-multi-profile/scripts/scoring-phase4-check.js +96 -0
  10. package/.claude/worktrees/agent-multi-profile/scripts/tests/phase-7-features.test.js +415 -0
  11. package/.claude/worktrees/agent-multi-profile/scripts/tests/signal-handling.test.js +275 -0
  12. package/.claude/worktrees/agent-multi-profile/scripts/tests/smart-watch-integration.test.js +554 -0
  13. package/.claude/worktrees/agent-multi-profile/scripts/tests/watch-service-integration.test.js +584 -0
  14. package/.claude/worktrees/agent-multi-profile/src/commands/AgentCommand.js +229 -0
  15. package/.claude/worktrees/agent-multi-profile/src/commands/AgentInitCommand.js +316 -0
  16. package/.claude/worktrees/agent-multi-profile/src/commands/DatastageCommand.js +164 -0
  17. package/.claude/worktrees/agent-multi-profile/src/commands/GDriveSyncCommand.js +475 -0
  18. package/.claude/worktrees/agent-multi-profile/src/commands/IdentifyCommand.js +708 -0
  19. package/.claude/worktrees/agent-multi-profile/src/commands/PollWorkerCommand.js +169 -0
  20. package/.claude/worktrees/agent-multi-profile/src/commands/PropagateCommand.js +636 -0
  21. package/.claude/worktrees/agent-multi-profile/src/commands/PushCommand.js +743 -0
  22. package/.claude/worktrees/agent-multi-profile/src/commands/ScanCommand.js +722 -0
  23. package/.claude/worktrees/agent-multi-profile/src/commands/UploadCommand.js +587 -0
  24. package/.claude/worktrees/agent-multi-profile/src/commands/WatchCommand.js +1342 -0
  25. package/.claude/worktrees/agent-multi-profile/src/commands/WorkerCommand.js +337 -0
  26. package/.claude/worktrees/agent-multi-profile/src/config/config.js +862 -0
  27. package/.claude/worktrees/agent-multi-profile/src/document-type-shared.js +131 -0
  28. package/.claude/worktrees/agent-multi-profile/src/document-types/_pedimento-shared-extractors.js +348 -0
  29. package/.claude/worktrees/agent-multi-profile/src/document-types/doda-pdf.js +121 -0
  30. package/.claude/worktrees/agent-multi-profile/src/document-types/doda-xml.js +118 -0
  31. package/.claude/worktrees/agent-multi-profile/src/document-types/factura-inter-agencia.js +186 -0
  32. package/.claude/worktrees/agent-multi-profile/src/document-types/facturas-comerciales.js +233 -0
  33. package/.claude/worktrees/agent-multi-profile/src/document-types/pedimento-completo-xml.js +372 -0
  34. package/.claude/worktrees/agent-multi-profile/src/document-types/pedimento-completo.js +108 -0
  35. package/.claude/worktrees/agent-multi-profile/src/document-types/pedimento-simplificado.js +76 -0
  36. package/.claude/worktrees/agent-multi-profile/src/document-types/proforma.js +29 -0
  37. package/.claude/worktrees/agent-multi-profile/src/document-types/support-document.js +200 -0
  38. package/.claude/worktrees/agent-multi-profile/src/errors/ErrorHandler.js +278 -0
  39. package/.claude/worktrees/agent-multi-profile/src/errors/ErrorTypes.js +104 -0
  40. package/.claude/worktrees/agent-multi-profile/src/file-detection.js +338 -0
  41. package/.claude/worktrees/agent-multi-profile/src/index.js +890 -0
  42. package/.claude/worktrees/agent-multi-profile/src/scoring/db-matcher-adapter.js +98 -0
  43. package/.claude/worktrees/agent-multi-profile/src/scoring/matchers-seed.js +386 -0
  44. package/.claude/worktrees/agent-multi-profile/src/scoring/scoring-engine.js +251 -0
  45. package/.claude/worktrees/agent-multi-profile/src/services/AdvancedFilterService.js +505 -0
  46. package/.claude/worktrees/agent-multi-profile/src/services/AutoProcessingService.js +749 -0
  47. package/.claude/worktrees/agent-multi-profile/src/services/BenchmarkingService.js +381 -0
  48. package/.claude/worktrees/agent-multi-profile/src/services/DatabaseService.js +2173 -0
  49. package/.claude/worktrees/agent-multi-profile/src/services/DatastageApiService.js +240 -0
  50. package/.claude/worktrees/agent-multi-profile/src/services/ErrorMonitor.js +275 -0
  51. package/.claude/worktrees/agent-multi-profile/src/services/GoogleDriveService.js +217 -0
  52. package/.claude/worktrees/agent-multi-profile/src/services/LoggingService.js +649 -0
  53. package/.claude/worktrees/agent-multi-profile/src/services/MonitoringService.js +401 -0
  54. package/.claude/worktrees/agent-multi-profile/src/services/PerformanceOptimizer.js +511 -0
  55. package/.claude/worktrees/agent-multi-profile/src/services/PipelineApiService.js +274 -0
  56. package/.claude/worktrees/agent-multi-profile/src/services/PipelineJobRunner.js +389 -0
  57. package/.claude/worktrees/agent-multi-profile/src/services/ProfileManager.js +164 -0
  58. package/.claude/worktrees/agent-multi-profile/src/services/ReportingService.js +511 -0
  59. package/.claude/worktrees/agent-multi-profile/src/services/ScanApiService.js +775 -0
  60. package/.claude/worktrees/agent-multi-profile/src/services/SignalHandler.js +255 -0
  61. package/.claude/worktrees/agent-multi-profile/src/services/SmartWatchDatabaseService.js +527 -0
  62. package/.claude/worktrees/agent-multi-profile/src/services/WatchService.js +783 -0
  63. package/.claude/worktrees/agent-multi-profile/src/services/upload/ApiUploadService.js +676 -0
  64. package/.claude/worktrees/agent-multi-profile/src/services/upload/BaseUploadService.js +36 -0
  65. package/.claude/worktrees/agent-multi-profile/src/services/upload/MultiApiUploadService.js +233 -0
  66. package/.claude/worktrees/agent-multi-profile/src/services/upload/SupabaseUploadService.js +148 -0
  67. package/.claude/worktrees/agent-multi-profile/src/services/upload/UploadServiceFactory.js +100 -0
  68. package/.claude/worktrees/agent-multi-profile/src/utils/CleanupManager.js +262 -0
  69. package/.claude/worktrees/agent-multi-profile/src/utils/FileOperations.js +192 -0
  70. package/.claude/worktrees/agent-multi-profile/src/utils/FileSanitizer.js +99 -0
  71. package/.claude/worktrees/agent-multi-profile/src/utils/PathDetector.js +198 -0
  72. package/.claude/worktrees/agent-multi-profile/src/utils/PathNormalizer.js +274 -0
  73. package/.claude/worktrees/agent-multi-profile/src/utils/WatchEventHandler.js +522 -0
  74. package/.claude/worktrees/agent-multi-profile/supabase/migrations/001_create_initial_schema.sql +366 -0
  75. package/.claude/worktrees/agent-multi-profile/supabase/migrations/002_align_with_arela_api_schema.sql +145 -0
  76. package/.claude/worktrees/agent-multi-profile/tests/commands/IdentifyCommand.test.js +570 -0
  77. package/.claude/worktrees/agent-multi-profile/tests/commands/PropagateCommand.test.js +568 -0
  78. package/.claude/worktrees/agent-multi-profile/tests/commands/PushCommand.test.js +754 -0
  79. package/.claude/worktrees/agent-multi-profile/tests/commands/ScanCommand.test.js +382 -0
  80. package/.claude/worktrees/agent-multi-profile/tests/unit/PathAndTableNameGeneration.test.js +1211 -0
  81. package/.claude/worktrees/agent-multi-profile/tests/unit/factura-inter-agencia.test.js +218 -0
  82. package/.claude/worktrees/agent-multi-profile/tests/unit/pedimento-completo-xml-matcher.test.js +271 -0
  83. package/.claude/worktrees/agent-multi-profile/tests/unit/pedimento-simplificado-matcher.test.js +185 -0
  84. package/.claude/worktrees/agent-multi-profile/tests/unit/scoring-engine.test.js +221 -0
  85. package/README.md +65 -0
  86. package/package.json +1 -1
  87. package/src/commands/AgentCommand.js +210 -0
  88. package/src/commands/AgentInitCommand.js +316 -0
  89. package/src/commands/PollWorkerCommand.js +11 -322
  90. package/src/config/config.js +44 -6
  91. package/src/document-type-shared.js +1 -1
  92. package/src/document-types/pedimento-completo.js +8 -1
  93. package/src/file-detection.js +9 -0
  94. package/src/index.js +55 -0
  95. package/src/scoring/scoring-engine.js +1 -1
  96. package/src/services/LoggingService.js +35 -0
  97. package/src/services/PipelineApiService.js +7 -1
  98. package/src/services/PipelineJobRunner.js +389 -0
  99. package/src/services/ProfileManager.js +164 -0
@@ -0,0 +1,862 @@
1
+ import { config } from 'dotenv';
2
+ import fs from 'fs';
3
+ import path from 'path';
4
+ import { fileURLToPath } from 'url';
5
+
6
+ import PathNormalizer from '../utils/PathNormalizer.js';
7
+
8
+ config();
9
+
10
+ /**
11
+ * Configuration management for Arela Uploader
12
+ * Centralizes all environment variable handling and validation
13
+ */
14
+ class Config {
15
+ constructor() {
16
+ this.packageVersion = this.#loadPackageVersion();
17
+ this.supabase = this.#loadSupabaseConfig();
18
+ this.api = this.#loadApiConfig();
19
+ this.upload = this.#loadUploadConfig();
20
+ this.scan = this.#loadScanConfig();
21
+ this.push = this.#loadPushConfig();
22
+ this.performance = this.#loadPerformanceConfig();
23
+ this.logging = this.#loadLoggingConfig();
24
+ this.watch = this.#loadWatchConfig();
25
+ this.redis = this.#loadRedisConfig();
26
+ this.worker = this.#loadWorkerConfig();
27
+ this.gdrive = this.#loadGDriveConfig();
28
+ }
29
+
30
+ /**
31
+ * Load package version from package.json
32
+ * @private
33
+ */
34
+ #loadPackageVersion() {
35
+ try {
36
+ const __filename = fileURLToPath(import.meta.url);
37
+ const __dirname = path.dirname(__filename);
38
+ const packageJsonPath = path.resolve(__dirname, '../../package.json');
39
+ const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
40
+ return packageJson.version || '1.2.0';
41
+ } catch (error) {
42
+ console.warn('⚠️ Could not read package.json version, using fallback');
43
+ return '1.2.0';
44
+ }
45
+ }
46
+
47
+ /**
48
+ * Load Supabase configuration
49
+ * @private
50
+ */
51
+ #loadSupabaseConfig() {
52
+ return {
53
+ url: process.env.SUPABASE_URL,
54
+ key: process.env.SUPABASE_KEY,
55
+ bucket: process.env.SUPABASE_BUCKET,
56
+ };
57
+ }
58
+
59
+ /**
60
+ * Load API configuration
61
+ * Supports 3 API targets: default, agencia, cliente
62
+ * Configure ARELA_API_CLIENTE_URL/TOKEN for the specific client you want to use
63
+ * @private
64
+ */
65
+ #loadApiConfig() {
66
+ return {
67
+ // Default/legacy API (backward compatible)
68
+ baseUrl: process.env.ARELA_API_URL,
69
+ token: process.env.ARELA_API_TOKEN,
70
+ // Multi-tenant API targets (only 3 options)
71
+ targets: {
72
+ agencia: {
73
+ baseUrl:
74
+ process.env.ARELA_API_AGENCIA_URL || process.env.ARELA_API_URL,
75
+ token:
76
+ process.env.ARELA_API_AGENCIA_TOKEN || process.env.ARELA_API_TOKEN,
77
+ },
78
+ cliente: {
79
+ baseUrl:
80
+ process.env.ARELA_API_CLIENTE_URL || process.env.ARELA_API_URL,
81
+ token:
82
+ process.env.ARELA_API_CLIENTE_TOKEN || process.env.ARELA_API_TOKEN,
83
+ },
84
+ // Dynamic targets - add more as needed via ARELA_API_{NAME}_URL
85
+ ktj: {
86
+ baseUrl: process.env.ARELA_API_KTJ_URL || process.env.ARELA_API_URL,
87
+ token: process.env.ARELA_API_KTJ_TOKEN || process.env.ARELA_API_TOKEN,
88
+ },
89
+ },
90
+ // Current active target (can be changed at runtime)
91
+ activeTarget: process.env.ARELA_API_TARGET || 'default',
92
+ };
93
+ }
94
+
95
+ /**
96
+ * Get API configuration for a specific target
97
+ * @param {string} target - API target: 'agencia', 'cliente', or 'default'
98
+ * @returns {Object} API configuration with baseUrl and token
99
+ */
100
+ getApiConfig(target = null) {
101
+ const targetName = target || this.api.activeTarget || 'default';
102
+
103
+ if (targetName === 'default') {
104
+ return {
105
+ baseUrl: this.api.baseUrl,
106
+ token: this.api.token,
107
+ };
108
+ }
109
+
110
+ const targetConfig = this.api.targets[targetName];
111
+ if (!targetConfig) {
112
+ console.warn(
113
+ `⚠️ Unknown API target '${targetName}', falling back to default`,
114
+ );
115
+ return {
116
+ baseUrl: this.api.baseUrl,
117
+ token: this.api.token,
118
+ };
119
+ }
120
+
121
+ return targetConfig;
122
+ }
123
+
124
+ /**
125
+ * Set the active API target at runtime
126
+ * @param {string} target - API target: 'default', 'agencia', or 'cliente'
127
+ */
128
+ setApiTarget(target) {
129
+ const validTargets = ['default', ...Object.keys(this.api.targets)];
130
+ if (!validTargets.includes(target.toLowerCase())) {
131
+ throw new Error(
132
+ `Invalid API target '${target}'. Must be one of: ${validTargets.join(', ')}`,
133
+ );
134
+ }
135
+ this.api.activeTarget = target.toLowerCase();
136
+ const config = this.getApiConfig(target.toLowerCase());
137
+ console.log(`🎯 API target set to: ${target} → ${config.baseUrl}`);
138
+ }
139
+
140
+ /**
141
+ * Set source and target APIs for cross-tenant operations
142
+ * @param {string} sourceTarget - Source API target (for reading data)
143
+ * @param {string} targetTarget - Target API target (for writing data)
144
+ */
145
+ setCrossTenantTargets(sourceTarget, targetTarget) {
146
+ const validTargets = ['default', ...Object.keys(this.api.targets)];
147
+
148
+ if (!validTargets.includes(sourceTarget.toLowerCase())) {
149
+ throw new Error(
150
+ `Invalid source API target '${sourceTarget}'. Must be one of: ${validTargets.join(', ')}`,
151
+ );
152
+ }
153
+ if (!validTargets.includes(targetTarget.toLowerCase())) {
154
+ throw new Error(
155
+ `Invalid target API target '${targetTarget}'. Must be one of: ${validTargets.join(', ')}`,
156
+ );
157
+ }
158
+
159
+ this.api.sourceTarget = sourceTarget.toLowerCase();
160
+ this.api.targetTarget = targetTarget.toLowerCase();
161
+
162
+ const sourceConfig = this.getApiConfig(sourceTarget.toLowerCase());
163
+ const targetConfig = this.getApiConfig(targetTarget.toLowerCase());
164
+
165
+ console.log(`🔗 Cross-tenant mode enabled:`);
166
+ console.log(
167
+ ` 📖 Source (read): ${sourceTarget} → ${sourceConfig.baseUrl}`,
168
+ );
169
+ console.log(
170
+ ` 📝 Target (write): ${targetTarget} → ${targetConfig.baseUrl}`,
171
+ );
172
+ }
173
+
174
+ /**
175
+ * Upsert an API target in memory (does not touch process.env).
176
+ * Used by agent mode to register per-profile targets at runtime.
177
+ * @param {string} name - Target name, e.g. 'cliente', 'agencia'
178
+ * @param {{baseUrl: string, token: string}} targetConfig
179
+ */
180
+ setTargetConfig(name, { baseUrl, token }) {
181
+ this.api.targets[name.toLowerCase()] = { baseUrl, token };
182
+ }
183
+
184
+ /**
185
+ * Apply an agent profile: API targets, server id and env overrides.
186
+ * Idempotent — the agent calls it before every poll/job of the profile so the
187
+ * singleton always reflects the ACTIVE profile (jobs reference targets by
188
+ * symbolic name: 'agencia', 'cliente').
189
+ * @param {Object} profile - Normalized profile from ProfileManager
190
+ */
191
+ applyProfile(profile) {
192
+ this.api.baseUrl = profile.url;
193
+ this.api.token = profile.token;
194
+ for (const [name, targetConfig] of Object.entries(profile.targets)) {
195
+ this.setTargetConfig(name, targetConfig);
196
+ }
197
+ if (profile.serverId) {
198
+ process.env.ARELA_SERVER_ID = profile.serverId;
199
+ }
200
+ for (const [key, value] of Object.entries(profile.env || {})) {
201
+ process.env[key] = String(value);
202
+ }
203
+ // Clear cross-tenant state left over from a previous profile's job
204
+ this.api.sourceTarget = null;
205
+ this.api.targetTarget = null;
206
+ this.api.activeTarget = 'default';
207
+ }
208
+
209
+ /**
210
+ * Get source API config for cross-tenant operations
211
+ * @returns {Object} Source API configuration
212
+ */
213
+ getSourceApiConfig() {
214
+ if (this.api.sourceTarget) {
215
+ return this.getApiConfig(this.api.sourceTarget);
216
+ }
217
+ return this.getApiConfig();
218
+ }
219
+
220
+ /**
221
+ * Get target API config for cross-tenant operations
222
+ * @returns {Object} Target API configuration
223
+ */
224
+ getTargetApiConfig() {
225
+ if (this.api.targetTarget) {
226
+ return this.getApiConfig(this.api.targetTarget);
227
+ }
228
+ return this.getApiConfig();
229
+ }
230
+
231
+ /**
232
+ * Check if cross-tenant mode is enabled
233
+ * @returns {boolean}
234
+ */
235
+ isCrossTenantMode() {
236
+ return !!(
237
+ this.api.sourceTarget &&
238
+ this.api.targetTarget &&
239
+ this.api.sourceTarget !== this.api.targetTarget
240
+ );
241
+ }
242
+
243
+ /**
244
+ * Get API target by RFC
245
+ * @param {string} rfc - The RFC to find API target for
246
+ * @returns {string|null} The API target name or null if not found
247
+ */
248
+ getApiTargetByRfc(rfc) {
249
+ if (!rfc) return null;
250
+
251
+ const normalizedRfc = rfc.toUpperCase();
252
+ for (const [targetName, targetConfig] of Object.entries(this.api.targets)) {
253
+ if (
254
+ targetConfig.rfc &&
255
+ targetConfig.rfc.toUpperCase() === normalizedRfc
256
+ ) {
257
+ return targetName;
258
+ }
259
+ }
260
+ return null;
261
+ }
262
+
263
+ /**
264
+ * Load upload configuration
265
+ * @private
266
+ */
267
+ #loadUploadConfig() {
268
+ const basePath = process.env.UPLOAD_BASE_PATH;
269
+ const sources = process.env.UPLOAD_SOURCES?.split('|')
270
+ .map((s) => s.trim())
271
+ .filter(Boolean);
272
+
273
+ const uploadRfcs = process.env.UPLOAD_RFCS?.split('|')
274
+ .map((s) => s.trim())
275
+ .filter(Boolean);
276
+
277
+ const uploadYears = process.env.UPLOAD_YEARS?.split('|')
278
+ .map((s) => s.trim())
279
+ .filter(Boolean);
280
+
281
+ return {
282
+ basePath,
283
+ sources,
284
+ rfcs: uploadRfcs,
285
+ years: uploadYears,
286
+ };
287
+ }
288
+
289
+ /**
290
+ * Load scan configuration
291
+ * @private
292
+ */
293
+ #loadScanConfig() {
294
+ const companySlug = process.env.ARELA_COMPANY_SLUG;
295
+ const serverId = process.env.ARELA_SERVER_ID;
296
+ let basePathLabel = process.env.ARELA_BASE_PATH_LABEL;
297
+
298
+ // Auto-derive basePathLabel from UPLOAD_BASE_PATH if not set
299
+ // IMPORTANT: Always resolve to absolute path for uniqueness
300
+ if (!basePathLabel && process.env.UPLOAD_BASE_PATH) {
301
+ const basePath = process.env.UPLOAD_BASE_PATH;
302
+ // '*' is a wildcard sentinel for cross-drive scenarios — keep as-is
303
+ if (basePath === '*') {
304
+ basePathLabel = '*';
305
+ } else {
306
+ // Resolve to absolute path (handles ../sample vs ./sample correctly)
307
+ // Note: toAbsolutePath handles Windows paths (O:\...) even on macOS/Linux
308
+ basePathLabel = PathNormalizer.toAbsolutePath(basePath);
309
+ }
310
+ }
311
+
312
+ // If basePathLabel is provided, ensure it's absolute (skip wildcard)
313
+ // Use PathNormalizer.isAbsolutePath for cross-platform Windows path detection
314
+ if (
315
+ basePathLabel &&
316
+ basePathLabel !== '*' &&
317
+ !PathNormalizer.isAbsolutePath(basePathLabel)
318
+ ) {
319
+ basePathLabel = PathNormalizer.toAbsolutePath(basePathLabel);
320
+ }
321
+
322
+ // Parse exclude patterns
323
+ const defaultExcludePatterns =
324
+ '.DS_Store,Thumbs.db,desktop.ini,__pycache__,.pyc,.tmp,.swp,$RECYCLE.BIN,System Volume Information,~$*';
325
+ const excludePatterns = (
326
+ process.env.SCAN_EXCLUDE_PATTERNS || defaultExcludePatterns
327
+ )
328
+ .split(',')
329
+ .map((p) => p.trim())
330
+ .filter(Boolean);
331
+
332
+ // Parse inter-agency CFDI filename patterns. Files whose basename matches
333
+ // any of these regex patterns are flagged at scan time (likelyInterAgencia)
334
+ // so the API forces them through detection and the factura_inter_agencia
335
+ // matcher can classify them. The push pipeline then excludes them (see
336
+ // NON_PUSHABLE_TYPES_SQL in arela-api). Comma-separated regex source list.
337
+ // Default: ^SICINGR — covers NORCOM's SICINGR70-NNNNNN(...).pdf/.XML files.
338
+ const defaultInterAgenciaPatterns = '^SICINGR';
339
+ const interAgenciaPatterns = (
340
+ process.env.SCAN_INTER_AGENCIA_PATTERNS || defaultInterAgenciaPatterns
341
+ )
342
+ .split(',')
343
+ .map((p) => p.trim())
344
+ .filter(Boolean)
345
+ .map((p) => {
346
+ try {
347
+ return new RegExp(p, 'i');
348
+ } catch (err) {
349
+ console.warn(
350
+ `⚠️ Invalid SCAN_INTER_AGENCIA_PATTERNS regex "${p}": ${err.message}`,
351
+ );
352
+ return null;
353
+ }
354
+ })
355
+ .filter(Boolean);
356
+
357
+ // Generate table name if all components are available
358
+ // Note: This is just for reference; actual table names are generated dynamically
359
+ // in ScanCommand based on discovered directories and levels
360
+ let tableName = null;
361
+ if (companySlug && serverId && basePathLabel && basePathLabel !== '*') {
362
+ tableName = PathNormalizer.generateTableName({
363
+ companySlug,
364
+ serverId,
365
+ basePathLabel,
366
+ });
367
+ }
368
+
369
+ return {
370
+ companySlug,
371
+ serverId,
372
+ basePathFull: basePathLabel, // Renamed for consistency
373
+ tableName,
374
+ excludePatterns,
375
+ interAgenciaPatterns,
376
+ batchSize: parseInt(process.env.SCAN_BATCH_SIZE) || 2000,
377
+ directoryLevel: parseInt(process.env.SCAN_DIRECTORY_LEVEL) || 0,
378
+ };
379
+ }
380
+
381
+ /**
382
+ * Load push configuration
383
+ * @private
384
+ */
385
+ #loadPushConfig() {
386
+ const pushRfcs = process.env.PUSH_RFCS?.split('|')
387
+ .map((s) => s.trim().toUpperCase())
388
+ .filter(Boolean);
389
+
390
+ const pushYears = process.env.PUSH_YEARS?.split('|')
391
+ .map((s) => parseInt(s.trim(), 10))
392
+ .filter((y) => !isNaN(y));
393
+
394
+ // Clean folder structure: remove leading/trailing slashes
395
+ const folderStructure = process.env.PUSH_FOLDER_STRUCTURE?.trim()
396
+ .replace(/^\/+/, '')
397
+ .replace(/\/+$/, '');
398
+
399
+ return {
400
+ rfcs: pushRfcs,
401
+ years: pushYears,
402
+ batchSize: parseInt(process.env.PUSH_BATCH_SIZE) || 50,
403
+ uploadBatchSize: parseInt(process.env.PUSH_UPLOAD_BATCH_SIZE) || 10,
404
+ bucket:
405
+ process.env.PUSH_BUCKET || process.env.SUPABASE_BUCKET || 'archivos',
406
+ folderStructure: folderStructure || '', // Prefix for storage path (e.g., "documents" or "uploads/2024")
407
+ };
408
+ }
409
+
410
+ /**
411
+ * Validate scan configuration
412
+ * @throws {Error} If required scan configuration is missing
413
+ */
414
+ validateScanConfig() {
415
+ const errors = [];
416
+
417
+ if (!this.scan.companySlug) {
418
+ errors.push(
419
+ 'ARELA_COMPANY_SLUG is required (e.g., "acme_corp", "cliente_123")',
420
+ );
421
+ } else if (!/^[a-zA-Z0-9_-]+$/.test(this.scan.companySlug)) {
422
+ errors.push(
423
+ 'ARELA_COMPANY_SLUG must contain only alphanumeric characters, dashes, and underscores',
424
+ );
425
+ }
426
+
427
+ if (!this.scan.serverId) {
428
+ errors.push(
429
+ 'ARELA_SERVER_ID is required (e.g., "nas01", "server-mx", "storage-01")',
430
+ );
431
+ } else if (!/^[a-zA-Z0-9_-]+$/.test(this.scan.serverId)) {
432
+ errors.push(
433
+ 'ARELA_SERVER_ID must contain only alphanumeric characters, dashes, and underscores',
434
+ );
435
+ }
436
+
437
+ if (!this.upload.basePath) {
438
+ errors.push(
439
+ 'UPLOAD_BASE_PATH is required to determine the scan base path',
440
+ );
441
+ }
442
+
443
+ if (!this.scan.basePathFull) {
444
+ errors.push(
445
+ 'Could not determine base path. Set ARELA_BASE_PATH_LABEL or UPLOAD_BASE_PATH',
446
+ );
447
+ }
448
+
449
+ if (errors.length > 0) {
450
+ throw new Error(
451
+ '⚠️ Scan configuration errors:\n - ' + errors.join('\n - '),
452
+ );
453
+ }
454
+ }
455
+
456
+ /**
457
+ * Get scan configuration
458
+ * @returns {Object} Scan configuration
459
+ */
460
+ getScanConfig() {
461
+ // Return scan config with basePathFull already resolved to absolute path
462
+ return this.scan;
463
+ }
464
+
465
+ /**
466
+ * Get push configuration
467
+ * @returns {Object} Push configuration
468
+ */
469
+ getPushConfig() {
470
+ return {
471
+ ...this.push,
472
+ rfcs: this.push.rfcs || [],
473
+ years: this.push.years || [],
474
+ };
475
+ }
476
+
477
+ /**
478
+ * Load performance configuration
479
+ * @private
480
+ */
481
+ #loadPerformanceConfig() {
482
+ return {
483
+ batchDelay: parseInt(process.env.BATCH_DELAY) || 0, // Removed default delay
484
+ batchSize: parseInt(process.env.BATCH_SIZE) || 50, // Increased from 10 to 50
485
+ maxConcurrentSources: parseInt(process.env.MAX_CONCURRENT_SOURCES) || 2,
486
+ maxApiConnections: parseInt(process.env.MAX_API_CONNECTIONS) || 10, // New: API replica support
487
+ apiConnectionTimeout:
488
+ parseInt(process.env.API_CONNECTION_TIMEOUT) || 60000, // New: API timeout
489
+ progressUpdateInterval:
490
+ parseInt(process.env.PROGRESS_UPDATE_INTERVAL) || 10,
491
+ logBufferSize: 100,
492
+ logFlushInterval: 5000,
493
+ };
494
+ }
495
+
496
+ /**
497
+ * Load logging configuration
498
+ * @private
499
+ */
500
+ #loadLoggingConfig() {
501
+ return {
502
+ verbose: process.env.VERBOSE_LOGGING === 'true',
503
+ logFilePath: path.resolve(process.cwd(), 'arela-upload.log'),
504
+ };
505
+ }
506
+
507
+ /**
508
+ * Load watch configuration
509
+ * @private
510
+ */
511
+ #loadWatchConfig() {
512
+ // Parse directory configs from JSON format
513
+ let directoryConfigs = {};
514
+ if (process.env.WATCH_DIRECTORY_CONFIGS) {
515
+ try {
516
+ directoryConfigs = JSON.parse(process.env.WATCH_DIRECTORY_CONFIGS);
517
+ } catch (error) {
518
+ console.warn(
519
+ '⚠️ Invalid JSON in WATCH_DIRECTORY_CONFIGS, using empty config',
520
+ );
521
+ directoryConfigs = {};
522
+ }
523
+ }
524
+
525
+ // Fallback to WATCH_DIRECTORIES for backward compatibility
526
+ let directories = [];
527
+ if (
528
+ Object.keys(directoryConfigs).length === 0 &&
529
+ process.env.WATCH_DIRECTORIES
530
+ ) {
531
+ directories = process.env.WATCH_DIRECTORIES.split(',')
532
+ .map((d) => d.trim())
533
+ .filter(Boolean);
534
+ }
535
+
536
+ // Parse ignore patterns
537
+ const ignorePatterns =
538
+ process.env.WATCH_IGNORE_PATTERNS?.split(',')
539
+ .map((p) => p.trim())
540
+ .filter(Boolean) || [];
541
+
542
+ return {
543
+ enabled: process.env.WATCH_ENABLED === 'true' || false,
544
+ directories,
545
+ directoryConfigs,
546
+ strategy: process.env.WATCH_STRATEGY || 'batch',
547
+ debounceMs: parseInt(process.env.WATCH_DEBOUNCE_MS) || 1000,
548
+ batchSize: parseInt(process.env.WATCH_BATCH_SIZE) || 10,
549
+ usePolling: process.env.WATCH_USE_POLLING === 'true' || false,
550
+ pollInterval: parseInt(process.env.WATCH_POLL_INTERVAL) || 100,
551
+ stabilityThreshold:
552
+ parseInt(process.env.WATCH_STABILITY_THRESHOLD) || 300,
553
+ ignorePatterns,
554
+ autoDetect: process.env.WATCH_AUTO_DETECT === 'true' || false,
555
+ autoOrganize: process.env.WATCH_AUTO_ORGANIZE === 'true' || false,
556
+ };
557
+ }
558
+
559
+ /**
560
+ * Check if API mode is available
561
+ * @param {string} target - Optional API target to check
562
+ * @returns {boolean}
563
+ */
564
+ isApiModeAvailable(target = null) {
565
+ const config = this.getApiConfig(target);
566
+ return !!(config.baseUrl && config.token);
567
+ }
568
+
569
+ /**
570
+ * Check if Supabase mode is available
571
+ * @returns {boolean}
572
+ */
573
+ isSupabaseModeAvailable() {
574
+ return !!(this.supabase.url && this.supabase.key && this.supabase.bucket);
575
+ }
576
+
577
+ /**
578
+ * Load Redis configuration for BullMQ worker mode
579
+ * @private
580
+ */
581
+ #loadRedisConfig() {
582
+ return {
583
+ host: process.env.REDIS_HOST || 'localhost',
584
+ port: parseInt(process.env.REDIS_PORT) || 6379,
585
+ password: process.env.REDIS_PASSWORD || undefined,
586
+ db: parseInt(process.env.REDIS_DB) || 0,
587
+ maxRetriesPerRequest: null, // Required by BullMQ
588
+ };
589
+ }
590
+
591
+ /**
592
+ * Load worker configuration
593
+ * @private
594
+ */
595
+ #loadWorkerConfig() {
596
+ const queues = process.env.WORKER_QUEUES?.split(',')
597
+ .map((q) => q.trim())
598
+ .filter(Boolean) || [
599
+ 'pipeline-scan',
600
+ 'pipeline-identify',
601
+ 'pipeline-propagate',
602
+ 'pipeline-push',
603
+ ];
604
+
605
+ return {
606
+ queues,
607
+ concurrency: parseInt(process.env.WORKER_CONCURRENCY) || 1,
608
+ lockDuration: parseInt(process.env.WORKER_LOCK_DURATION) || 30000,
609
+ stalledInterval: parseInt(process.env.WORKER_STALLED_INTERVAL) || 30000,
610
+ // HTTP Polling mode configuration
611
+ poll: {
612
+ enabled: process.env.WORKER_POLL_MODE === 'true',
613
+ interval: parseInt(process.env.WORKER_POLL_INTERVAL) || 5000,
614
+ apiTarget: process.env.WORKER_POLL_API_TARGET || 'agencia',
615
+ },
616
+ };
617
+ }
618
+
619
+ /**
620
+ * Get Redis configuration
621
+ * @returns {Object} Redis connection options for ioredis/BullMQ
622
+ */
623
+ getRedisConfig() {
624
+ return this.redis;
625
+ }
626
+
627
+ /**
628
+ * Get worker configuration
629
+ * @returns {Object} Worker settings
630
+ */
631
+ getWorkerConfig() {
632
+ return this.worker;
633
+ }
634
+
635
+ /**
636
+ * Get the server ID for this instance
637
+ * Used for worker job assignment in poll mode
638
+ * @returns {string|null} Server ID or null if not configured
639
+ */
640
+ getServerId() {
641
+ return process.env.ARELA_SERVER_ID || null;
642
+ }
643
+
644
+ /**
645
+ * Load Google Drive sync configuration
646
+ * @private
647
+ */
648
+ #loadGDriveConfig() {
649
+ const rootFolderId = process.env.GDRIVE_ROOT_FOLDER_ID || null;
650
+
651
+ // Default mirror destination: <UPLOAD_BASE_PATH>/_gdrive_mirror
652
+ let localMirrorPath = process.env.GDRIVE_LOCAL_MIRROR_PATH || null;
653
+ if (!localMirrorPath && process.env.UPLOAD_BASE_PATH) {
654
+ const base = process.env.UPLOAD_BASE_PATH;
655
+ if (base !== '*') {
656
+ localMirrorPath = path.resolve(
657
+ PathNormalizer.toAbsolutePath(base),
658
+ '_gdrive_mirror',
659
+ );
660
+ }
661
+ } else if (localMirrorPath) {
662
+ localMirrorPath = PathNormalizer.toAbsolutePath(localMirrorPath);
663
+ }
664
+
665
+ return {
666
+ rootFolderId,
667
+ localMirrorPath,
668
+ serviceAccountFile: process.env.GDRIVE_SERVICE_ACCOUNT_FILE || null,
669
+ serviceAccountJson: process.env.GDRIVE_SERVICE_ACCOUNT_JSON || null,
670
+ skipNativeDocs: process.env.GDRIVE_SKIP_NATIVE_DOCS !== 'false',
671
+ followShortcuts: process.env.GDRIVE_FOLLOW_SHORTCUTS !== 'false',
672
+ concurrency: parseInt(process.env.GDRIVE_CONCURRENCY) || 5,
673
+ pageSize: parseInt(process.env.GDRIVE_PAGE_SIZE) || 1000,
674
+ maxFileSizeBytes:
675
+ parseInt(process.env.GDRIVE_MAX_FILE_SIZE_BYTES) ||
676
+ 2 * 1024 * 1024 * 1024, // 2GB default
677
+ };
678
+ }
679
+
680
+ /**
681
+ * Get Google Drive configuration
682
+ * @returns {Object} GDrive sync settings
683
+ */
684
+ getGDriveConfig() {
685
+ return this.gdrive;
686
+ }
687
+
688
+ /**
689
+ * Validate Google Drive configuration
690
+ * @throws {Error} If required gdrive configuration is missing
691
+ */
692
+ validateGDriveConfig() {
693
+ const errors = [];
694
+
695
+ if (!this.gdrive.rootFolderId) {
696
+ errors.push(
697
+ 'GDRIVE_ROOT_FOLDER_ID is required (Drive folder ID to sync)',
698
+ );
699
+ }
700
+
701
+ if (!this.gdrive.localMirrorPath) {
702
+ errors.push(
703
+ 'Could not determine local mirror path. Set GDRIVE_LOCAL_MIRROR_PATH or UPLOAD_BASE_PATH',
704
+ );
705
+ }
706
+
707
+ if (!this.gdrive.serviceAccountFile && !this.gdrive.serviceAccountJson) {
708
+ errors.push(
709
+ 'Either GDRIVE_SERVICE_ACCOUNT_FILE (path to JSON) or GDRIVE_SERVICE_ACCOUNT_JSON (inline JSON) is required',
710
+ );
711
+ }
712
+
713
+ if (this.gdrive.serviceAccountFile) {
714
+ const resolved = PathNormalizer.toAbsolutePath(
715
+ this.gdrive.serviceAccountFile,
716
+ );
717
+ if (!fs.existsSync(resolved)) {
718
+ errors.push(`GDRIVE_SERVICE_ACCOUNT_FILE not found: ${resolved}`);
719
+ }
720
+ }
721
+
722
+ if (errors.length > 0) {
723
+ throw new Error(
724
+ '⚠️ Google Drive configuration errors:\n - ' + errors.join('\n - '),
725
+ );
726
+ }
727
+ }
728
+
729
+ /**
730
+ * Check if worker mode is available (Redis configured)
731
+ * @returns {boolean}
732
+ */
733
+ isWorkerModeAvailable() {
734
+ return !!(this.redis.host && this.redis.port);
735
+ }
736
+
737
+ /**
738
+ * Validate configuration for the requested mode
739
+ * @param {boolean} forceSupabase - Whether to force Supabase mode
740
+ * @throws {Error} If required configuration is missing
741
+ */
742
+ validateConfiguration(forceSupabase = false) {
743
+ if (forceSupabase) {
744
+ if (!this.isSupabaseModeAvailable()) {
745
+ throw new Error(
746
+ '⚠️ Missing Supabase credentials. Please set SUPABASE_URL, SUPABASE_KEY, and SUPABASE_BUCKET',
747
+ );
748
+ }
749
+ return;
750
+ }
751
+
752
+ if (!this.isApiModeAvailable() && !this.isSupabaseModeAvailable()) {
753
+ throw new Error(
754
+ '⚠️ Missing credentials. Please set either:\n' +
755
+ ' - ARELA_API_URL and ARELA_API_TOKEN for API mode, or\n' +
756
+ ' - SUPABASE_URL, SUPABASE_KEY, and SUPABASE_BUCKET for direct mode',
757
+ );
758
+ }
759
+ }
760
+
761
+ /**
762
+ * Get upload sources with validation
763
+ * @returns {string[]} Array of upload sources (defaults to ['.'] if not configured)
764
+ */
765
+ getUploadSources() {
766
+ if (!this.upload.sources || this.upload.sources.length === 0) {
767
+ return ['.'];
768
+ }
769
+ return this.upload.sources;
770
+ }
771
+
772
+ /**
773
+ * Get base path with validation
774
+ * Returns the path as configured (may be relative for legacy compatibility)
775
+ * @returns {string} Base path for uploads
776
+ * @throws {Error} If base path is not configured
777
+ */
778
+ getBasePath() {
779
+ if (!this.upload.basePath) {
780
+ throw new Error(
781
+ '⚠️ No base path configured. Please set UPLOAD_BASE_PATH environment variable.',
782
+ );
783
+ }
784
+ return this.upload.basePath;
785
+ }
786
+
787
+ /**
788
+ * Get watch directories with validation
789
+ * @param {string[]} cliDirs - Directories from CLI options
790
+ * @returns {string[]} Validated watch directories
791
+ * @throws {Error} If no directories are configured
792
+ */
793
+ getWatchDirectories(cliDirs = null) {
794
+ // CLI options take precedence over environment variables
795
+ const directories =
796
+ cliDirs && cliDirs.length > 0 ? cliDirs : this.watch.directories;
797
+
798
+ if (!directories || directories.length === 0) {
799
+ throw new Error(
800
+ '⚠️ No watch directories configured. Please use --directories option or set WATCH_DIRECTORIES environment variable.',
801
+ );
802
+ }
803
+
804
+ return directories;
805
+ }
806
+
807
+ /**
808
+ * Get watch configuration with validation
809
+ * @returns {Object} Watch configuration object
810
+ */
811
+ getWatchConfig() {
812
+ return {
813
+ ...this.watch,
814
+ };
815
+ }
816
+
817
+ /**
818
+ * Reload upload, scan, push and performance config from current process.env
819
+ * values. Must be called after modifying env vars at runtime (e.g.,
820
+ * PipelineJobRunner applying a job's scanConfig/tuning overrides).
821
+ */
822
+ reloadScanConfig() {
823
+ this.upload = this.#loadUploadConfig();
824
+ this.scan = this.#loadScanConfig();
825
+ this.push = this.#loadPushConfig();
826
+ this.performance = this.#loadPerformanceConfig();
827
+ }
828
+
829
+ /**
830
+ * Validate watch configuration
831
+ * @param {string[]} directories - Directories to validate
832
+ * @throws {Error} If configuration is invalid
833
+ */
834
+ validateWatchConfig(directories) {
835
+ if (!directories || directories.length === 0) {
836
+ throw new Error(
837
+ 'At least one directory must be specified for watch mode',
838
+ );
839
+ }
840
+
841
+ // Validate strategy
842
+ const validStrategies = ['individual', 'batch', 'full-structure'];
843
+ if (!validStrategies.includes(this.watch.strategy)) {
844
+ throw new Error(
845
+ `Invalid watch strategy: ${this.watch.strategy}. Must be one of: ${validStrategies.join(', ')}`,
846
+ );
847
+ }
848
+
849
+ // Validate debounce and batch size
850
+ if (this.watch.debounceMs < 0) {
851
+ throw new Error('Debounce time must be >= 0');
852
+ }
853
+
854
+ if (this.watch.batchSize < 1) {
855
+ throw new Error('Batch size must be >= 1');
856
+ }
857
+ }
858
+ }
859
+
860
+ // Export singleton instance
861
+ export const appConfig = new Config();
862
+ export default appConfig;