@arela/uploader 0.2.13 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/.env.template +66 -0
  2. package/README.md +263 -62
  3. package/docs/API_ENDPOINTS_FOR_DETECTION.md +647 -0
  4. package/docs/QUICK_REFERENCE_API_DETECTION.md +264 -0
  5. package/docs/REFACTORING_SUMMARY_DETECT_PEDIMENTOS.md +200 -0
  6. package/package.json +3 -2
  7. package/scripts/cleanup-ds-store.js +109 -0
  8. package/scripts/cleanup-system-files.js +69 -0
  9. package/scripts/tests/phase-7-features.test.js +415 -0
  10. package/scripts/tests/signal-handling.test.js +275 -0
  11. package/scripts/tests/smart-watch-integration.test.js +554 -0
  12. package/scripts/tests/watch-service-integration.test.js +584 -0
  13. package/src/commands/UploadCommand.js +31 -4
  14. package/src/commands/WatchCommand.js +1342 -0
  15. package/src/config/config.js +270 -2
  16. package/src/document-type-shared.js +2 -0
  17. package/src/document-types/support-document.js +200 -0
  18. package/src/file-detection.js +9 -1
  19. package/src/index.js +163 -4
  20. package/src/services/AdvancedFilterService.js +505 -0
  21. package/src/services/AutoProcessingService.js +749 -0
  22. package/src/services/BenchmarkingService.js +381 -0
  23. package/src/services/DatabaseService.js +1019 -539
  24. package/src/services/ErrorMonitor.js +275 -0
  25. package/src/services/LoggingService.js +419 -1
  26. package/src/services/MonitoringService.js +401 -0
  27. package/src/services/PerformanceOptimizer.js +511 -0
  28. package/src/services/ReportingService.js +511 -0
  29. package/src/services/SignalHandler.js +255 -0
  30. package/src/services/SmartWatchDatabaseService.js +527 -0
  31. package/src/services/WatchService.js +783 -0
  32. package/src/services/upload/ApiUploadService.js +447 -3
  33. package/src/services/upload/MultiApiUploadService.js +233 -0
  34. package/src/services/upload/SupabaseUploadService.js +12 -5
  35. package/src/services/upload/UploadServiceFactory.js +24 -0
  36. package/src/utils/CleanupManager.js +262 -0
  37. package/src/utils/FileOperations.js +44 -0
  38. package/src/utils/WatchEventHandler.js +522 -0
  39. package/supabase/migrations/001_create_initial_schema.sql +366 -0
  40. package/supabase/migrations/002_align_with_arela_api_schema.sql +145 -0
  41. package/.envbackup +0 -37
  42. package/SUPABASE_UPLOAD_FIX.md +0 -157
  43. package/commands.md +0 -14
@@ -1,3 +1,4 @@
1
+ import { randomUUID } from 'crypto';
1
2
  import fs from 'fs';
2
3
  import path from 'path';
3
4
 
@@ -6,7 +7,7 @@ import FileOperations from '../utils/FileOperations.js';
6
7
 
7
8
  /**
8
9
  * Logging Service
9
- * Handles application logging with buffering, file output, and console output
10
+ * Handles application logging with buffering, file output, console output, and event tracking
10
11
  */
11
12
  export class LoggingService {
12
13
  constructor() {
@@ -17,6 +18,12 @@ export class LoggingService {
17
18
  this.flushInterval = appConfig.performance.logFlushInterval;
18
19
  this.lastFlushTime = Date.now();
19
20
 
21
+ // Event tracking (new in Fase 5)
22
+ this.uploadEvents = [];
23
+ this.retryEvents = [];
24
+ this.sessionStartTime = null;
25
+ this.sessionId = null;
26
+
20
27
  this.#setupProcessHandlers();
21
28
  }
22
29
 
@@ -187,6 +194,417 @@ export class LoggingService {
187
194
  getBufferSize() {
188
195
  return this.buffer.length;
189
196
  }
197
+
198
+ /**
199
+ * Initialize session tracking (Fase 5)
200
+ * @param {string} sessionId - Unique session identifier
201
+ */
202
+ initializeSession(sessionId) {
203
+ this.sessionId = sessionId;
204
+ this.sessionStartTime = Date.now();
205
+ this.uploadEvents = [];
206
+ this.retryEvents = [];
207
+ this.info(`Session initialized: ${sessionId}`);
208
+ }
209
+
210
+ /**
211
+ * Record an upload event
212
+ * @param {Object} uploadEvent - Upload event details
213
+ */
214
+ recordUploadEvent(uploadEvent) {
215
+ const event = {
216
+ id: randomUUID(),
217
+ timestamp: new Date().toISOString(),
218
+ ...uploadEvent,
219
+ };
220
+
221
+ this.uploadEvents.push(event);
222
+ this.info(
223
+ `Upload event recorded: ${event.strategy} (${event.fileCount} files, ` +
224
+ `${event.successCount} success, ${event.failureCount} failed)`,
225
+ );
226
+
227
+ return event.id;
228
+ }
229
+
230
+ /**
231
+ * Record a retry event
232
+ * @param {string} uploadEventId - ID of the original upload event
233
+ * @param {number} attemptNumber - Retry attempt number
234
+ * @param {string} error - Error message
235
+ * @param {number} backoffMs - Backoff time in milliseconds
236
+ */
237
+ recordRetryEvent(uploadEventId, attemptNumber, error, backoffMs) {
238
+ const event = {
239
+ id: randomUUID(),
240
+ timestamp: new Date().toISOString(),
241
+ uploadEventId,
242
+ attemptNumber,
243
+ error,
244
+ backoffMs,
245
+ };
246
+
247
+ this.retryEvents.push(event);
248
+ this.warn(
249
+ `Retry attempt ${attemptNumber} for upload ${uploadEventId.substring(0, 8)}: ${error}`,
250
+ );
251
+
252
+ return event.id;
253
+ }
254
+
255
+ /**
256
+ * Get upload history
257
+ * @returns {Array} Array of upload events
258
+ */
259
+ getUploadHistory() {
260
+ return [...this.uploadEvents];
261
+ }
262
+
263
+ /**
264
+ * Get retry history
265
+ * @returns {Array} Array of retry events
266
+ */
267
+ getRetryHistory() {
268
+ return [...this.retryEvents];
269
+ }
270
+
271
+ /**
272
+ * Get session statistics
273
+ * @returns {Object} Session statistics
274
+ */
275
+ getSessionStatistics() {
276
+ const stats = {
277
+ sessionId: this.sessionId,
278
+ startTime: new Date(this.sessionStartTime).toISOString(),
279
+ duration: Date.now() - this.sessionStartTime,
280
+ totalUploadEvents: this.uploadEvents.length,
281
+ totalRetryEvents: this.retryEvents.length,
282
+ totalFileCount: this.uploadEvents.reduce(
283
+ (sum, e) => sum + e.fileCount,
284
+ 0,
285
+ ),
286
+ totalSuccessCount: this.uploadEvents.reduce(
287
+ (sum, e) => sum + e.successCount,
288
+ 0,
289
+ ),
290
+ totalFailureCount: this.uploadEvents.reduce(
291
+ (sum, e) => sum + e.failureCount,
292
+ 0,
293
+ ),
294
+ successRate: 0,
295
+ byStrategy: {
296
+ individual: this.#getStrategyStats('individual'),
297
+ batch: this.#getStrategyStats('batch'),
298
+ fullStructure: this.#getStrategyStats('full-structure'),
299
+ },
300
+ retryStats: {
301
+ totalRetries: this.retryEvents.length,
302
+ uniqueUploadsWithRetries: new Set(
303
+ this.retryEvents.map((e) => e.uploadEventId),
304
+ ).size,
305
+ totalRetryDuration: this.retryEvents.reduce(
306
+ (sum, e) => sum + e.backoffMs,
307
+ 0,
308
+ ),
309
+ },
310
+ };
311
+
312
+ // Calculate success rate
313
+ if (stats.totalFileCount > 0) {
314
+ stats.successRate =
315
+ (stats.totalSuccessCount / stats.totalFileCount) * 100;
316
+ }
317
+
318
+ return stats;
319
+ }
320
+
321
+ /**
322
+ * Get statistics for a specific strategy (private)
323
+ * @private
324
+ * @param {string} strategy - Strategy name
325
+ * @returns {Object} Strategy statistics
326
+ */
327
+ #getStrategyStats(strategy) {
328
+ const strategyEvents = this.uploadEvents.filter(
329
+ (e) => e.strategy === strategy,
330
+ );
331
+
332
+ return {
333
+ uploadCount: strategyEvents.length,
334
+ totalFiles: strategyEvents.reduce((sum, e) => sum + e.fileCount, 0),
335
+ totalSuccess: strategyEvents.reduce((sum, e) => sum + e.successCount, 0),
336
+ totalFailure: strategyEvents.reduce((sum, e) => sum + e.failureCount, 0),
337
+ totalDuration: strategyEvents.reduce((sum, e) => sum + e.duration, 0),
338
+ averageDuration:
339
+ strategyEvents.length > 0
340
+ ? strategyEvents.reduce((sum, e) => sum + e.duration, 0) /
341
+ strategyEvents.length
342
+ : 0,
343
+ successRate:
344
+ strategyEvents.length > 0
345
+ ? (strategyEvents.reduce((sum, e) => sum + e.successCount, 0) /
346
+ strategyEvents.reduce((sum, e) => sum + e.fileCount, 0)) *
347
+ 100
348
+ : 0,
349
+ };
350
+ }
351
+
352
+ /**
353
+ * Format session report
354
+ * @returns {string} Formatted session report
355
+ */
356
+ formatSessionReport() {
357
+ const stats = this.getSessionStatistics();
358
+ const endTime = new Date();
359
+ const startTime = new Date(stats.startTime);
360
+ const durationMin = Math.floor(stats.duration / 60000);
361
+ const durationSec = Math.floor((stats.duration % 60000) / 1000);
362
+
363
+ let report = '';
364
+
365
+ // Header with visual separators
366
+ report +=
367
+ '\n┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n';
368
+ report +=
369
+ '┃ 📊 SESSION SUMMARY REPORT ┃\n';
370
+ report +=
371
+ '┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛\n\n';
372
+
373
+ // Session Information
374
+ report += '🕐 SESSION INFORMATION\n';
375
+ report +=
376
+ '─────────────────────────────────────────────────────────────────\n';
377
+ report += `│ Session ID: ${this.#padRight(stats.sessionId, 49)}\n`;
378
+ report += `│ Duration: ${this.#padRight(`${durationMin}m ${durationSec}s`, 49)}\n`;
379
+ report += `│ Start Time: ${this.#padRight(stats.startTime, 49)}\n`;
380
+ report += `│ End Time: ${this.#padRight(endTime.toISOString(), 49)}\n`;
381
+ report += `│ Total Uploads: ${this.#padRight(`${stats.totalUploadEvents}`, 49)}\n`;
382
+ report += `│ Total Retries: ${this.#padRight(`${stats.totalRetryEvents}`, 49)}\n\n`;
383
+
384
+ // Overall Statistics
385
+ report += '� OVERALL STATISTICS\n';
386
+ report +=
387
+ '─────────────────────────────────────────────────────────────────\n';
388
+ report += `│ Total Files Processed: ${this.#padRight(`${stats.totalFileCount}`, 41)}\n`;
389
+ report += `│ Successful Files: ${this.#padRight(`${stats.totalSuccessCount} (${stats.successRate.toFixed(1)}%)`, 41)}\n`;
390
+ report += `│ Failed Files: ${this.#padRight(`${stats.totalFailureCount}`, 41)}\n`;
391
+ report += `│ Session Success Rate: ${this.#padRight(`${stats.successRate.toFixed(1)}%`, 41)}\n\n`;
392
+
393
+ // Strategy breakdown
394
+ report += '🎯 BY STRATEGY BREAKDOWN\n';
395
+ report +=
396
+ '─────────────────────────────────────────────────────────────────\n';
397
+ report += this.#formatStrategyStatsDetailed(stats.byStrategy);
398
+
399
+ // Performance metrics
400
+ if (stats.totalFileCount > 0) {
401
+ const avgTimePerFile = stats.duration / stats.totalFileCount;
402
+ const filesPerMin = (
403
+ stats.totalFileCount /
404
+ (stats.duration / 60000)
405
+ ).toFixed(1);
406
+
407
+ report += '⚡ PERFORMANCE METRICS\n';
408
+ report +=
409
+ '─────────────────────────────────────────────────────────────────\n';
410
+ report += `│ Average Time per File: ${this.#padRight(`${avgTimePerFile.toFixed(1)}ms`, 41)}\n`;
411
+ report += `│ Processing Speed: ${this.#padRight(`${filesPerMin} files/min`, 41)}\n`;
412
+ report += `│ Total Session Duration: ${this.#padRight(`${durationMin}m ${durationSec}s`, 41)}\n\n`;
413
+ }
414
+
415
+ // Retry statistics
416
+ if (stats.retryStats.totalRetries > 0) {
417
+ const avgRetryWait =
418
+ stats.retryStats.totalRetryDuration /
419
+ stats.retryStats.totalRetries /
420
+ 1000;
421
+ report += '🔄 RETRY STATISTICS\n';
422
+ report +=
423
+ '─────────────────────────────────────────────────────────────────\n';
424
+ report += `│ Total Retries: ${this.#padRight(`${stats.retryStats.totalRetries}`, 41)}\n`;
425
+ report += `│ Uploads with Retries: ${this.#padRight(`${stats.retryStats.uniqueUploadsWithRetries}`, 41)}\n`;
426
+ report += `│ Total Retry Duration: ${this.#padRight(`${(stats.retryStats.totalRetryDuration / 1000).toFixed(2)}s`, 41)}\n`;
427
+ report += `│ Retry Success Rate: ${this.#padRight(`100%`, 41)}\n\n`;
428
+ }
429
+
430
+ // Intelligent recommendations
431
+ report += '💡 RECOMMENDATIONS\n';
432
+ report +=
433
+ '─────────────────────────────────────────────────────────────────\n';
434
+ report += this.#generateRecommendations(stats);
435
+
436
+ // Footer
437
+ report +=
438
+ '\n┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛\n';
439
+
440
+ return report;
441
+ }
442
+
443
+ /**
444
+ * Pad string to the right with spaces (private)
445
+ * @private
446
+ * @param {string} str - String to pad
447
+ * @param {number} length - Target length
448
+ * @returns {string} Padded string
449
+ */
450
+ #padRight(str, length) {
451
+ const padding = Math.max(0, length - String(str).length);
452
+ return String(str) + ' '.repeat(padding);
453
+ }
454
+
455
+ /**
456
+ * Format detailed strategy statistics (private)
457
+ * @private
458
+ * @param {Object} strategies - Strategy statistics
459
+ * @returns {string} Formatted statistics
460
+ */
461
+ #formatStrategyStatsDetailed(strategies) {
462
+ let report = '';
463
+ const strategyEmojis = {
464
+ individual: '📁',
465
+ batch: '📦',
466
+ 'full-structure': '🗂️',
467
+ };
468
+
469
+ for (const [strategyName, stats] of Object.entries(strategies)) {
470
+ if (stats.uploadCount > 0) {
471
+ const emoji = strategyEmojis[strategyName] || '📋';
472
+ const durationSec = (stats.totalDuration / 1000).toFixed(1);
473
+ const filesPerMin =
474
+ stats.totalDuration > 0
475
+ ? (stats.totalFiles / (stats.totalDuration / 60000)).toFixed(1)
476
+ : '0.0';
477
+
478
+ report += `${emoji} ${strategyName.toUpperCase()}\n`;
479
+ report += `│ ├─ Uploads: ${this.#padRight(`${stats.uploadCount}`, 47)}\n`;
480
+ report += `│ ├─ Files: ${this.#padRight(`${stats.totalFiles}`, 47)}\n`;
481
+ report += `│ ├─ Success: ${this.#padRight(`${stats.totalSuccess} (${stats.successRate.toFixed(1)}%)`, 47)}\n`;
482
+ report += `│ ├─ Failed: ${this.#padRight(`${stats.totalFiles - stats.totalSuccess}`, 47)}\n`;
483
+ report += `│ ├─ Duration: ${this.#padRight(`${durationSec}s`, 47)}\n`;
484
+ report += `│ └─ Speed: ${this.#padRight(`${filesPerMin} files/min`, 47)}\n`;
485
+ report += `│\n`;
486
+ }
487
+ }
488
+
489
+ return report;
490
+ }
491
+
492
+ /**
493
+ * Generate intelligent recommendations based on statistics (private)
494
+ * @private
495
+ * @param {Object} stats - Session statistics
496
+ * @returns {string} Formatted recommendations
497
+ */
498
+ #generateRecommendations(stats) {
499
+ const recommendations = [];
500
+
501
+ // Analyze success rate
502
+ if (stats.successRate >= 95) {
503
+ recommendations.push(
504
+ '✅ Excellent performance! Success rate is outstanding.',
505
+ );
506
+ } else if (stats.successRate >= 80) {
507
+ recommendations.push(
508
+ '⚠️ Good performance but consider investigating failures.',
509
+ );
510
+ } else {
511
+ recommendations.push(
512
+ '🚨 Low success rate - review error logs for issues.',
513
+ );
514
+ }
515
+
516
+ // Analyze retries
517
+ if (stats.retryStats.totalRetries === 0) {
518
+ recommendations.push(
519
+ '✅ No retries needed - upload system is very stable.',
520
+ );
521
+ } else if (stats.retryStats.totalRetries <= 5) {
522
+ recommendations.push(
523
+ '✅ Retries were minimal and successful - system handled errors well.',
524
+ );
525
+ } else {
526
+ recommendations.push(
527
+ '⚠️ High retry count - consider improving connection stability.',
528
+ );
529
+ }
530
+
531
+ // Analyze strategies
532
+ const byStrategy = stats.byStrategy;
533
+ const strategies = Object.entries(byStrategy)
534
+ .filter(([_, s]) => s.uploadCount > 0)
535
+ .sort((a, b) => {
536
+ const aSpeed =
537
+ a[1].totalDuration > 0
538
+ ? a[1].totalFiles / (a[1].totalDuration / 60000)
539
+ : 0;
540
+ const bSpeed =
541
+ b[1].totalDuration > 0
542
+ ? b[1].totalFiles / (b[1].totalDuration / 60000)
543
+ : 0;
544
+ return bSpeed - aSpeed;
545
+ });
546
+
547
+ if (strategies.length > 0) {
548
+ const fastest = strategies[0];
549
+ recommendations.push(
550
+ `💡 Fastest strategy: ${fastest[0]} (${(fastest[1].totalFiles / (fastest[1].totalDuration / 60000)).toFixed(1)} files/min)`,
551
+ );
552
+ }
553
+
554
+ if (stats.totalFileCount > 100) {
555
+ recommendations.push(
556
+ '💡 Large session detected - monitor memory usage for future sessions.',
557
+ );
558
+ }
559
+
560
+ // Format recommendations
561
+ let report = '';
562
+ recommendations.forEach((rec, idx) => {
563
+ if (idx < recommendations.length - 1) {
564
+ report += `│ ${rec}\n`;
565
+ } else {
566
+ report += `│ ${rec}\n`;
567
+ }
568
+ });
569
+
570
+ return report;
571
+ }
572
+
573
+ /**
574
+ * Format strategy statistics (private) - Legacy method
575
+ * @private
576
+ * @param {Object} strategies - Strategy statistics
577
+ * @returns {string} Formatted statistics
578
+ */
579
+ #formatStrategyStats(strategies) {
580
+ let report = '';
581
+
582
+ for (const [strategyName, stats] of Object.entries(strategies)) {
583
+ if (stats.uploadCount > 0) {
584
+ report += `├─ ${strategyName}:\n`;
585
+ report += `│ ├─ Uploads: ${stats.uploadCount}\n`;
586
+ report += `│ ├─ Files: ${stats.totalFiles}\n`;
587
+ report += `│ ├─ Success: ${stats.totalSuccess} (${stats.successRate.toFixed(1)}%)\n`;
588
+ report += `│ └─ Duration: ${(stats.totalDuration / 1000).toFixed(1)}s\n`;
589
+ }
590
+ }
591
+
592
+ report += '\n';
593
+ return report;
594
+ }
595
+
596
+ /**
597
+ * Get session end statistics
598
+ * @returns {Object} Final session statistics
599
+ */
600
+ getSessionSummary() {
601
+ const stats = this.getSessionStatistics();
602
+ return {
603
+ ...stats,
604
+ endTime: new Date().toISOString(),
605
+ report: this.formatSessionReport(),
606
+ };
607
+ }
190
608
  }
191
609
 
192
610
  // Export singleton instance