@arela/uploader 1.0.2 → 1.0.3

package/src/commands/IdentifyCommand.js

@@ -0,0 +1,486 @@
+import cliProgress from 'cli-progress';
+import fs from 'fs';
+import pLimit from 'p-limit';
+
+import logger from '../services/LoggingService.js';
+
+import appConfig from '../config/config.js';
+import ErrorHandler from '../errors/ErrorHandler.js';
+import { ConfigurationError } from '../errors/ErrorTypes.js';
+import FileDetectionService from '../file-detection.js';
+
+/**
+ * Identify Command Handler
+ * Optimized replacement for "detect --detect-pdfs"
+ * Identifies pedimento-simplificado documents in files scanned by "arela scan"
+ *
+ * Key improvements over legacy detect command:
+ * - Works with dynamic scan_* tables instead of uploader table
+ * - Uses configured API instead of direct Supabase access
+ * - Detects locally to leverage CLI host resources
+ * - Batch processing for efficient API communication
+ * - Real-time progress tracking
+ */
+export class IdentifyCommand {
+  constructor() {
+    this.errorHandler = new ErrorHandler(logger);
+    this.scanApiService = null; // Will be initialized in execute
+    this.detectionService = new FileDetectionService();
+  }
+
+  /**
+   * Execute the identify command
+   * @param {Object} options - Command options
+   * @param {string} options.api - API target (default, agencia, cliente)
+   * @param {number} options.batchSize - Batch size for API operations
+   * @param {boolean} options.showStats - Show performance statistics
+   */
+  async execute(options = {}) {
+    const startTime = Date.now();
+
+    try {
+      // Validate scan configuration (need same config as scan command)
+      appConfig.validateScanConfig();
+
+      // Import ScanApiService dynamically
+      const { default: ScanApiService } = await import(
+        '../services/ScanApiService.js'
+      );
+      this.scanApiService = new ScanApiService();
+
+      // Set API target if specified
+      if (options.api) {
+        appConfig.setApiTarget(options.api);
+        this.scanApiService = new ScanApiService();
+      }
+
+      const scanConfig = appConfig.getScanConfig();
+      const batchSize = parseInt(options.batchSize) || 100;
+
+      logger.info('🔍 Starting arela identify command');
+      logger.info(`🎯 API Target: ${options.api || 'default'}`);
+      logger.info(`📦 Batch Size: ${batchSize}`);
+
+      // Fetch all tables for this instance
+      logger.info('\n📊 Fetching instance tables...');
+      const tables = await this.scanApiService.getInstanceTables(
+        scanConfig.companySlug,
+        scanConfig.serverId,
+        scanConfig.basePathLabel,
+      );
+
+      if (tables.length === 0) {
+        throw new ConfigurationError(
+          'No tables found for this instance. Run "arela scan" first.',
+        );
+      }
+
+      logger.info(
+        `📋 Found ${tables.length} table${tables.length === 1 ? '' : 's'} to process`,
+      );
+      for (const table of tables) {
+        logger.info(`   - ${table.tableName}`);
+      }
+
+      // Process each table
+      let totalStats = {
+        processed: 0,
+        detected: 0,
+        errors: 0,
+        pending: 0,
+      };
+
+      for (const table of tables) {
+        logger.info(`\n🔍 Processing table: ${table.tableName}`);
+
+        // Get detection statistics for this table
+        const stats = await this.#processTable(
+          table.tableName,
+          batchSize,
+          startTime,
+        );
+
+        totalStats.processed += stats.processed;
+        totalStats.detected += stats.detected;
+        totalStats.errors += stats.errors;
+      }
+
+      const duration = ((Date.now() - startTime) / 1000).toFixed(2);
+      const avgSpeed =
+        duration > 0 ? Math.round(totalStats.processed / duration) : 0;
+
+      logger.success(`\n✅ Identification Complete!`);
+      logger.info(`\n📊 Total Results:`);
+      logger.info(`   Tables Processed: ${tables.length}`);
+      logger.info(`   Files Processed: ${totalStats.processed}`);
+      logger.info(`   Pedimentos Detected: ${totalStats.detected}`);
+      logger.info(`   Errors: ${totalStats.errors}`);
+      logger.info(`   Duration: ${duration}s`);
+      logger.info(`   Speed: ${avgSpeed} files/sec`);
+
+      if (options.showStats) {
+        this.#showDetailedStats(
+          startTime,
+          totalStats.processed,
+          totalStats.detected,
+          totalStats.errors,
+        );
+      }
+    } catch (error) {
+      logger.error('❌ Identification failed:', error.message);
+
+      if (error instanceof ConfigurationError) {
+        logger.error('\nConfiguration errors:');
+        error.errors.forEach((err) => logger.error(`  - ${err}`));
+      }
+
+      throw error;
+    }
+  }
+
+  /**
+   * Process a single table
+   * @private
+   * @param {string} tableName - Table name to process
+   * @param {number} batchSize - Batch size
+   * @param {number} startTime - Start time for speed calculation
+   * @returns {Promise<Object>} Processing statistics
+   */
+  async #processTable(tableName, batchSize, startTime) {
+    // Get detection statistics first
+    const initialStats = await this.scanApiService.getDetectionStats(tableName);
+    logger.info(`   Total PDFs: ${initialStats.totalPdfs}`);
+    logger.info(`   Detected: ${initialStats.detected}`);
+    logger.info(`   Pending: ${initialStats.pending}`);
+    logger.info(`   Not Pedimento: ${initialStats.notPedimento || 0}`);
+    logger.info(
+      `   Max Attempts Reached: ${initialStats.maxAttemptsReached || 0}`,
+    );
+    logger.info(`   Errors: ${initialStats.errors}`);
+
+    if (initialStats.pending === 0) {
+      logger.info('   ✅ All PDFs processed. Skipping.');
+
+      if (initialStats.maxAttemptsReached > 0) {
+        logger.info(
+          `   ⚠️  ${initialStats.maxAttemptsReached} PDFs reached max attempts.`,
+        );
+      }
+
+      return {
+        processed: 0,
+        detected: 0,
+        errors: 0,
+      };
+    }
+
+    logger.info(`   🚀 Processing ${initialStats.pending} pending PDFs...`);
+
+    logger.info(`   🚀 Processing ${initialStats.pending} pending PDFs...`);
+
+    // Setup progress bar
+    const progressBar = new cliProgress.SingleBar({
+      format:
+        '   📄 |{bar}| {percentage}% | {value}/{total} files | {speed} files/sec',
+      barCompleteChar: '\u2588',
+      barIncompleteChar: '\u2591',
+      hideCursor: true,
+    });
+
+    progressBar.start(initialStats.pending, 0, { speed: 0 });
+
+    let processedCount = 0;
+    let detectedCount = 0;
+    let errorCount = 0;
+    let hasMore = true;
+    let batchNumber = 0;
+
+    // Process in batches
+    while (hasMore) {
+      batchNumber++;
+
+      // Fetch from offset 0 since processed files are filtered out
+      const response = await this.scanApiService.fetchPdfsForDetection(
+        tableName,
+        0,
+        batchSize,
+      );
+
+      if (!response.data || response.data.length === 0) {
+        break;
+      }
+
+      const files = response.data;
+
+      // Detect files locally with concurrent processing
+      const detectionResults = await this.#detectFilesLocally(files, 10);
+
+      // Batch update to API
+      const updateResult = await this.scanApiService.batchUpdateDetection(
+        tableName,
+        detectionResults,
+      );
+
+      // Update statistics
+      processedCount += files.length;
+      detectedCount += detectionResults.filter(
+        (r) => r.detectedType === 'pedimento_simplificado',
+      ).length;
+      errorCount += detectionResults.filter((r) => r.detectionError).length;
+
+      // Update progress bar
+      const elapsed = (Date.now() - startTime) / 1000;
+      const speed = elapsed > 0 ? Math.round(processedCount / elapsed) : 0;
+      progressBar.update(processedCount, { speed });
+
+      // Check if there are more files
+      hasMore = response.hasMore;
+    }
+
+    progressBar.stop();
+
+    return {
+      processed: processedCount,
+      detected: detectedCount,
+      errors: errorCount,
+    };
+  }
+
+  /**
+   * Detect files locally using FileDetectionService
+   * @private
+   * @param {Array} files - Files to detect
+   * @param {number} concurrency - Maximum concurrent detections
+   * @returns {Promise<Array>} Detection results
+   */
+  async #detectFilesLocally(files, concurrency = 10) {
+    const limit = pLimit(concurrency);
+    const basePath = appConfig.getBasePath();
+
+    const detectionPromises = files.map((file) =>
+      limit(async () => {
+        try {
+          // Check if file exists on filesystem
+          const absolutePath = file.absolute_path;
+
+          if (!fs.existsSync(absolutePath)) {
+            return {
+              id: file.id,
+              detectedType: null,
+              detectedPedimento: null,
+              detectedPedimentoYear: null,
+              rfc: null,
+              arelaPath: null,
+              detectionError:
+                'FILE_NOT_FOUND: File does not exist on filesystem. May have been moved or deleted after scan.',
+              isNotPedimento: false,
+            };
+          }
+
+          // Check file size - skip very large files
+          const stats = fs.statSync(absolutePath);
+          const maxSizeBytes = 50 * 1024 * 1024; // 50MB
+          if (stats.size > maxSizeBytes) {
+            return {
+              id: file.id,
+              detectedType: null,
+              detectedPedimento: null,
+              detectedPedimentoYear: null,
+              rfc: null,
+              arelaPath: null,
+              detectionError: `FILE_TOO_LARGE: File size ${(stats.size / 1024 / 1024).toFixed(2)}MB exceeds ${maxSizeBytes / 1024 / 1024}MB limit.`,
+              isNotPedimento: false,
+            };
+          }
+
+          // Detect using existing FileDetectionService
+          const result = await this.detectionService.detectFile(absolutePath);
+
+          // If detection succeeded and found a pedimento
+          if (result.detectedType === 'pedimento_simplificado') {
+            return {
+              id: file.id,
+              detectedType: result.detectedType,
+              detectedPedimento: result.detectedPedimento,
+              detectedPedimentoYear: result.detectedPedimentoYear,
+              rfc: result.rfc,
+              arelaPath: result.arelaPath,
+              detectionError: result.error,
+              isNotPedimento: false,
+            };
+          }
+
+          // If no detection, determine if it's definitely not a pedimento
+          // This helps avoid re-processing files we know aren't pedimentos
+          const isNotPedimento = this.#isDefinitelyNotPedimento(result, file);
+
+          // Build descriptive error message
+          let detectionError = null;
+          if (result.error) {
+            detectionError = `DETECTION_ERROR: ${result.error}`;
+          } else if (isNotPedimento) {
+            detectionError =
+              'NOT_PEDIMENTO: File does not match pedimento-simplificado pattern. Missing key markers: "FORMA SIMPLIFICADA DE PEDIMENTO".';
+          } else {
+            // Partial match - might be a pedimento with missing fields
+            const missingFields = this.#getMissingFields(result);
+            if (missingFields.length > 0) {
+              detectionError = `INCOMPLETE_PEDIMENTO: Detected as potential pedimento but missing fields: ${missingFields.join(', ')}. Matcher may need improvement.`;
+            } else {
+              detectionError =
+                'UNKNOWN_ERROR: Detection completed but no pedimento found. Check file content and matcher patterns.';
+            }
+          }
+
+          return {
+            id: file.id,
+            detectedType: result.detectedType,
+            detectedPedimento: result.detectedPedimento,
+            detectedPedimentoYear: result.detectedPedimentoYear,
+            rfc: result.rfc,
+            arelaPath: result.arelaPath,
+            detectionError,
+            isNotPedimento,
+          };
+        } catch (error) {
+          logger.warn(
+            `Failed to detect ${file.relative_path}: ${error.message}`,
+          );
+
+          // Categorize the error
+          let errorCategory = 'UNKNOWN_ERROR';
+          if (error.message.includes('ENOENT')) {
+            errorCategory = 'FILE_NOT_FOUND';
+          } else if (error.message.includes('timeout')) {
+            errorCategory = 'TIMEOUT';
+          } else if (error.message.includes('PDF')) {
+            errorCategory = 'PDF_PARSE_ERROR';
+          } else if (error.message.includes('extract')) {
+            errorCategory = 'TEXT_EXTRACTION_ERROR';
+          }
+
+          return {
+            id: file.id,
+            detectedType: null,
+            detectedPedimento: null,
+            detectedPedimentoYear: null,
+            rfc: null,
+            arelaPath: null,
+            detectionError: `${errorCategory}: ${error.message}`,
+            isNotPedimento: false,
+          };
+        }
+      }),
+    );
+
+    return Promise.all(detectionPromises);
+  }
+
+  /**
+   * Determine if a file is definitely not a pedimento
+   * @private
+   * @param {Object} result - Detection result
+   * @param {Object} file - File metadata
+   * @returns {boolean} True if definitely not a pedimento
+   */
+  #isDefinitelyNotPedimento(result, file) {
+    // If we got any pedimento-related fields, it might be a pedimento
+    if (result.detectedPedimento || result.rfc || result.arelaPath) {
+      return false;
+    }
+
+    // Check if the text contains the required pedimento marker
+    // This must match the criteria in pedimento-simplificado.js match function
+    const text = result.text || '';
+    const hasRequiredMarker = /FORMA SIMPLIFICADA DE PEDIMENTO/i.test(text);
+
+    // If the required marker is not found, it's definitely not a pedimento
+    return !hasRequiredMarker;
+  }
+
+  /**
+   * Get list of missing required fields for pedimento
+   * @private
+   * @param {Object} result - Detection result
+   * @returns {Array<string>} Missing field names
+   */
+  #getMissingFields(result) {
+    const requiredFields = [
+      { key: 'detectedPedimento', name: 'numPedimento' },
+      { key: 'rfc', name: 'rfc' },
+      { key: 'detectedPedimentoYear', name: 'year' },
+    ];
+
+    const missing = [];
+    for (const field of requiredFields) {
+      if (!result[field.key]) {
+        missing.push(field.name);
+      }
+    }
+
+    return missing;
+  }
+
+  /**
+   * Generate table name from scan config (same logic as scan command)
+   * @private
+   * @param {Object} scanConfig - Scan configuration
+   * @returns {string} Table name
+   */
+  #generateTableName(scanConfig) {
+    const { companySlug, serverId, basePathLabel } = scanConfig;
+
+    // Combine components
+    const rawName = `${companySlug}_${serverId}_${basePathLabel}`;
+
+    // Sanitize: lowercase, replace special chars with underscore
+    let sanitized = rawName
+      .toLowerCase()
+      .replace(/[^a-z0-9_]/g, '_')
+      .replace(/_+/g, '_')
+      .replace(/^_|_$/g, '');
+
+    // Add prefix
+    const tableName = 'scan_' + sanitized;
+
+    // Note: Hash truncation logic should match backend
+    // For simplicity, we rely on backend validation
+    return tableName;
+  }
+
+  /**
+   * Show detailed performance statistics
+   * @private
+   */
+  #showDetailedStats(startTime, processedCount, detectedCount, errorCount) {
+    const duration = (Date.now() - startTime) / 1000;
+    const avgSpeed = duration > 0 ? (processedCount / duration).toFixed(2) : 0;
+    const detectionRate =
+      processedCount > 0
+        ? ((detectedCount / processedCount) * 100).toFixed(1)
+        : 0;
+
+    logger.info('\n📈 Detailed Statistics:');
+    logger.info(`   Total Processing Time: ${duration.toFixed(2)}s`);
+    logger.info(`   Average Speed: ${avgSpeed} files/sec`);
+    logger.info(`   Detection Rate: ${detectionRate}%`);
+    logger.info(
+      `   Error Rate: ${errorCount > 0 ? ((errorCount / processedCount) * 100).toFixed(1) : 0}%`,
+    );
+
+    // Memory usage
+    const memUsage = process.memoryUsage();
+    logger.info('\n💾 Memory Usage:');
+    logger.info(`   RSS: ${(memUsage.rss / 1024 / 1024).toFixed(2)} MB`);
+    logger.info(
+      `   Heap Used: ${(memUsage.heapUsed / 1024 / 1024).toFixed(2)} MB`,
+    );
+    logger.info(
+      `   Heap Total: ${(memUsage.heapTotal / 1024 / 1024).toFixed(2)} MB`,
+    );
+  }
+}
+
+// Export singleton instance
+const identifyCommand = new IdentifyCommand();
+export default identifyCommand;