@arela/uploader 0.2.0 → 0.2.1

package/src/index.js

@@ -43,6 +43,7 @@ const sources = process.env.UPLOAD_SOURCES?.split('|')
   .filter(Boolean);
 
 // Configuración de RFCs para upload
+console.log('🔧 Configured RFCs for upload:', process.env.UPLOAD_RFCS);
 const uploadRfcs = process.env.UPLOAD_RFCS?.split('|')
   .map((s) => s.trim())
   .filter(Boolean);
@@ -156,8 +157,8 @@ const checkCredentials = async (forceSupabase = false) => {
     if (!supabaseUrl || !supabaseKey || !bucket) {
       console.error(
         '⚠️ Missing credentials. Please set either:\n' +
-          '   - ARELA_API_URL and ARELA_API_TOKEN for API mode, or\n' +
-          '   - SUPABASE_URL, SUPABASE_KEY, and SUPABASE_BUCKET for direct mode',
+        '   - ARELA_API_URL and ARELA_API_TOKEN for API mode, or\n' +
+        '   - SUPABASE_URL, SUPABASE_KEY, and SUPABASE_BUCKET for direct mode',
       );
       process.exit(1);
     }
@@ -179,13 +180,107 @@ const checkCredentials = async (forceSupabase = false) => {
 };
 
 const logFilePath = path.resolve(process.cwd(), 'arela-upload.log');
+
+/**
+ * OPTIMIZED: Log buffer to reduce I/O operations
+ */
+let logBuffer = [];
+const LOG_BUFFER_SIZE = 100; // Flush every 100 log entries
+let lastFlushTime = Date.now();
+const LOG_FLUSH_INTERVAL = 5000; // Flush every 5 seconds
+
+const flushLogBuffer = () => {
+  if (logBuffer.length === 0) return;
+
+  try {
+    const logContent = logBuffer.join('\n') + '\n';
+    fs.appendFileSync(logFilePath, logContent);
+    logBuffer = [];
+    lastFlushTime = Date.now();
+  } catch (error) {
+    console.error(`❌ Error writing to log file: ${error.code} | ${error.message} | path: ${logFilePath}`);
+  }
+};
+
 const writeLog = (message) => {
   try {
     const timestamp = new Date().toISOString();
-    fs.appendFileSync(logFilePath, `[${timestamp}] ${message}\n`);
+    logBuffer.push(`[${timestamp}] ${message}`);
+
+    // Flush if buffer is full or enough time has passed
+    const now = Date.now();
+    if (logBuffer.length >= LOG_BUFFER_SIZE || (now - lastFlushTime) >= LOG_FLUSH_INTERVAL) {
+      flushLogBuffer();
+    }
   } catch (error) {
-    console.error(`❌ Error writing to log file: ${error.code} | ${error.message} | path: ${logFilePath}`);
+    console.error(`❌ Error buffering log message: ${error.message}`);
+  }
+};
+
+// Ensure logs are flushed on process exit
+process.on('exit', flushLogBuffer);
+process.on('SIGINT', () => {
+  flushLogBuffer();
+  process.exit(0);
+});
+process.on('SIGTERM', () => {
+  flushLogBuffer();
+  process.exit(0);
+});
+
+/**
+ * OPTIMIZED: Conditional logging to reduce console overhead
+ */
+const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true';
+const BATCH_DELAY = parseInt(process.env.BATCH_DELAY) || 100; // Configurable delay between batches
+const PROGRESS_UPDATE_INTERVAL = parseInt(process.env.PROGRESS_UPDATE_INTERVAL) || 10; // Update progress every N items
+
+const logVerbose = (message) => {
+  if (VERBOSE_LOGGING) {
+    console.log(message);
+  }
+};
+const batchReadFileStats = (filePaths) => {
+  const results = [];
+
+  for (const filePath of filePaths) {
+    try {
+      const stats = fs.statSync(filePath);
+      results.push({ path: filePath, stats, error: null });
+    } catch (error) {
+      results.push({ path: filePath, stats: null, error: error.message });
+    }
+  }
+
+  return results;
+};
+
+/**
+ * OPTIMIZED: Cache for year/pedimento detection results to avoid redundant parsing
+ */
+const pathDetectionCache = new Map();
+
+/**
+ * OPTIMIZED: Clear the path detection cache (useful for testing or long-running processes)
+ */
+const clearPathDetectionCache = () => {
+  pathDetectionCache.clear();
+};
+
+/**
+ * OPTIMIZED: Get detection results with caching
+ */
+const getCachedPathDetection = (filePath, basePath) => {
+  const cacheKey = `${filePath}|${basePath}`;
+
+  if (pathDetectionCache.has(cacheKey)) {
+    return pathDetectionCache.get(cacheKey);
   }
+
+  const detection = extractYearAndPedimentoFromPath(filePath, basePath);
+  pathDetectionCache.set(cacheKey, detection);
+
+  return detection;
 };
 
 /**
@@ -276,23 +371,49 @@ const extractYearAndPedimentoFromPath = (filePath, basePath) => {
   }
 };
 
+/**
+ * OPTIMIZED: Get processed paths with caching and buffered log reading
+ */
+let processedPathsCache = null;
+let lastLogModTime = 0;
+
 const getProcessedPaths = () => {
-  const processed = new Set();
-  const lines = fs.existsSync(logFilePath)
-    ? fs.readFileSync(logFilePath, 'utf-8').split('\n')
-    : [];
-
-  for (const line of lines) {
-    const match = line.match(/(SUCCESS|SKIPPED): .*? -> (.+)/);
-    if (match) {
-      const [, , path] = match;
+  try {
+    // Check if log file exists
+    if (!fs.existsSync(logFilePath)) {
+      return new Set();
+    }
+
+    // Check if cache is still valid
+    const logStats = fs.statSync(logFilePath);
+    if (processedPathsCache && logStats.mtime.getTime() === lastLogModTime) {
+      return processedPathsCache;
+    }
+
+    // Read and parse log file
+    const processed = new Set();
+    const content = fs.readFileSync(logFilePath, 'utf-8');
+
+    // Use more efficient regex with global flag
+    const regex = /(SUCCESS|SKIPPED): .*? -> (.+)/g;
+    let match;
+
+    while ((match = regex.exec(content)) !== null) {
+      const path = match[2];
       if (path) {
         processed.add(path.trim());
       }
     }
-  }
 
-  return processed;
+    // Update cache
+    processedPathsCache = processed;
+    lastLogModTime = logStats.mtime.getTime();
+
+    return processed;
+  } catch (error) {
+    console.error(`⚠️ Error reading processed paths: ${error.message}`);
+    return new Set();
+  }
 };
 
 /**
@@ -314,6 +435,7 @@ const uploadToApi = async (files, options) => {
 
   // Nueva funcionalidad: estructura de carpetas personalizada
   let combinedStructure = null;
+  let cachedDetection = null; // Cache detection result to avoid redundant calls
 
   if (
     options.folderStructure &&
@@ -322,12 +444,10 @@ const uploadToApi = async (files, options) => {
   ) {
     // Combine custom folder structure with auto-detection
     const firstFile = files[0];
-    const detection = extractYearAndPedimentoFromPath(
-      firstFile.path,
-      process.cwd(),
-    );
-    if (detection.detected) {
-      const autoStructure = `${detection.year}/${detection.pedimento}`;
+    cachedDetection = getCachedPathDetection(firstFile.path, process.cwd());
+
+    if (cachedDetection.detected) {
+      const autoStructure = `${cachedDetection.year}/${cachedDetection.pedimento}`;
       combinedStructure = `${options.folderStructure}/${autoStructure}`;
       formData.append('folderStructure', combinedStructure);
       console.log(
@@ -346,12 +466,10 @@ const uploadToApi = async (files, options) => {
   } else if (options.autoDetectStructure && files.length > 0) {
     // Try to auto-detect from the first file if no explicit structure is provided
     const firstFile = files[0];
-    const detection = extractYearAndPedimentoFromPath(
-      firstFile.path,
-      process.cwd(),
-    );
-    if (detection.detected) {
-      const autoStructure = `${detection.year}/${detection.pedimento}`;
+    cachedDetection = getCachedPathDetection(firstFile.path, process.cwd());
+
+    if (cachedDetection.detected) {
+      const autoStructure = `${cachedDetection.year}/${cachedDetection.pedimento}`;
       formData.append('folderStructure', autoStructure);
     }
   }
@@ -420,9 +538,10 @@ const insertStatsToUploaderTable = async (files, options) => {
   const records = [];
 
   for (const file of files) {
-    const stats = fs.statSync(file.path);
+    // OPTIMIZED: Use pre-computed stats if available, otherwise call fs.statSync
+    const stats = file.stats || fs.statSync(file.path);
     const originalPath = options.clientPath || file.path;
-    
+
     // Check if record already exists
     const { data: existingRecords, error: checkError } = await supabase
       .from('uploader')
@@ -439,7 +558,7 @@ const insertStatsToUploaderTable = async (files, options) => {
       console.log(`⏭️ Skipping duplicate: ${path.basename(file.path)}`);
       continue;
     }
-    
+
     // Initialize record with basic file stats
     const record = {
       document_type: null,
@@ -457,17 +576,17 @@ const insertStatsToUploaderTable = async (files, options) => {
     if (detectionService.isSupportedFileType(file.path)) {
       try {
         const detection = await detectionService.detectFile(file.path);
-        
+
         if (detection.detectedType) {
           record.document_type = detection.detectedType;
           record.num_pedimento = detection.detectedPedimento;
           record.status = 'detected';
-          
+
           // Set arela_path for pedimento_simplificado documents
           if (detection.arelaPath) {
             record.arela_path = detection.arelaPath;
           }
-          
+
           // Extract RFC from fields if available
           const rfcField = detection.fields.find(f => f.name === 'rfc' && f.found);
           if (rfcField) {
@@ -498,7 +617,7 @@ const insertStatsToUploaderTable = async (files, options) => {
   }
 
   console.log(`💾 Inserting ${records.length} new records into uploader table...`);
-  
+
   const { data, error } = await supabase
     .from('uploader')
     .insert(records)
@@ -511,6 +630,266 @@ const insertStatsToUploaderTable = async (files, options) => {
   return data;
 };
 
+/**
+ * OPTIMIZED: Insert ONLY file stats into uploader table (Phase 1)
+ * No file reading, no detection - just filesystem metadata
+ * Returns summary statistics instead of full records for better performance
+ */
+const insertStatsOnlyToUploaderTable = async (files, options) => {
+  if (!supabase) {
+    throw new Error('Supabase client not initialized. Stats mode requires Supabase connection.');
+  }
+
+  const batchSize = 1000; // Large batch size for performance
+  const allRecords = [];
+
+  // Prepare all file stats data first - OPTIMIZED to use pre-computed stats
+  console.log('📊 Collecting filesystem stats...');
+  for (const file of files) {
+    try {
+      // Use pre-computed stats if available, otherwise call fs.statSync
+      const stats = file.stats || fs.statSync(file.path);
+      const originalPath = options.clientPath || file.path;
+      const fileExtension = path.extname(file.path).toLowerCase().replace('.', '');
+
+      const record = {
+        document_type: null,
+        size: stats.size,
+        num_pedimento: null,
+        filename: file.originalName || path.basename(file.path),
+        original_path: originalPath,
+        arela_path: null,
+        status: 'fs-stats',
+        rfc: null,
+        message: null,
+        file_extension: fileExtension,
+        created_at: new Date().toISOString(),
+        modified_at: stats.mtime.toISOString()
+      };
+
+      allRecords.push(record);
+    } catch (error) {
+      console.error(`❌ Error reading stats for ${file.path}:`, error.message);
+    }
+  }
+
+  if (allRecords.length === 0) {
+    console.log('📝 No file stats to insert');
+    return { totalInserted: 0, totalSkipped: 0, totalProcessed: 0 };
+  }
+
+  console.log(`💾 Bulk inserting ${allRecords.length} file stats in batches of ${batchSize}...`);
+
+  let totalInserted = 0;
+  let totalSkipped = 0;
+
+  // Process in batches for optimal performance
+  for (let i = 0; i < allRecords.length; i += batchSize) {
+    const batch = allRecords.slice(i, i + batchSize);
+
+    try {
+      // OPTIMIZED: Use upsert without select to avoid unnecessary data transfer
+      const { error, count } = await supabase
+        .from('uploader')
+        .upsert(batch, {
+          onConflict: 'original_path',
+          ignoreDuplicates: false,
+          count: 'exact'
+        });
+
+      if (error) {
+        console.error(`❌ Error inserting batch ${Math.floor(i / batchSize) + 1}:`, error.message);
+        continue;
+      }
+
+      // For upsert operations, we can't easily distinguish between inserts and updates
+      // from the count alone, but we can estimate based on the assumption that most
+      // operations in --stats-only mode are likely new inserts
+      const batchProcessed = batch.length;
+
+      // Since we're using upsert with ignoreDuplicates: false, the count represents
+      // the actual number of rows affected (both inserts and updates)
+      const affected = count || batchProcessed;
+
+      // For simplicity and performance, we'll assume most are new inserts in stats-only mode
+      // This is reasonable since stats-only is typically run on new file sets
+      totalInserted += affected;
+
+      console.log(`✅ Batch ${Math.floor(i / batchSize) + 1}: ${affected} rows processed`);
+    } catch (error) {
+      console.error(`❌ Unexpected error in batch ${Math.floor(i / batchSize) + 1}:`, error.message);
+    }
+  }
+
+  // Calculate skipped as difference between total records and inserted
+  totalSkipped = allRecords.length - totalInserted;
+
+  console.log(`📊 Phase 1 Summary: ${totalInserted} records processed, estimated ${totalSkipped} were updates`);
+
+  return {
+    totalInserted,
+    totalSkipped,
+    totalProcessed: allRecords.length
+  };
+};
+
+/**
+ * PHASE 2: Process PDF files for pedimento-simplificado detection
+ * Only processes files with status 'fs-stats' and file_extension 'pdf'
+ */
+const detectPedimentosInDatabase = async (options = {}) => {
+  if (!supabase) {
+    throw new Error('Supabase client not initialized.');
+  }
+
+  console.log('🔍 Phase 2: Starting PDF detection for pedimento-simplificado documents...');
+
+  // Get all PDF files that need detection (status = 'fs-stats' and extension = 'pdf')
+  let allPdfRecords = [];
+  let hasMore = true;
+  let offset = 0;
+  const queryBatchSize = 1000;
+
+  console.log('📥 Fetching PDF files from database...');
+
+  while (hasMore) {
+    const { data: batch, error: queryError } = await supabase
+      .from('uploader')
+      .select('id, original_path, filename, file_extension, status')
+      .eq('status', 'fs-stats')
+      .eq('file_extension', 'pdf')
+      .ilike('filename', '%simp%')
+      .range(offset, offset + queryBatchSize - 1);
+
+    if (queryError) {
+      throw new Error(`Failed to fetch PDF records: ${queryError.message}`);
+    }
+
+    if (!batch || batch.length === 0) {
+      hasMore = false;
+    } else {
+      allPdfRecords.push(...batch);
+      offset += queryBatchSize;
+      console.log(`📄 Fetched ${batch.length} PDF records (total: ${allPdfRecords.length})`);
+    }
+  }
+
+  if (allPdfRecords.length === 0) {
+    console.log('📝 No PDF files found for detection');
+    return { detectedCount: 0, processedCount: 0, errorCount: 0 };
+  }
+
+  console.log(`🔍 Processing ${allPdfRecords.length} PDF files for detection...`);
+
+  const detectionService = new FileDetectionService();
+  const batchSize = parseInt(options.batchSize) || 10; // Smaller batches for file I/O
+  let totalDetected = 0;
+  let totalProcessed = 0;
+  let totalErrors = 0;
+
+  // Create progress bar
+  const progressBar = new cliProgress.SingleBar({
+    format: '🔍 PDF Detection |{bar}| {percentage}% | {value}/{total} | Detected: {detected} | Errors: {errors}',
+    barCompleteChar: '█',
+    barIncompleteChar: '░',
+    hideCursor: true,
+  });
+
+  progressBar.start(allPdfRecords.length, 0, { detected: 0, errors: 0 });
+
+  // Process files in smaller batches to avoid overwhelming the system
+  for (let i = 0; i < allPdfRecords.length; i += batchSize) {
+    const batch = allPdfRecords.slice(i, i + batchSize);
+    const updatePromises = [];
+
+    for (const record of batch) {
+      try {
+        // Check if file still exists
+        if (!fs.existsSync(record.original_path)) {
+          updatePromises.push(
+            supabase
+              .from('uploader')
+              .update({
+                status: 'file-not-found',
+                message: 'File no longer exists at original path'
+              })
+              .eq('id', record.id)
+          );
+          totalErrors++;
+          continue;
+        }
+
+        // Perform detection
+        const detection = await detectionService.detectFile(record.original_path);
+        totalProcessed++;
+
+        const updateData = {
+          status: detection.detectedType ? 'detected' : 'not-detected',
+          document_type: detection.detectedType,
+          num_pedimento: detection.detectedPedimento,
+          arela_path: detection.arelaPath,
+          message: detection.error || null
+        };
+
+        // Extract RFC from fields if available
+        if (detection.fields) {
+          const rfcField = detection.fields.find(f => f.name === 'rfc' && f.found);
+          if (rfcField) {
+            updateData.rfc = rfcField.value;
+          }
+        }
+
+        if (detection.detectedType) {
+          totalDetected++;
+        }
+
+        updatePromises.push(
+          supabase
+            .from('uploader')
+            .update(updateData)
+            .eq('id', record.id)
+        );
+
+      } catch (error) {
+        console.error(`❌ Error detecting ${record.filename}:`, error.message);
+        totalErrors++;
+
+        updatePromises.push(
+          supabase
+            .from('uploader')
+            .update({
+              status: 'detection-error',
+              message: error.message
+            })
+            .eq('id', record.id)
+        );
+      }
+    }
+
+    // Execute all updates in parallel for this batch
+    try {
+      await Promise.all(updatePromises);
+    } catch (error) {
+      console.error(`❌ Error updating batch:`, error.message);
+    }
+
+    // Update progress
+    progressBar.update(Math.min(i + batchSize, allPdfRecords.length), {
+      detected: totalDetected,
+      errors: totalErrors
+    });
+  }
+
+  progressBar.stop();
+
+  console.log(`📊 Phase 2 Summary: ${totalDetected} detected, ${totalProcessed} processed, ${totalErrors} errors`);
+  return {
+    detectedCount: totalDetected,
+    processedCount: totalProcessed,
+    errorCount: totalErrors
+  };
+};
+
 const processFilesInBatches = async (
   files,
   batchSize,
@@ -528,7 +907,7 @@ const processFilesInBatches = async (
 
   const messageBuffer = [];
 
-  const progressBarFormat = options.statsOnly 
+  const progressBarFormat = options.statsOnly
     ? '📊 Processing [{bar}] {percentage}% | {value}/{total} files | Stats: {successCount} | Errors: {failureCount} | Duplicates: {skippedCount}'
     : '📂 Processing [{bar}] {percentage}% | {value}/{total} files | Success: {successCount} | Errors: {failureCount} | Skipped: {skippedCount}';
 
@@ -546,98 +925,59 @@ const processFilesInBatches = async (
   });
 
   if (options.statsOnly) {
-    // Stats-only mode - Read file stats and insert to uploader table
-    console.log('📊 Processing files in stats-only mode...');
-    
-    let totalDetected = 0;
-    let totalNotDetected = 0;
-    let totalUnsupported = 0;
-    let totalDetectionErrors = 0;
-    
+    // OPTIMIZED Stats-only mode - Only read filesystem stats, no file detection
+    console.log('📊 Phase 1: Processing files in optimized stats-only mode (no detection)...');
+
     for (let i = 0; i < files.length; i += batchSize) {
       const batch = files.slice(i, i + batchSize);
-      
-      const statsFiles = batch.map((file) => {
-        const originalFileName = path.basename(file);
-        
-        return {
-          path: file,
-          originalName: originalFileName,
-        };
-      });
 
-      try {
-        const insertedRecords = await insertStatsToUploaderTable(statsFiles, options);
-        const actualInserted = insertedRecords.length;
-        const skippedDuplicates = statsFiles.length - actualInserted;
-        
-        totalUploaded += actualInserted;
-        totalSkipped += skippedDuplicates;
-        
-        // Count detection results from inserted records
-        insertedRecords.forEach(record => {
-          switch (record.status) {
-            case 'detected':
-              totalDetected++;
-              break;
-            case 'not-detected':
-              totalNotDetected++;
-              break;
-            case 'unsupported':
-              totalUnsupported++;
-              break;
-            case 'detection-error':
-              totalDetectionErrors++;
-              break;
-          }
-        });
-        
-        statsFiles.forEach((file) => {
-          const wasInserted = insertedRecords.some(record => 
-            record.original_path === (options.clientPath || file.path)
-          );
-          if (wasInserted) {
-            writeLog(`STATS: ${file.path} -> uploader table`);
-          } else {
-            writeLog(`DUPLICATE: ${file.path} -> already exists in uploader table`);
-          }
+      // OPTIMIZED: Batch read file stats to reduce I/O overhead
+      const fileStatsResults = batchReadFileStats(batch);
+      const statsFiles = fileStatsResults
+        .filter(result => result.stats !== null) // Only include files with valid stats
+        .map((result) => {
+          const originalFileName = path.basename(result.path);
+
+          return {
+            path: result.path,
+            originalName: originalFileName,
+            stats: result.stats, // Pass pre-computed stats to avoid redundant calls
+          };
         });
-        
-        if (actualInserted > 0) {
-          console.log(`📈 Inserted ${actualInserted} stats records`);
-        }
-        if (skippedDuplicates > 0) {
-          console.log(`⏭️ Skipped ${skippedDuplicates} duplicates`);
-        }
-        if (options.detect !== false) {
-          console.log(`   🔍 Detected: ${totalDetected}, Not detected: ${totalNotDetected}, Unsupported: ${totalUnsupported}, Errors: ${totalDetectionErrors}`);
-        }
-        
-      } catch (error) {
-        totalErrors += statsFiles.length;
-        statsFiles.forEach((file) => {
-          writeLog(`ERROR: ${file.path}: ${error.message}`);
-          messageBuffer.push(`❌ ${file.originalName}: ${error.message}`);
+
+      // Log any files that couldn't be read
+      const failedFiles = fileStatsResults.filter(result => result.error !== null);
+      if (failedFiles.length > 0) {
+        console.log(`⚠️ Could not read stats for ${failedFiles.length} files in batch`);
+        failedFiles.forEach(failed => {
+          console.error(`   ❌ ${failed.path}: ${failed.error}`);
         });
       }
 
-      progressBar.update(i + batch.length, {
-        successCount: totalUploaded,
-        failureCount: totalErrors,
-        skippedCount: totalSkipped,
-      });
+      try {
+        const result = await insertStatsOnlyToUploaderTable(statsFiles, options);
 
-      if (i + batchSize < files.length) {
-        await new Promise((resolve) => setTimeout(resolve, 200));
+        totalUploaded += result.totalInserted;
+        totalSkipped += result.totalSkipped;
+        totalErrors += failedFiles.length; // Count failed file reads as errors
+
+        progressBar.update(Math.min(i + batch.length, files.length), {
+          successCount: totalUploaded,
+          failureCount: totalErrors,
+          skippedCount: totalSkipped,
+        });
+
+      } catch (error) {
+        console.error(`❌ Error processing stats batch:`, error.message);
+        totalErrors += batch.length;
+
+        progressBar.update(Math.min(i + batch.length, files.length), {
+          successCount: totalUploaded,
+          failureCount: totalErrors,
+          skippedCount: totalSkipped,
+        });
       }
     }
-    
-    // Store detection stats for summary
-    totalDetected = totalDetected || 0;
-    totalNotDetected = totalNotDetected || 0;
-    totalUnsupported = totalUnsupported || 0;
-    totalDetectionErrors = totalDetectionErrors || 0;
-    
   } else if (apiMode && !options.forceSupabase) {
     // API Mode - Process in batches
     for (let i = 0; i < files.length; i += batchSize) {
@@ -661,7 +1001,8 @@ const processFilesInBatches = async (
 
           // Handle combined folder structure + auto-detection
           if (options.folderStructure && options.autoDetectStructure) {
-            const detection = extractYearAndPedimentoFromPath(file, basePath);
+            // OPTIMIZED: Use cached detection to avoid redundant parsing
+            const detection = getCachedPathDetection(file, basePath);
             if (detection.detected) {
               const autoStructure = `${detection.year}/${detection.pedimento}`;
               const combinedStructure = `${options.folderStructure}/${autoStructure}`;
@@ -669,7 +1010,7 @@ const processFilesInBatches = async (
                 combinedStructure,
                 sanitizedFileName,
               );
-              console.log(
+              logVerbose(
                 `📁 Combined structure: ${options.folderStructure}/${autoStructure} for ${originalFileName} -> ${uploadPath}`,
               );
             } else {
@@ -678,7 +1019,7 @@ const processFilesInBatches = async (
                 options.folderStructure,
                 sanitizedFileName,
               );
-              console.log(
+              logVerbose(
                 `📁 Custom structure (auto-detection failed): ${uploadPath}`,
               );
             }
@@ -688,10 +1029,10 @@ const processFilesInBatches = async (
               options.folderStructure,
               sanitizedFileName,
             );
-            console.log(`📁 Custom structure: ${uploadPath}`);
+            logVerbose(`📁 Custom structure: ${uploadPath}`);
           } else if (options.autoDetectStructure) {
-            // Auto-detect structure from path if enabled
-            const detection = extractYearAndPedimentoFromPath(file, basePath);
+            // Auto-detect structure from path if enabled - OPTIMIZED: Use cached detection
+            const detection = getCachedPathDetection(file, basePath);
             if (detection.detected) {
               const autoStructure = `${detection.year}/${detection.pedimento}`;
               uploadPath = path.posix.join(autoStructure, sanitizedFileName);
@@ -737,10 +1078,8 @@ const processFilesInBatches = async (
 
           if (!clientPath && apiFiles.length > 0) {
             const firstFile = apiFiles[0];
-            const detection = extractYearAndPedimentoFromPath(
-              firstFile.path,
-              basePath,
-            );
+            // OPTIMIZED: Use cached detection to avoid redundant parsing
+            const detection = getCachedPathDetection(firstFile.path, basePath);
             if (detection.detected) {
               // clientPath = `${detection.year}/${detection.pedimento}/`;
               clientPath = path
@@ -796,7 +1135,7 @@ const processFilesInBatches = async (
       });
 
       if (i + batchSize < files.length) {
-        await new Promise((resolve) => setTimeout(resolve, 200));
+        await new Promise((resolve) => setTimeout(resolve, BATCH_DELAY));
       }
     }
   } else {
@@ -809,7 +1148,7 @@ const processFilesInBatches = async (
 
         // Handle combined folder structure + auto-detection
         if (options.folderStructure && options.autoDetectStructure) {
-          const detection = extractYearAndPedimentoFromPath(file, basePath);
+          const detection = getCachedPathDetection(file, basePath);
           if (detection.detected) {
             const autoStructure = `${detection.year}/${detection.pedimento}`;
             const combinedStructure = `${options.folderStructure}/${autoStructure}`;
@@ -832,8 +1171,8 @@ const processFilesInBatches = async (
           uploadPath = path.join(options.folderStructure, fileName);
           console.log(`📁 Custom structure: ${uploadPath}`);
         } else if (options.autoDetectStructure) {
-          // Auto-detect structure from path if enabled
-          const detection = extractYearAndPedimentoFromPath(file, basePath);
+          // Auto-detect structure from path if enabled - OPTIMIZED: Use cached detection
+          const detection = getCachedPathDetection(file, basePath);
           if (detection.detected) {
             const autoStructure = `${detection.year}/${detection.pedimento}`;
             const fileName = path.basename(file);
@@ -943,7 +1282,7 @@ const uploadFilesByRfc = async (options = {}) => {
   const queryBatchSize = 1000;
 
   console.log('📥 Fetching all related files (with pagination)...');
-  
+
   while (hasMore) {
     const { data: batch, error: queryError } = await supabase
       .from('uploader')
@@ -962,7 +1301,7 @@ const uploadFilesByRfc = async (options = {}) => {
     } else {
       allRelatedFiles = allRelatedFiles.concat(batch);
       offset += queryBatchSize;
-      
+
       // If we got less than queryBatchSize, we've reached the end
       if (batch.length < queryBatchSize) {
         hasMore = false;
@@ -976,7 +1315,7 @@ const uploadFilesByRfc = async (options = {}) => {
   }
 
   console.log(`📁 Found ${allRelatedFiles.length} total files to upload (including supporting documents)`);
-  
+
   // Group by RFC and arela_path for better organization
   const filesByRfc = allRelatedFiles.reduce((acc, record) => {
     const rfc = record.rfc || 'No RFC';
@@ -1037,18 +1376,18 @@ const uploadFilesByRfc = async (options = {}) => {
     const batch = allRelatedFiles.slice(i, i + batchSize);
     const batchNumber = Math.floor(i / batchSize) + 1;
     const totalBatches = Math.ceil(allRelatedFiles.length / batchSize);
-    
+
     console.log(`\n📦 Processing batch ${batchNumber}/${totalBatches} (${batch.length} files)`);
 
     // Prepare files for upload
     const filesToUpload = [];
-    
+
     for (const record of batch) {
       totalProcessed++;
-      
+
       try {
         const originalPath = record.original_path;
-        
+
         // Check if file exists
         if (!fs.existsSync(originalPath)) {
           console.log(`   ⚠️  File not found: ${originalPath}`);
@@ -1056,24 +1395,24 @@ const uploadFilesByRfc = async (options = {}) => {
           continue;
         }
 
-        const fileStats = fs.statSync(originalPath);
+        // OPTIMIZED: Read file and get size from buffer instead of separate fs.statSync call
         const fileBuffer = fs.readFileSync(originalPath);
-        
+
         filesToUpload.push({
           path: originalPath,
           buffer: fileBuffer,
-          size: fileStats.size,
+          size: fileBuffer.length, // Get size from buffer instead of fs.statSync
           name: record.filename,
           arelaPath: record.arela_path,
           rfc: record.rfc,
           documentType: record.document_type,
         });
-        
+
       } catch (error) {
         console.error(`   ❌ Error reading file ${record.original_path}:`, error.message);
         totalErrors++;
       }
-      
+
       if (options.showProgress !== false) {
         progressBar.update(totalProcessed, {
           uploaded: totalUploaded,
@@ -1087,9 +1426,9 @@ const uploadFilesByRfc = async (options = {}) => {
     if (filesToUpload.length > 0) {
       try {
         console.log(`   🚀 Uploading ${filesToUpload.length} files to Arela API...`);
-        
+
         const formData = new FormData();
-        
+
         // Add files to form data
         filesToUpload.forEach((file, index) => {
           formData.append(`files`, file.buffer, {
@@ -1112,7 +1451,7 @@ const uploadFilesByRfc = async (options = {}) => {
         // Upload each group separately with its folder structure
         for (const [arelaPath, pathFiles] of Object.entries(filesByPath)) {
           const pathFormData = new FormData();
-          
+
           pathFiles.forEach((file) => {
             pathFormData.append('files', file.buffer, {
               filename: file.name,
@@ -1121,7 +1460,7 @@ const uploadFilesByRfc = async (options = {}) => {
           });
 
           // Set folder structure for this group - concatenate custom prefix with arela_path
-          const folderStructure = options.folderStructure 
+          const folderStructure = options.folderStructure
             ? `${options.folderStructure}/${arelaPath}`.replace(/\/+/g, '/').replace(/\/$/, '')
             : arelaPath;
           pathFormData.append('folderStructure', folderStructure);
@@ -1149,14 +1488,14 @@ const uploadFilesByRfc = async (options = {}) => {
           }
 
           const result = await response.json();
-          
+
           // Check if upload was successful based on stats rather than success field
           const isSuccessful = result.stats && result.stats.uploadedCount > 0 && result.stats.errorCount === 0;
-          
+
           if (isSuccessful) {
             console.log(`     ✅ Group uploaded: ${result.stats.uploadedCount} files to ${folderStructure}`);
             totalUploaded += result.stats.uploadedCount;
-            
+
             if (result.stats.detectedCount > 0) {
               console.log(`     🔍 Files detected: ${result.stats.detectedCount}`);
             }
@@ -1185,7 +1524,7 @@ const uploadFilesByRfc = async (options = {}) => {
 
     // Small delay between batches
     if (i + batchSize < allRelatedFiles.length) {
-      await new Promise(resolve => setTimeout(resolve, 200));
+      await new Promise(resolve => setTimeout(resolve, BATCH_DELAY));
     }
   }
 
@@ -1263,19 +1602,19 @@ const propagateArelaPath = async (options = {}) => {
   for (const pedimento of pedimentoRecords) {
     try {
       totalProcessed++;
-      
+
       // Extract base path from original_path (remove filename)
       const basePath = path.dirname(pedimento.original_path);
-      
+
       console.log(`\n🔍 Processing: ${pedimento.filename}`);
       console.log(`   📁 Base path: ${basePath}`);
-      
+
       // Extract folder part from existing arela_path by removing the filename
       const existingPath = pedimento.arela_path;
-      const folderArelaPath = existingPath.includes('/') ? 
-        existingPath.substring(0, existingPath.lastIndexOf('/')) + '/' : 
+      const folderArelaPath = existingPath.includes('/') ?
+        existingPath.substring(0, existingPath.lastIndexOf('/')) + '/' :
         existingPath.endsWith('/') ? existingPath : existingPath + '/';
-        
+
       console.log(`   🎯 Original arela path: ${existingPath}`);
       console.log(`   📁 Folder arela path: ${folderArelaPath}`);
 
@@ -1299,13 +1638,13 @@ const propagateArelaPath = async (options = {}) => {
       }
 
       console.log(`   📄 Found ${relatedFiles.length} related files to update:`);
-      
+
       // Show first 10 files, then indicate if there are more
       const filesToShow = relatedFiles.slice(0, 10);
       filesToShow.forEach(file => {
         console.log(`      - ${file.filename}`);
       });
-      
+
       if (relatedFiles.length > 10) {
         console.log(`      ... and ${relatedFiles.length - 10} more files`);
       }
@@ -1322,7 +1661,7 @@ const propagateArelaPath = async (options = {}) => {
         const batchIds = fileIds.slice(i, i + BATCH_SIZE);
         const batchNumber = Math.floor(i / BATCH_SIZE) + 1;
         const totalBatches = Math.ceil(fileIds.length / BATCH_SIZE);
-        
+
         console.log(`   📦 Batch ${batchNumber}/${totalBatches}: Updating ${batchIds.length} files...`);
 
         try {
@@ -1422,25 +1761,53 @@ program
     'Automatically detect year/pedimento from file paths',
   )
   .option('--client-path <path>', 'Client path for metadata tracking')
-  .option('--stats-only', 'Only read file stats and insert to uploader table, skip file upload')
+  .option('--stats-only', 'Phase 1: Only read filesystem stats and insert to database (no file reading or detection)')
   .option('--no-detect', 'Disable document type detection in stats-only mode')
-  .option('--propagate-arela-path', 'Propagate arela_path from pedimento_simplificado records to related files with same base path')
-  .option('--upload-by-rfc', 'Upload files to Arela API based on RFC values from UPLOAD_RFCS environment variable')
+  .option('--detect-pdfs', 'Phase 2: Process PDF files in database for pedimento-simplificado detection')
+  .option('--propagate-arela-path', 'Phase 3: Propagate arela_path from pedimento_simplificado records to related files with same base path')
+  .option('--upload-by-rfc', 'Phase 4: Upload files to Arela API based on RFC values from UPLOAD_RFCS environment variable')
+  .option('--run-all-phases', 'Run all 4 phases in sequence: stats → detect → propagate → upload')
   .action(async (options) => {
     if (options.version) {
       console.log(packageVersion);
       process.exit(0);
     }
 
+    // Handle detect-pdfs option (Phase 2)
+    if (options.detectPdfs) {
+      console.log('🔍 Starting Phase 2: PDF Detection');
+      await checkCredentials(true); // Force Supabase mode
+
+      const result = await detectPedimentosInDatabase({
+        batchSize: parseInt(options.batchSize) || 10,
+      });
+
+      console.log(`✅ Phase 2 Complete: ${result.detectedCount} detected, ${result.errorCount} errors`);
+      return;
+    }
+
+    // Handle run-all-phases option
+    if (options.runAllPhases) {
+      console.log('🚀 Starting all 4 phases in sequence...');
+      await checkCredentials(true); // Force Supabase mode
+
+      // Phase 1: Stats collection
+      console.log('\n📊 === PHASE 1: Filesystem Stats ===');
+      options.statsOnly = true;
+      // Continue with normal processing to run Phase 1
+
+      // The rest will be handled after Phase 1 completes
+    }
+
     // Handle propagate-arela-path option
     if (options.propagateArelaPath) {
       // Initialize Supabase credentials for propagation
       await checkCredentials(true); // Force Supabase mode
-      
+
       const result = await propagateArelaPath({
         showProgress: options.showStats || true,
       });
-      
+
       if (result.errorCount > 0) {
         process.exit(1);
       }
@@ -1451,7 +1818,7 @@ program
     if (options.uploadByRfc) {
       // RFC upload needs both Supabase (for database queries) and API (for uploads)
       await checkCredentials(false); // Initialize API mode
-      
+
       // Also initialize Supabase for database queries
       if (!supabase) {
         if (!supabaseUrl || !supabaseKey) {
@@ -1459,17 +1826,17 @@ program
           console.error('   Please set SUPABASE_URL and SUPABASE_KEY environment variables.');
           process.exit(1);
         }
-        
+
         supabase = createClient(supabaseUrl, supabaseKey);
         console.log('✅ Connected to Supabase for database queries');
       }
-      
+
       const result = await uploadFilesByRfc({
         showProgress: options.showStats || true,
         batchSize: parseInt(options.batchSize) || 10,
         folderStructure: options.folderStructure,
       });
-      
+
       if (result.errorCount > 0) {
         process.exit(1);
       }
@@ -1590,12 +1957,65 @@ program
     console.log(`   📜 Log file: ${logFilePath}`);
     console.log(`${'='.repeat(60)}\n`);
 
-    if (options.showStats && sanitizationCache.size > 0) {
+    // Continue with remaining phases if running all phases
+    if (options.runAllPhases && options.statsOnly) {
+      try {
+        // Phase 2: PDF Detection
+        console.log('\n🔍 === PHASE 2: PDF Detection ===');
+        const detectionResult = await detectPedimentosInDatabase({
+          batchSize: parseInt(options.batchSize) || 10,
+        });
+        console.log(`✅ Phase 2 Complete: ${detectionResult.detectedCount} detected, ${detectionResult.errorCount} errors`);
+
+        // Phase 3: Propagate arela_path
+        console.log('\n📁 === PHASE 3: Propagate Arela Paths ===');
+        const propagateResult = await propagateArelaPath({
+          showProgress: options.showStats || true,
+        });
+        console.log(`✅ Phase 3 Complete: ${propagateResult.updatedCount || 0} paths propagated`);
+
+        // Phase 4: Upload by RFC
+        if (uploadRfcs && uploadRfcs.length > 0) {
+          console.log('\n🚀 === PHASE 4: Upload by RFC ===');
+
+          // Initialize API mode for uploads
+          await checkCredentials(false);
+
+          const uploadResult = await uploadFilesByRfc({
+            showProgress: options.showStats || true,
+            batchSize: parseInt(options.batchSize) || 10,
+            folderStructure: options.folderStructure,
+          });
+          console.log(`✅ Phase 4 Complete: Upload finished`);
+        } else {
+          console.log('\n⚠️ === PHASE 4: Upload by RFC ===');
+          console.log('⚠️ UPLOAD_RFCS environment variable not configured, skipping Phase 4');
+        }
+
+        console.log('\n🎉 All 4 phases completed successfully!');
+
+      } catch (error) {
+        console.error(`❌ Error in multi-phase execution:`, error.message);
+        process.exit(1);
+      }
+    }
+
+    if (options.showStats && (sanitizationCache.size > 0 || pathDetectionCache.size > 0)) {
       console.log(`📊 Performance Statistics:`);
-      console.log(
-        `   🗂️  Sanitization cache entries: ${sanitizationCache.size}`,
-      );
+      if (sanitizationCache.size > 0) {
+        console.log(
+          `   🗂️  Sanitization cache entries: ${sanitizationCache.size}`,
+        );
+      }
+      if (pathDetectionCache.size > 0) {
+        console.log(
+          `   📁  Path detection cache entries: ${pathDetectionCache.size}`,
+        );
+      }
     }
+
+    // OPTIMIZED: Ensure log buffer is flushed before exit
+    flushLogBuffer();
   });
 
 program.parse();