@arela/uploader 0.2.0 → 0.2.2

package/src/index.js

@@ -9,6 +9,7 @@ import { globby } from 'globby';
 import mime from 'mime-types';
 import fetch from 'node-fetch';
 import path from 'path';
+
 import { FileDetectionService } from './file-detection.js';
 
 config();
@@ -43,6 +44,7 @@ const sources = process.env.UPLOAD_SOURCES?.split('|')
   .filter(Boolean);
 
 // Configuración de RFCs para upload
+console.log('🔧 Configured RFCs for upload:', process.env.UPLOAD_RFCS);
 const uploadRfcs = process.env.UPLOAD_RFCS?.split('|')
   .map((s) => s.trim())
   .filter(Boolean);
@@ -179,13 +181,111 @@ const checkCredentials = async (forceSupabase = false) => {
 };
 
 const logFilePath = path.resolve(process.cwd(), 'arela-upload.log');
+
+/**
+ * OPTIMIZED: Log buffer to reduce I/O operations
+ */
+let logBuffer = [];
+const LOG_BUFFER_SIZE = 100; // Flush every 100 log entries
+let lastFlushTime = Date.now();
+const LOG_FLUSH_INTERVAL = 5000; // Flush every 5 seconds
+
+const flushLogBuffer = () => {
+  if (logBuffer.length === 0) return;
+
+  try {
+    const logContent = logBuffer.join('\n') + '\n';
+    fs.appendFileSync(logFilePath, logContent);
+    logBuffer = [];
+    lastFlushTime = Date.now();
+  } catch (error) {
+    console.error(`❌ Error writing to log file: ${error.code} | ${error.message} | path: ${logFilePath}`);
+  }
+};
+
 const writeLog = (message) => {
   try {
     const timestamp = new Date().toISOString();
-    fs.appendFileSync(logFilePath, `[${timestamp}] ${message}\n`);
+    logBuffer.push(`[${timestamp}] ${message}`);
+
+    // Flush if buffer is full or enough time has passed
+    const now = Date.now();
+    if (
+      logBuffer.length >= LOG_BUFFER_SIZE ||
+      now - lastFlushTime >= LOG_FLUSH_INTERVAL
+    ) {
+      flushLogBuffer();
+    }
   } catch (error) {
-    console.error(`❌ Error writing to log file: ${error.code} | ${error.message} | path: ${logFilePath}`);
+    console.error(`❌ Error buffering log message: ${error.message}`);
+  }
+};
+
+// Ensure logs are flushed on process exit
+process.on('exit', flushLogBuffer);
+process.on('SIGINT', () => {
+  flushLogBuffer();
+  process.exit(0);
+});
+process.on('SIGTERM', () => {
+  flushLogBuffer();
+  process.exit(0);
+});
+
+/**
+ * OPTIMIZED: Conditional logging to reduce console overhead
+ */
+const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true';
+const BATCH_DELAY = parseInt(process.env.BATCH_DELAY) || 100; // Configurable delay between batches
+const PROGRESS_UPDATE_INTERVAL =
+  parseInt(process.env.PROGRESS_UPDATE_INTERVAL) || 10; // Update progress every N items
+
+const logVerbose = (message) => {
+  if (VERBOSE_LOGGING) {
+    console.log(message);
+  }
+};
+const batchReadFileStats = (filePaths) => {
+  const results = [];
+
+  for (const filePath of filePaths) {
+    try {
+      const stats = fs.statSync(filePath);
+      results.push({ path: filePath, stats, error: null });
+    } catch (error) {
+      results.push({ path: filePath, stats: null, error: error.message });
+    }
   }
+
+  return results;
+};
+
+/**
+ * OPTIMIZED: Cache for year/pedimento detection results to avoid redundant parsing
+ */
+const pathDetectionCache = new Map();
+
+/**
+ * OPTIMIZED: Clear the path detection cache (useful for testing or long-running processes)
+ */
+const clearPathDetectionCache = () => {
+  pathDetectionCache.clear();
+};
+
+/**
+ * OPTIMIZED: Get detection results with caching
+ */
+const getCachedPathDetection = (filePath, basePath) => {
+  const cacheKey = `${filePath}|${basePath}`;
+
+  if (pathDetectionCache.has(cacheKey)) {
+    return pathDetectionCache.get(cacheKey);
+  }
+
+  const detection = extractYearAndPedimentoFromPath(filePath, basePath);
+  pathDetectionCache.set(cacheKey, detection);
+
+  return detection;
 };
 
 /**
@@ -276,23 +376,49 @@ const extractYearAndPedimentoFromPath = (filePath, basePath) => {
   }
 };
 
+/**
+ * OPTIMIZED: Get processed paths with caching and buffered log reading
+ */
+let processedPathsCache = null;
+let lastLogModTime = 0;
+
 const getProcessedPaths = () => {
-  const processed = new Set();
-  const lines = fs.existsSync(logFilePath)
-    ? fs.readFileSync(logFilePath, 'utf-8').split('\n')
-    : [];
-
-  for (const line of lines) {
-    const match = line.match(/(SUCCESS|SKIPPED): .*? -> (.+)/);
-    if (match) {
-      const [, , path] = match;
+  try {
+    // Check if log file exists
+    if (!fs.existsSync(logFilePath)) {
+      return new Set();
+    }
+
+    // Check if cache is still valid
+    const logStats = fs.statSync(logFilePath);
+    if (processedPathsCache && logStats.mtime.getTime() === lastLogModTime) {
+      return processedPathsCache;
+    }
+
+    // Read and parse log file
+    const processed = new Set();
+    const content = fs.readFileSync(logFilePath, 'utf-8');
+
+    // Use more efficient regex with global flag
+    const regex = /(SUCCESS|SKIPPED): .*? -> (.+)/g;
+    let match;
+
+    while ((match = regex.exec(content)) !== null) {
+      const path = match[2];
       if (path) {
         processed.add(path.trim());
       }
     }
-  }
 
-  return processed;
+    // Update cache
+    processedPathsCache = processed;
+    lastLogModTime = logStats.mtime.getTime();
+
+    return processed;
+  } catch (error) {
+    console.error(`⚠️ Error reading processed paths: ${error.message}`);
+    return new Set();
+  }
 };
 
 /**
@@ -314,6 +440,7 @@ const uploadToApi = async (files, options) => {
 
   // Nueva funcionalidad: estructura de carpetas personalizada
   let combinedStructure = null;
+  let cachedDetection = null; // Cache detection result to avoid redundant calls
 
   if (
     options.folderStructure &&
@@ -322,12 +449,10 @@ const uploadToApi = async (files, options) => {
   ) {
     // Combine custom folder structure with auto-detection
     const firstFile = files[0];
-    const detection = extractYearAndPedimentoFromPath(
-      firstFile.path,
-      process.cwd(),
-    );
-    if (detection.detected) {
-      const autoStructure = `${detection.year}/${detection.pedimento}`;
+    cachedDetection = getCachedPathDetection(firstFile.path, process.cwd());
+
+    if (cachedDetection.detected) {
+      const autoStructure = `${cachedDetection.year}/${cachedDetection.pedimento}`;
       combinedStructure = `${options.folderStructure}/${autoStructure}`;
       formData.append('folderStructure', combinedStructure);
       console.log(
@@ -346,12 +471,10 @@ const uploadToApi = async (files, options) => {
   } else if (options.autoDetectStructure && files.length > 0) {
     // Try to auto-detect from the first file if no explicit structure is provided
     const firstFile = files[0];
-    const detection = extractYearAndPedimentoFromPath(
-      firstFile.path,
-      process.cwd(),
-    );
-    if (detection.detected) {
-      const autoStructure = `${detection.year}/${detection.pedimento}`;
+    cachedDetection = getCachedPathDetection(firstFile.path, process.cwd());
+
+    if (cachedDetection.detected) {
+      const autoStructure = `${cachedDetection.year}/${cachedDetection.pedimento}`;
       formData.append('folderStructure', autoStructure);
     }
   }
@@ -413,16 +536,19 @@ const uploadToSupabase = async (file, uploadPath) => {
  */
 const insertStatsToUploaderTable = async (files, options) => {
   if (!supabase) {
-    throw new Error('Supabase client not initialized. Stats mode requires Supabase connection.');
+    throw new Error(
+      'Supabase client not initialized. Stats mode requires Supabase connection.',
+    );
   }
 
   const detectionService = new FileDetectionService();
   const records = [];
 
   for (const file of files) {
-    const stats = fs.statSync(file.path);
+    // OPTIMIZED: Use pre-computed stats if available, otherwise call fs.statSync
+    const stats = file.stats || fs.statSync(file.path);
     const originalPath = options.clientPath || file.path;
-    
+
     // Check if record already exists
     const { data: existingRecords, error: checkError } = await supabase
       .from('uploader')
@@ -431,7 +557,9 @@ const insertStatsToUploaderTable = async (files, options) => {
       .limit(1);
 
     if (checkError) {
-      console.error(`❌ Error checking for existing record: ${checkError.message}`);
+      console.error(
+        `❌ Error checking for existing record: ${checkError.message}`,
+      );
       continue;
     }
 
@@ -439,7 +567,7 @@ const insertStatsToUploaderTable = async (files, options) => {
       console.log(`⏭️ Skipping duplicate: ${path.basename(file.path)}`);
       continue;
     }
-    
+
     // Initialize record with basic file stats
     const record = {
       document_type: null,
@@ -450,26 +578,28 @@ const insertStatsToUploaderTable = async (files, options) => {
       arela_path: null,
       status: 'stats',
       rfc: null,
-      message: null
+      message: null,
     };
 
     // Try to detect document type for supported files
     if (detectionService.isSupportedFileType(file.path)) {
       try {
         const detection = await detectionService.detectFile(file.path);
-        
+
         if (detection.detectedType) {
           record.document_type = detection.detectedType;
           record.num_pedimento = detection.detectedPedimento;
           record.status = 'detected';
-          
+
           // Set arela_path for pedimento_simplificado documents
           if (detection.arelaPath) {
             record.arela_path = detection.arelaPath;
           }
-          
+
           // Extract RFC from fields if available
-          const rfcField = detection.fields.find(f => f.name === 'rfc' && f.found);
+          const rfcField = detection.fields.find(
+            (f) => f.name === 'rfc' && f.found,
+          );
           if (rfcField) {
             record.rfc = rfcField.value;
           }
@@ -497,8 +627,10 @@ const insertStatsToUploaderTable = async (files, options) => {
     return [];
   }
 
-  console.log(`💾 Inserting ${records.length} new records into uploader table...`);
-  
+  console.log(
+    `💾 Inserting ${records.length} new records into uploader table...`,
+  );
+
   const { data, error } = await supabase
     .from('uploader')
     .insert(records)
@@ -511,6 +643,320 @@ const insertStatsToUploaderTable = async (files, options) => {
   return data;
 };
 
+/**
+ * OPTIMIZED: Insert ONLY file stats into uploader table (Phase 1)
+ * No file reading, no detection - just filesystem metadata
+ * Returns summary statistics instead of full records for better performance
+ */
+const insertStatsOnlyToUploaderTable = async (files, options) => {
+  if (!supabase) {
+    throw new Error(
+      'Supabase client not initialized. Stats mode requires Supabase connection.',
+    );
+  }
+
+  const batchSize = 1000; // Large batch size for performance
+  const allRecords = [];
+
+  // Prepare all file stats data first - OPTIMIZED to use pre-computed stats
+  console.log('📊 Collecting filesystem stats...');
+  for (const file of files) {
+    try {
+      // Use pre-computed stats if available, otherwise call fs.statSync
+      const stats = file.stats || fs.statSync(file.path);
+      const originalPath = options.clientPath || file.path;
+      const fileExtension = path
+        .extname(file.path)
+        .toLowerCase()
+        .replace('.', '');
+
+      const record = {
+        document_type: null,
+        size: stats.size,
+        num_pedimento: null,
+        filename: file.originalName || path.basename(file.path),
+        original_path: originalPath,
+        arela_path: null,
+        status: 'fs-stats',
+        rfc: null,
+        message: null,
+        file_extension: fileExtension,
+        created_at: new Date().toISOString(),
+        modified_at: stats.mtime.toISOString(),
+      };
+
+      allRecords.push(record);
+    } catch (error) {
+      console.error(`❌ Error reading stats for ${file.path}:`, error.message);
+    }
+  }
+
+  if (allRecords.length === 0) {
+    console.log('📝 No file stats to insert');
+    return { totalInserted: 0, totalSkipped: 0, totalProcessed: 0 };
+  }
+
+  console.log(
+    `💾 Bulk inserting ${allRecords.length} file stats in batches of ${batchSize}...`,
+  );
+
+  let totalInserted = 0;
+  let totalSkipped = 0;
+
+  // Process in batches for optimal performance
+  for (let i = 0; i < allRecords.length; i += batchSize) {
+    const batch = allRecords.slice(i, i + batchSize);
+
+    try {
+      // OPTIMIZED: Use upsert without select to avoid unnecessary data transfer
+      const { error, count } = await supabase.from('uploader').upsert(batch, {
+        onConflict: 'original_path',
+        ignoreDuplicates: false,
+        count: 'exact',
+      });
+
+      if (error) {
+        console.error(
+          `❌ Error inserting batch ${Math.floor(i / batchSize) + 1}:`,
+          error.message,
+        );
+        continue;
+      }
+
+      // For upsert operations, we can't easily distinguish between inserts and updates
+      // from the count alone, but we can estimate based on the assumption that most
+      // operations in --stats-only mode are likely new inserts
+      const batchProcessed = batch.length;
+
+      // Since we're using upsert with ignoreDuplicates: false, the count represents
+      // the actual number of rows affected (both inserts and updates)
+      const affected = count || batchProcessed;
+
+      // For simplicity and performance, we'll assume most are new inserts in stats-only mode
+      // This is reasonable since stats-only is typically run on new file sets
+      totalInserted += affected;
+
+      console.log(
+        `✅ Batch ${Math.floor(i / batchSize) + 1}: ${affected} rows processed`,
+      );
+    } catch (error) {
+      console.error(
+        `❌ Unexpected error in batch ${Math.floor(i / batchSize) + 1}:`,
+        error.message,
+      );
+    }
+  }
+
+  // Calculate skipped as difference between total records and inserted
+  totalSkipped = allRecords.length - totalInserted;
+
+  console.log(
+    `📊 Phase 1 Summary: ${totalInserted} records processed, estimated ${totalSkipped} were updates`,
+  );
+
+  return {
+    totalInserted,
+    totalSkipped,
+    totalProcessed: allRecords.length,
+  };
+};
+
+/**
+ * PHASE 2: Process PDF files for pedimento-simplificado detection
+ * Only processes files with status 'fs-stats' and file_extension 'pdf'
+ * Processes records in chunks of 1000 to avoid loading all records into memory
+ */
+const detectPedimentosInDatabase = async (options = {}) => {
+  if (!supabase) {
+    throw new Error('Supabase client not initialized.');
+  }
+
+  console.log(
+    '🔍 Phase 2: Starting PDF detection for pedimento-simplificado documents...',
+  );
+
+  const detectionService = new FileDetectionService();
+  const processingBatchSize = parseInt(options.batchSize) || 10; // Smaller batches for file I/O
+  const queryBatchSize = 1000; // Process 1000 records at a time
+
+  let totalDetected = 0;
+  let totalProcessed = 0;
+  let totalErrors = 0;
+  let offset = 0;
+  let chunkNumber = 1;
+
+  console.log('� Processing PDF files in chunks of 1000 records...');
+
+  // Process records in chunks of 1000
+  while (true) {
+    console.log(
+      `\n📥 Fetching chunk ${chunkNumber} (records ${offset + 1} to ${offset + queryBatchSize})...`,
+    );
+
+    // Fetch next chunk of PDF records
+    const { data: pdfRecords, error: queryError } = await supabase
+      .from('uploader')
+      .select('id, original_path, filename, file_extension, status')
+      .eq('status', 'fs-stats')
+      .eq('file_extension', 'pdf')
+      .ilike('filename', '%simp%')
+      .range(offset, offset + queryBatchSize - 1);
+
+    if (queryError) {
+      throw new Error(
+        `Failed to fetch PDF records chunk ${chunkNumber}: ${queryError.message}`,
+      );
+    }
+
+    // If no records found, we're done
+    if (!pdfRecords || pdfRecords.length === 0) {
+      console.log(`📝 No more PDF files found. Processing completed.`);
+      break;
+    }
+
+    console.log(
+      `� Processing chunk ${chunkNumber}: ${pdfRecords.length} PDF records`,
+    );
+
+    // Create progress bar for this chunk
+    const progressBar = new cliProgress.SingleBar({
+      format: `🔍 Chunk ${chunkNumber} |{bar}| {percentage}% | {value}/{total} | Detected: {detected} | Errors: {errors}`,
+      barCompleteChar: '█',
+      barIncompleteChar: '░',
+      hideCursor: true,
+    });
+
+    progressBar.start(pdfRecords.length, 0, { detected: 0, errors: 0 });
+
+    let chunkDetected = 0;
+    let chunkProcessed = 0;
+    let chunkErrors = 0;
+
+    // Process files in smaller batches within this chunk
+    for (let i = 0; i < pdfRecords.length; i += processingBatchSize) {
+      const batch = pdfRecords.slice(i, i + processingBatchSize);
+      const updatePromises = [];
+
+      for (const record of batch) {
+        try {
+          // Check if file still exists
+          if (!fs.existsSync(record.original_path)) {
+            updatePromises.push(
+              supabase
+                .from('uploader')
+                .update({
+                  status: 'file-not-found',
+                  message: 'File no longer exists at original path',
+                })
+                .eq('id', record.id),
+            );
+            chunkErrors++;
+            totalErrors++;
+            continue;
+          }
+
+          // Perform detection
+          const detection = await detectionService.detectFile(
+            record.original_path,
+          );
+          chunkProcessed++;
+          totalProcessed++;
+
+          const updateData = {
+            status: detection.detectedType ? 'detected' : 'not-detected',
+            document_type: detection.detectedType,
+            num_pedimento: detection.detectedPedimento,
+            arela_path: detection.arelaPath,
+            message: detection.error || null,
+          };
+
+          // Extract RFC from fields if available
+          if (detection.fields) {
+            const rfcField = detection.fields.find(
+              (f) => f.name === 'rfc' && f.found,
+            );
+            if (rfcField) {
+              updateData.rfc = rfcField.value;
+            }
+          }
+
+          if (detection.detectedType) {
+            chunkDetected++;
+            totalDetected++;
+          }
+
+          updatePromises.push(
+            supabase.from('uploader').update(updateData).eq('id', record.id),
+          );
+        } catch (error) {
+          console.error(
+            `❌ Error detecting ${record.filename}:`,
+            error.message,
+          );
+          chunkErrors++;
+          totalErrors++;
+
+          updatePromises.push(
+            supabase
+              .from('uploader')
+              .update({
+                status: 'detection-error',
+                message: error.message,
+              })
+              .eq('id', record.id),
+          );
+        }
+      }
+
+      // Execute all updates in parallel for this batch
+      try {
+        await Promise.all(updatePromises);
+      } catch (error) {
+        console.error(
+          `❌ Error updating batch in chunk ${chunkNumber}:`,
+          error.message,
+        );
+      }
+
+      // Update progress for this chunk
+      progressBar.update(Math.min(i + processingBatchSize, pdfRecords.length), {
+        detected: chunkDetected,
+        errors: chunkErrors,
+      });
+    }
+
+    progressBar.stop();
+
+    console.log(
+      `✅ Chunk ${chunkNumber} completed: ${chunkDetected} detected, ${chunkProcessed} processed, ${chunkErrors} errors`,
+    );
+
+    // Move to next chunk
+    offset += queryBatchSize;
+    chunkNumber++;
+
+    // If we got fewer records than queryBatchSize, we've reached the end
+    if (pdfRecords.length < queryBatchSize) {
+      console.log(
+        `📝 Reached end of records (chunk had ${pdfRecords.length} records).`,
+      );
+      break;
+    }
+
+    // Small delay between chunks to avoid overwhelming the database
+    await new Promise((resolve) => setTimeout(resolve, 500));
+  }
+
+  console.log(
+    `📊 Phase 2 Summary: ${totalDetected} detected, ${totalProcessed} processed, ${totalErrors} errors`,
+  );
+  return {
+    detectedCount: totalDetected,
+    processedCount: totalProcessed,
+    errorCount: totalErrors,
+  };
+};
+
 const processFilesInBatches = async (
   files,
   batchSize,
@@ -528,7 +974,7 @@ const processFilesInBatches = async (
 
   const messageBuffer = [];
 
-  const progressBarFormat = options.statsOnly 
+  const progressBarFormat = options.statsOnly
     ? '📊 Processing [{bar}] {percentage}% | {value}/{total} files | Stats: {successCount} | Errors: {failureCount} | Duplicates: {skippedCount}'
     : '📂 Processing [{bar}] {percentage}% | {value}/{total} files | Success: {successCount} | Errors: {failureCount} | Skipped: {skippedCount}';
 
@@ -546,98 +992,67 @@ const processFilesInBatches = async (
   });
 
   if (options.statsOnly) {
-    // Stats-only mode - Read file stats and insert to uploader table
-    console.log('📊 Processing files in stats-only mode...');
-    
-    let totalDetected = 0;
-    let totalNotDetected = 0;
-    let totalUnsupported = 0;
-    let totalDetectionErrors = 0;
-    
+    // OPTIMIZED Stats-only mode - Only read filesystem stats, no file detection
+    console.log(
+      '📊 Phase 1: Processing files in optimized stats-only mode (no detection)...',
+    );
+
     for (let i = 0; i < files.length; i += batchSize) {
       const batch = files.slice(i, i + batchSize);
-      
-      const statsFiles = batch.map((file) => {
-        const originalFileName = path.basename(file);
-        
-        return {
-          path: file,
-          originalName: originalFileName,
-        };
-      });
 
-      try {
-        const insertedRecords = await insertStatsToUploaderTable(statsFiles, options);
-        const actualInserted = insertedRecords.length;
-        const skippedDuplicates = statsFiles.length - actualInserted;
-        
-        totalUploaded += actualInserted;
-        totalSkipped += skippedDuplicates;
-        
-        // Count detection results from inserted records
-        insertedRecords.forEach(record => {
-          switch (record.status) {
-            case 'detected':
-              totalDetected++;
-              break;
-            case 'not-detected':
-              totalNotDetected++;
-              break;
-            case 'unsupported':
-              totalUnsupported++;
-              break;
-            case 'detection-error':
-              totalDetectionErrors++;
-              break;
-          }
-        });
-        
-        statsFiles.forEach((file) => {
-          const wasInserted = insertedRecords.some(record => 
-            record.original_path === (options.clientPath || file.path)
-          );
-          if (wasInserted) {
-            writeLog(`STATS: ${file.path} -> uploader table`);
-          } else {
-            writeLog(`DUPLICATE: ${file.path} -> already exists in uploader table`);
-          }
+      // OPTIMIZED: Batch read file stats to reduce I/O overhead
+      const fileStatsResults = batchReadFileStats(batch);
+      const statsFiles = fileStatsResults
+        .filter((result) => result.stats !== null) // Only include files with valid stats
+        .map((result) => {
+          const originalFileName = path.basename(result.path);
+
+          return {
+            path: result.path,
+            originalName: originalFileName,
+            stats: result.stats, // Pass pre-computed stats to avoid redundant calls
+          };
         });
-        
-        if (actualInserted > 0) {
-          console.log(`📈 Inserted ${actualInserted} stats records`);
-        }
-        if (skippedDuplicates > 0) {
-          console.log(`⏭️ Skipped ${skippedDuplicates} duplicates`);
-        }
-        if (options.detect !== false) {
-          console.log(`   🔍 Detected: ${totalDetected}, Not detected: ${totalNotDetected}, Unsupported: ${totalUnsupported}, Errors: ${totalDetectionErrors}`);
-        }
-        
-      } catch (error) {
-        totalErrors += statsFiles.length;
-        statsFiles.forEach((file) => {
-          writeLog(`ERROR: ${file.path}: ${error.message}`);
-          messageBuffer.push(`❌ ${file.originalName}: ${error.message}`);
+
+      // Log any files that couldn't be read
+      const failedFiles = fileStatsResults.filter(
+        (result) => result.error !== null,
+      );
+      if (failedFiles.length > 0) {
+        console.log(
+          `⚠️ Could not read stats for ${failedFiles.length} files in batch`,
+        );
+        failedFiles.forEach((failed) => {
+          console.error(`   ❌ ${failed.path}: ${failed.error}`);
         });
       }
 
-      progressBar.update(i + batch.length, {
-        successCount: totalUploaded,
-        failureCount: totalErrors,
-        skippedCount: totalSkipped,
-      });
+      try {
+        const result = await insertStatsOnlyToUploaderTable(
+          statsFiles,
+          options,
+        );
 
-      if (i + batchSize < files.length) {
-        await new Promise((resolve) => setTimeout(resolve, 200));
+        totalUploaded += result.totalInserted;
+        totalSkipped += result.totalSkipped;
+        totalErrors += failedFiles.length; // Count failed file reads as errors
+
+        progressBar.update(Math.min(i + batch.length, files.length), {
+          successCount: totalUploaded,
+          failureCount: totalErrors,
+          skippedCount: totalSkipped,
+        });
+      } catch (error) {
+        console.error(`❌ Error processing stats batch:`, error.message);
+        totalErrors += batch.length;
+
+        progressBar.update(Math.min(i + batch.length, files.length), {
+          successCount: totalUploaded,
+          failureCount: totalErrors,
+          skippedCount: totalSkipped,
+        });
       }
     }
-    
-    // Store detection stats for summary
-    totalDetected = totalDetected || 0;
-    totalNotDetected = totalNotDetected || 0;
-    totalUnsupported = totalUnsupported || 0;
-    totalDetectionErrors = totalDetectionErrors || 0;
-    
   } else if (apiMode && !options.forceSupabase) {
     // API Mode - Process in batches
     for (let i = 0; i < files.length; i += batchSize) {
@@ -661,7 +1076,8 @@ const processFilesInBatches = async (
 
           // Handle combined folder structure + auto-detection
           if (options.folderStructure && options.autoDetectStructure) {
-            const detection = extractYearAndPedimentoFromPath(file, basePath);
+            // OPTIMIZED: Use cached detection to avoid redundant parsing
+            const detection = getCachedPathDetection(file, basePath);
             if (detection.detected) {
               const autoStructure = `${detection.year}/${detection.pedimento}`;
               const combinedStructure = `${options.folderStructure}/${autoStructure}`;
@@ -669,7 +1085,7 @@ const processFilesInBatches = async (
                 combinedStructure,
                 sanitizedFileName,
               );
-              console.log(
+              logVerbose(
                 `📁 Combined structure: ${options.folderStructure}/${autoStructure} for ${originalFileName} -> ${uploadPath}`,
               );
             } else {
@@ -678,7 +1094,7 @@ const processFilesInBatches = async (
                 options.folderStructure,
                 sanitizedFileName,
               );
-              console.log(
+              logVerbose(
                 `📁 Custom structure (auto-detection failed): ${uploadPath}`,
               );
             }
@@ -688,10 +1104,10 @@ const processFilesInBatches = async (
               options.folderStructure,
               sanitizedFileName,
             );
-            console.log(`📁 Custom structure: ${uploadPath}`);
+            logVerbose(`📁 Custom structure: ${uploadPath}`);
           } else if (options.autoDetectStructure) {
-            // Auto-detect structure from path if enabled
-            const detection = extractYearAndPedimentoFromPath(file, basePath);
+            // Auto-detect structure from path if enabled - OPTIMIZED: Use cached detection
+            const detection = getCachedPathDetection(file, basePath);
             if (detection.detected) {
               const autoStructure = `${detection.year}/${detection.pedimento}`;
               uploadPath = path.posix.join(autoStructure, sanitizedFileName);
@@ -737,10 +1153,8 @@ const processFilesInBatches = async (
 
           if (!clientPath && apiFiles.length > 0) {
             const firstFile = apiFiles[0];
-            const detection = extractYearAndPedimentoFromPath(
-              firstFile.path,
-              basePath,
-            );
+            // OPTIMIZED: Use cached detection to avoid redundant parsing
+            const detection = getCachedPathDetection(firstFile.path, basePath);
             if (detection.detected) {
               // clientPath = `${detection.year}/${detection.pedimento}/`;
               clientPath = path
@@ -796,7 +1210,7 @@ const processFilesInBatches = async (
       });
 
       if (i + batchSize < files.length) {
-        await new Promise((resolve) => setTimeout(resolve, 200));
+        await new Promise((resolve) => setTimeout(resolve, BATCH_DELAY));
       }
     }
   } else {
@@ -809,7 +1223,7 @@ const processFilesInBatches = async (
 
         // Handle combined folder structure + auto-detection
         if (options.folderStructure && options.autoDetectStructure) {
-          const detection = extractYearAndPedimentoFromPath(file, basePath);
+          const detection = getCachedPathDetection(file, basePath);
           if (detection.detected) {
             const autoStructure = `${detection.year}/${detection.pedimento}`;
             const combinedStructure = `${options.folderStructure}/${autoStructure}`;
@@ -832,8 +1246,8 @@ const processFilesInBatches = async (
           uploadPath = path.join(options.folderStructure, fileName);
           console.log(`📁 Custom structure: ${uploadPath}`);
         } else if (options.autoDetectStructure) {
-          // Auto-detect structure from path if enabled
-          const detection = extractYearAndPedimentoFromPath(file, basePath);
+          // Auto-detect structure from path if enabled - OPTIMIZED: Use cached detection
+          const detection = getCachedPathDetection(file, basePath);
           if (detection.detected) {
             const autoStructure = `${detection.year}/${detection.pedimento}`;
             const fileName = path.basename(file);
@@ -899,13 +1313,19 @@ const uploadFilesByRfc = async (options = {}) => {
   }
 
   if (!API_BASE_URL || !API_TOKEN) {
-    console.error('❌ Arela API configuration missing. Please set ARELA_API_URL and ARELA_API_TOKEN environment variables.');
+    console.error(
+      '❌ Arela API configuration missing. Please set ARELA_API_URL and ARELA_API_TOKEN environment variables.',
+    );
     process.exit(1);
   }
 
   if (!uploadRfcs || uploadRfcs.length === 0) {
-    console.error('❌ No RFCs specified. Please set UPLOAD_RFCS environment variable with pipe-separated RFC values.');
-    console.error('   Example: UPLOAD_RFCS="RFC123456789|RFC987654321|RFC555444333"');
+    console.error(
+      '❌ No RFCs specified. Please set UPLOAD_RFCS environment variable with pipe-separated RFC values.',
+    );
+    console.error(
+      '   Example: UPLOAD_RFCS="RFC123456789|RFC987654321|RFC555444333"',
+    );
     process.exit(1);
   }
 
@@ -927,13 +1347,17 @@ const uploadFilesByRfc = async (options = {}) => {
 
   if (!rfcRecords || rfcRecords.length === 0) {
     console.log('ℹ️  No files found for the specified RFCs with arela_path');
-    console.log(`   Make sure files for RFCs [${uploadRfcs.join(', ')}] have been processed and have arela_path values`);
+    console.log(
+      `   Make sure files for RFCs [${uploadRfcs.join(', ')}] have been processed and have arela_path values`,
+    );
     return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
   }
 
   // Step 2: Get unique arela_paths from the RFC matches
-  const uniqueArelaPaths = [...new Set(rfcRecords.map(r => r.arela_path))];
-  console.log(`� Found ${uniqueArelaPaths.length} unique arela_path(s) for the specified RFCs`);
+  const uniqueArelaPaths = [...new Set(rfcRecords.map((r) => r.arela_path))];
+  console.log(
+    `� Found ${uniqueArelaPaths.length} unique arela_path(s) for the specified RFCs`,
+  );
 
   // Step 3: Get ALL files that have these arela_paths (including supporting documents)
   // Use pagination to ensure we get all files, regardless of count
@@ -943,7 +1367,7 @@ const uploadFilesByRfc = async (options = {}) => {
   const queryBatchSize = 1000;
 
   console.log('📥 Fetching all related files (with pagination)...');
-  
+
   while (hasMore) {
     const { data: batch, error: queryError } = await supabase
       .from('uploader')
@@ -962,7 +1386,7 @@ const uploadFilesByRfc = async (options = {}) => {
     } else {
       allRelatedFiles = allRelatedFiles.concat(batch);
       offset += queryBatchSize;
-      
+
       // If we got less than queryBatchSize, we've reached the end
       if (batch.length < queryBatchSize) {
         hasMore = false;
@@ -975,8 +1399,10 @@ const uploadFilesByRfc = async (options = {}) => {
     return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
   }
 
-  console.log(`📁 Found ${allRelatedFiles.length} total files to upload (including supporting documents)`);
-  
+  console.log(
+    `📁 Found ${allRelatedFiles.length} total files to upload (including supporting documents)`,
+  );
+
   // Group by RFC and arela_path for better organization
   const filesByRfc = allRelatedFiles.reduce((acc, record) => {
     const rfc = record.rfc || 'No RFC';
@@ -989,8 +1415,12 @@ const uploadFilesByRfc = async (options = {}) => {
 
   console.log('📊 Files by RFC (including supporting documents):');
   for (const [rfc, files] of Object.entries(filesByRfc)) {
-    const documentTypes = [...new Set(files.map(f => f.document_type || 'Unknown'))];
-    console.log(`   ${rfc}: ${files.length} files (${documentTypes.join(', ')})`);
+    const documentTypes = [
+      ...new Set(files.map((f) => f.document_type || 'Unknown')),
+    ];
+    console.log(
+      `   ${rfc}: ${files.length} files (${documentTypes.join(', ')})`,
+    );
   }
 
   // Group by arela_path for upload organization
@@ -1015,7 +1445,8 @@ const uploadFilesByRfc = async (options = {}) => {
 
   // Create progress bar
   const progressBar = new cliProgress.SingleBar({
-    format: '🚀 Uploading files |{bar}| {percentage}% | {value}/{total} | Uploaded: {uploaded} | Errors: {errors} | Skipped: {skipped}',
+    format:
+      '🚀 Uploading files |{bar}| {percentage}% | {value}/{total} | Uploaded: {uploaded} | Errors: {errors} | Skipped: {skipped}',
     barCompleteChar: '█',
     barIncompleteChar: '░',
     hideCursor: true,
@@ -1037,18 +1468,20 @@ const uploadFilesByRfc = async (options = {}) => {
     const batch = allRelatedFiles.slice(i, i + batchSize);
     const batchNumber = Math.floor(i / batchSize) + 1;
     const totalBatches = Math.ceil(allRelatedFiles.length / batchSize);
-    
-    console.log(`\n📦 Processing batch ${batchNumber}/${totalBatches} (${batch.length} files)`);
+
+    console.log(
+      `\n📦 Processing batch ${batchNumber}/${totalBatches} (${batch.length} files)`,
+    );
 
     // Prepare files for upload
     const filesToUpload = [];
-    
+
     for (const record of batch) {
       totalProcessed++;
-      
+
       try {
         const originalPath = record.original_path;
-        
+
         // Check if file exists
         if (!fs.existsSync(originalPath)) {
           console.log(`   ⚠️  File not found: ${originalPath}`);
@@ -1056,24 +1489,26 @@ const uploadFilesByRfc = async (options = {}) => {
           continue;
         }
 
-        const fileStats = fs.statSync(originalPath);
+        // OPTIMIZED: Read file and get size from buffer instead of separate fs.statSync call
         const fileBuffer = fs.readFileSync(originalPath);
-        
+
         filesToUpload.push({
           path: originalPath,
           buffer: fileBuffer,
-          size: fileStats.size,
+          size: fileBuffer.length, // Get size from buffer instead of fs.statSync
           name: record.filename,
           arelaPath: record.arela_path,
           rfc: record.rfc,
           documentType: record.document_type,
         });
-        
       } catch (error) {
-        console.error(`   ❌ Error reading file ${record.original_path}:`, error.message);
+        console.error(
+          `   ❌ Error reading file ${record.original_path}:`,
+          error.message,
+        );
         totalErrors++;
       }
-      
+
       if (options.showProgress !== false) {
         progressBar.update(totalProcessed, {
           uploaded: totalUploaded,
@@ -1086,10 +1521,12 @@ const uploadFilesByRfc = async (options = {}) => {
     // Upload the batch if we have files
     if (filesToUpload.length > 0) {
       try {
-        console.log(`   🚀 Uploading ${filesToUpload.length} files to Arela API...`);
-        
+        console.log(
+          `   🚀 Uploading ${filesToUpload.length} files to Arela API...`,
+        );
+
         const formData = new FormData();
-        
+
         // Add files to form data
         filesToUpload.forEach((file, index) => {
           formData.append(`files`, file.buffer, {
@@ -1112,7 +1549,7 @@ const uploadFilesByRfc = async (options = {}) => {
         // Upload each group separately with its folder structure
         for (const [arelaPath, pathFiles] of Object.entries(filesByPath)) {
           const pathFormData = new FormData();
-          
+
           pathFiles.forEach((file) => {
             pathFormData.append('files', file.buffer, {
               filename: file.name,
@@ -1121,8 +1558,10 @@ const uploadFilesByRfc = async (options = {}) => {
           });
 
           // Set folder structure for this group - concatenate custom prefix with arela_path
-          const folderStructure = options.folderStructure 
-            ? `${options.folderStructure}/${arelaPath}`.replace(/\/+/g, '/').replace(/\/$/, '')
+          const folderStructure = options.folderStructure
+            ? `${options.folderStructure}/${arelaPath}`
+                .replace(/\/+/g, '/')
+                .replace(/\/$/, '')
             : arelaPath;
           pathFormData.append('folderStructure', folderStructure);
           pathFormData.append('autoDetect', 'true');
@@ -1133,15 +1572,20 @@ const uploadFilesByRfc = async (options = {}) => {
             pathFormData.append('bucket', bucket);
           }
 
-          console.log(`     📁 Uploading ${pathFiles.length} files to: ${folderStructure}`);
+          console.log(
+            `     📁 Uploading ${pathFiles.length} files to: ${folderStructure}`,
+          );
 
-          const response = await fetch(`${API_BASE_URL}/api/storage/batch-upload-and-process`, {
-            method: 'POST',
-            headers: {
-              'x-api-key': API_TOKEN,
+          const response = await fetch(
+            `${API_BASE_URL}/api/storage/batch-upload-and-process`,
+            {
+              method: 'POST',
+              headers: {
+                'x-api-key': API_TOKEN,
+              },
+              body: pathFormData,
             },
-            body: pathFormData,
-          });
+          );
 
           if (!response.ok) {
             const errorText = await response.text();
@@ -1149,24 +1593,33 @@ const uploadFilesByRfc = async (options = {}) => {
           }
 
           const result = await response.json();
-          
+
           // Check if upload was successful based on stats rather than success field
-          const isSuccessful = result.stats && result.stats.uploadedCount > 0 && result.stats.errorCount === 0;
-          
+          const isSuccessful =
+            result.stats &&
+            result.stats.uploadedCount > 0 &&
+            result.stats.errorCount === 0;
+
           if (isSuccessful) {
-            console.log(`     ✅ Group uploaded: ${result.stats.uploadedCount} files to ${folderStructure}`);
+            console.log(
+              `     ✅ Group uploaded: ${result.stats.uploadedCount} files to ${folderStructure}`,
+            );
             totalUploaded += result.stats.uploadedCount;
-            
+
             if (result.stats.detectedCount > 0) {
-              console.log(`     🔍 Files detected: ${result.stats.detectedCount}`);
+              console.log(
+                `     🔍 Files detected: ${result.stats.detectedCount}`,
+              );
             }
             if (result.stats.organizedCount > 0) {
-              console.log(`     📁 Files organized: ${result.stats.organizedCount}`);
+              console.log(
+                `     📁 Files organized: ${result.stats.organizedCount}`,
+              );
             }
           } else {
             console.error(`     ❌ Upload failed for ${folderStructure}:`);
             if (result.errors && result.errors.length > 0) {
-              result.errors.forEach(error => {
+              result.errors.forEach((error) => {
                 console.error(`        - ${error.fileName}: ${error.error}`);
               });
             }
@@ -1174,18 +1627,20 @@ const uploadFilesByRfc = async (options = {}) => {
           }
 
           // Small delay between path groups
-          await new Promise(resolve => setTimeout(resolve, 100));
+          await new Promise((resolve) => setTimeout(resolve, 100));
         }
-
       } catch (error) {
-        console.error(`   ❌ Error uploading batch ${batchNumber}:`, error.message);
+        console.error(
+          `   ❌ Error uploading batch ${batchNumber}:`,
+          error.message,
+        );
         totalErrors += filesToUpload.length;
       }
     }
 
     // Small delay between batches
     if (i + batchSize < allRelatedFiles.length) {
-      await new Promise(resolve => setTimeout(resolve, 200));
+      await new Promise((resolve) => setTimeout(resolve, BATCH_DELAY));
     }
   }
 
@@ -1229,7 +1684,10 @@ const propagateArelaPath = async (options = {}) => {
     .not('arela_path', 'is', null);
 
   if (pedimentoError) {
-    console.error('❌ Error fetching pedimento records:', pedimentoError.message);
+    console.error(
+      '❌ Error fetching pedimento records:',
+      pedimentoError.message,
+    );
     return { processedCount: 0, updatedCount: 0, errorCount: 1 };
   }
 
@@ -1238,7 +1696,9 @@ const propagateArelaPath = async (options = {}) => {
     return { processedCount: 0, updatedCount: 0, errorCount: 0 };
   }
 
-  console.log(`📋 Found ${pedimentoRecords.length} pedimento records with arela_path`);
+  console.log(
+    `📋 Found ${pedimentoRecords.length} pedimento records with arela_path`,
+  );
 
   let totalProcessed = 0;
   let totalUpdated = 0;
@@ -1246,7 +1706,8 @@ const propagateArelaPath = async (options = {}) => {
 
   // Create progress bar
   const progressBar = new cliProgress.SingleBar({
-    format: '🔄 Propagating paths |{bar}| {percentage}% | {value}/{total} | Updated: {updated} | Errors: {errors}',
+    format:
+      '🔄 Propagating paths |{bar}| {percentage}% | {value}/{total} | Updated: {updated} | Errors: {errors}',
     barCompleteChar: '█',
     barIncompleteChar: '░',
     hideCursor: true,
@@ -1263,19 +1724,21 @@ const propagateArelaPath = async (options = {}) => {
   for (const pedimento of pedimentoRecords) {
     try {
       totalProcessed++;
-      
+
       // Extract base path from original_path (remove filename)
       const basePath = path.dirname(pedimento.original_path);
-      
+
       console.log(`\n🔍 Processing: ${pedimento.filename}`);
       console.log(`   📁 Base path: ${basePath}`);
-      
+
       // Extract folder part from existing arela_path by removing the filename
       const existingPath = pedimento.arela_path;
-      const folderArelaPath = existingPath.includes('/') ? 
-        existingPath.substring(0, existingPath.lastIndexOf('/')) + '/' : 
-        existingPath.endsWith('/') ? existingPath : existingPath + '/';
-        
+      const folderArelaPath = existingPath.includes('/')
+        ? existingPath.substring(0, existingPath.lastIndexOf('/')) + '/'
+        : existingPath.endsWith('/')
+          ? existingPath
+          : existingPath + '/';
+
       console.log(`   🎯 Original arela path: ${existingPath}`);
       console.log(`   📁 Folder arela path: ${folderArelaPath}`);
 
@@ -1288,7 +1751,10 @@ const propagateArelaPath = async (options = {}) => {
         .neq('id', pedimento.id); // Exclude the pedimento itself
 
       if (relatedError) {
-        console.error(`❌ Error finding related files for ${pedimento.filename}:`, relatedError.message);
+        console.error(
+          `❌ Error finding related files for ${pedimento.filename}:`,
+          relatedError.message,
+        );
         totalErrors++;
         continue;
       }
@@ -1298,32 +1764,38 @@ const propagateArelaPath = async (options = {}) => {
         continue;
       }
 
-      console.log(`   📄 Found ${relatedFiles.length} related files to update:`);
-      
+      console.log(
+        `   📄 Found ${relatedFiles.length} related files to update:`,
+      );
+
       // Show first 10 files, then indicate if there are more
       const filesToShow = relatedFiles.slice(0, 10);
-      filesToShow.forEach(file => {
+      filesToShow.forEach((file) => {
         console.log(`      - ${file.filename}`);
       });
-      
+
       if (relatedFiles.length > 10) {
         console.log(`      ... and ${relatedFiles.length - 10} more files`);
       }
 
       // Process files in batches to avoid URI length limitations
       const BATCH_SIZE = 50; // Process 50 files at a time
-      const fileIds = relatedFiles.map(f => f.id);
+      const fileIds = relatedFiles.map((f) => f.id);
       let batchErrors = 0;
       let batchUpdated = 0;
 
-      console.log(`   🔄 Processing ${relatedFiles.length} files in batches of ${BATCH_SIZE}...`);
+      console.log(
+        `   🔄 Processing ${relatedFiles.length} files in batches of ${BATCH_SIZE}...`,
+      );
 
       for (let i = 0; i < fileIds.length; i += BATCH_SIZE) {
         const batchIds = fileIds.slice(i, i + BATCH_SIZE);
         const batchNumber = Math.floor(i / BATCH_SIZE) + 1;
         const totalBatches = Math.ceil(fileIds.length / BATCH_SIZE);
-        
-        console.log(`   📦 Batch ${batchNumber}/${totalBatches}: Updating ${batchIds.length} files...`);
+
+        console.log(
+          `   📦 Batch ${batchNumber}/${totalBatches}: Updating ${batchIds.length} files...`,
+        );
 
         try {
           const { error: updateError } = await supabase
@@ -1332,33 +1804,45 @@ const propagateArelaPath = async (options = {}) => {
             .in('id', batchIds);
 
           if (updateError) {
-            console.error(`   ❌ Error in batch ${batchNumber}:`, updateError.message);
+            console.error(
+              `   ❌ Error in batch ${batchNumber}:`,
+              updateError.message,
+            );
             batchErrors++;
           } else {
-            console.log(`   ✅ Batch ${batchNumber} completed: ${batchIds.length} files updated`);
+            console.log(
+              `   ✅ Batch ${batchNumber} completed: ${batchIds.length} files updated`,
+            );
             batchUpdated += batchIds.length;
           }
         } catch (error) {
-          console.error(`   ❌ Exception in batch ${batchNumber}:`, error.message);
+          console.error(
+            `   ❌ Exception in batch ${batchNumber}:`,
+            error.message,
+          );
           batchErrors++;
         }
 
         // Small delay between batches to avoid overwhelming the database
         if (i + BATCH_SIZE < fileIds.length) {
-          await new Promise(resolve => setTimeout(resolve, 100));
+          await new Promise((resolve) => setTimeout(resolve, 100));
         }
       }
 
       if (batchErrors > 0) {
-        console.error(`❌ ${batchErrors} batch(es) failed for ${pedimento.filename}`);
+        console.error(
+          `❌ ${batchErrors} batch(es) failed for ${pedimento.filename}`,
+        );
         totalErrors++;
       } else {
         console.log(`   🎯 Successfully updated ${batchUpdated} related files`);
         totalUpdated += batchUpdated;
       }
-
     } catch (error) {
-      console.error(`❌ Error processing ${pedimento.filename}:`, error.message);
+      console.error(
+        `❌ Error processing ${pedimento.filename}:`,
+        error.message,
+      );
       totalErrors++;
     }
 
@@ -1422,25 +1906,70 @@ program
     'Automatically detect year/pedimento from file paths',
   )
   .option('--client-path <path>', 'Client path for metadata tracking')
-  .option('--stats-only', 'Only read file stats and insert to uploader table, skip file upload')
+  .option(
+    '--stats-only',
+    'Phase 1: Only read filesystem stats and insert to database (no file reading or detection)',
+  )
   .option('--no-detect', 'Disable document type detection in stats-only mode')
-  .option('--propagate-arela-path', 'Propagate arela_path from pedimento_simplificado records to related files with same base path')
-  .option('--upload-by-rfc', 'Upload files to Arela API based on RFC values from UPLOAD_RFCS environment variable')
+  .option(
+    '--detect-pdfs',
+    'Phase 2: Process PDF files in database for pedimento-simplificado detection',
+  )
+  .option(
+    '--propagate-arela-path',
+    'Phase 3: Propagate arela_path from pedimento_simplificado records to related files with same base path',
+  )
+  .option(
+    '--upload-by-rfc',
+    'Phase 4: Upload files to Arela API based on RFC values from UPLOAD_RFCS environment variable',
+  )
+  .option(
+    '--run-all-phases',
+    'Run all 4 phases in sequence: stats → detect → propagate → upload',
+  )
   .action(async (options) => {
     if (options.version) {
       console.log(packageVersion);
       process.exit(0);
     }
 
+    // Handle detect-pdfs option (Phase 2)
+    if (options.detectPdfs) {
+      console.log('🔍 Starting Phase 2: PDF Detection');
+      await checkCredentials(true); // Force Supabase mode
+
+      const result = await detectPedimentosInDatabase({
+        batchSize: parseInt(options.batchSize) || 10,
+      });
+
+      console.log(
+        `✅ Phase 2 Complete: ${result.detectedCount} detected, ${result.errorCount} errors`,
+      );
+      return;
+    }
+
+    // Handle run-all-phases option
+    if (options.runAllPhases) {
+      console.log('🚀 Starting all 4 phases in sequence...');
+      await checkCredentials(true); // Force Supabase mode
+
+      // Phase 1: Stats collection
+      console.log('\n📊 === PHASE 1: Filesystem Stats ===');
+      options.statsOnly = true;
+      // Continue with normal processing to run Phase 1
+
+      // The rest will be handled after Phase 1 completes
+    }
+
     // Handle propagate-arela-path option
     if (options.propagateArelaPath) {
       // Initialize Supabase credentials for propagation
       await checkCredentials(true); // Force Supabase mode
-      
+
       const result = await propagateArelaPath({
         showProgress: options.showStats || true,
       });
-      
+
       if (result.errorCount > 0) {
         process.exit(1);
       }
@@ -1451,25 +1980,29 @@ program
     if (options.uploadByRfc) {
       // RFC upload needs both Supabase (for database queries) and API (for uploads)
       await checkCredentials(false); // Initialize API mode
-      
+
       // Also initialize Supabase for database queries
       if (!supabase) {
         if (!supabaseUrl || !supabaseKey) {
-          console.error('❌ RFC upload requires Supabase credentials for database queries.');
-          console.error('   Please set SUPABASE_URL and SUPABASE_KEY environment variables.');
+          console.error(
+            '❌ RFC upload requires Supabase credentials for database queries.',
+          );
+          console.error(
+            '   Please set SUPABASE_URL and SUPABASE_KEY environment variables.',
+          );
           process.exit(1);
         }
-        
+
         supabase = createClient(supabaseUrl, supabaseKey);
         console.log('✅ Connected to Supabase for database queries');
       }
-      
+
       const result = await uploadFilesByRfc({
         showProgress: options.showStats || true,
         batchSize: parseInt(options.batchSize) || 10,
         folderStructure: options.folderStructure,
       });
-      
+
       if (result.errorCount > 0) {
         process.exit(1);
       }
@@ -1490,7 +2023,9 @@ program
     const concurrency = parseInt(options.concurrency) || 10;
 
     if (options.statsOnly) {
-      console.log('📊 Mode: Stats Only - Reading file stats and inserting to uploader table');
+      console.log(
+        '📊 Mode: Stats Only - Reading file stats and inserting to uploader table',
+      );
       console.log('🚫 Files will NOT be uploaded');
       if (options.detect !== false) {
         console.log('🔍 Document type detection ENABLED for supported files');
@@ -1590,12 +2125,73 @@ program
     console.log(`   📜 Log file: ${logFilePath}`);
     console.log(`${'='.repeat(60)}\n`);
 
-    if (options.showStats && sanitizationCache.size > 0) {
+    // Continue with remaining phases if running all phases
+    if (options.runAllPhases && options.statsOnly) {
+      try {
+        // Phase 2: PDF Detection
+        console.log('\n🔍 === PHASE 2: PDF Detection ===');
+        const detectionResult = await detectPedimentosInDatabase({
+          batchSize: parseInt(options.batchSize) || 10,
+        });
+        console.log(
+          `✅ Phase 2 Complete: ${detectionResult.detectedCount} detected, ${detectionResult.errorCount} errors`,
+        );
+
+        // Phase 3: Propagate arela_path
+        console.log('\n📁 === PHASE 3: Propagate Arela Paths ===');
+        const propagateResult = await propagateArelaPath({
+          showProgress: options.showStats || true,
+        });
+        console.log(
+          `✅ Phase 3 Complete: ${propagateResult.updatedCount || 0} paths propagated`,
+        );
+
+        // Phase 4: Upload by RFC
+        if (uploadRfcs && uploadRfcs.length > 0) {
+          console.log('\n🚀 === PHASE 4: Upload by RFC ===');
+
+          // Initialize API mode for uploads
+          await checkCredentials(false);
+
+          const uploadResult = await uploadFilesByRfc({
+            showProgress: options.showStats || true,
+            batchSize: parseInt(options.batchSize) || 10,
+            folderStructure: options.folderStructure,
+          });
+          console.log(`✅ Phase 4 Complete: Upload finished`);
+        } else {
+          console.log('\n⚠️ === PHASE 4: Upload by RFC ===');
+          console.log(
+            '⚠️ UPLOAD_RFCS environment variable not configured, skipping Phase 4',
+          );
+        }
+
+        console.log('\n🎉 All 4 phases completed successfully!');
+      } catch (error) {
+        console.error(`❌ Error in multi-phase execution:`, error.message);
+        process.exit(1);
+      }
+    }
+
+    if (
+      options.showStats &&
+      (sanitizationCache.size > 0 || pathDetectionCache.size > 0)
+    ) {
       console.log(`📊 Performance Statistics:`);
-      console.log(
-        `   🗂️  Sanitization cache entries: ${sanitizationCache.size}`,
-      );
+      if (sanitizationCache.size > 0) {
+        console.log(
+          `   🗂️  Sanitization cache entries: ${sanitizationCache.size}`,
+        );
+      }
+      if (pathDetectionCache.size > 0) {
+        console.log(
+          `   📁  Path detection cache entries: ${pathDetectionCache.size}`,
+        );
+      }
     }
+
+    // OPTIMIZED: Ensure log buffer is flushed before exit
+    flushLogBuffer();
   });
 
 program.parse();