@arela/uploader 0.2.1 → 0.2.3

package/src/index.js

@@ -9,6 +9,7 @@ import { globby } from 'globby';
 import mime from 'mime-types';
 import fetch from 'node-fetch';
 import path from 'path';
+
 import { FileDetectionService } from './file-detection.js';
 
 config();
@@ -209,7 +210,10 @@ const writeLog = (message) => {
 
     // Flush if buffer is full or enough time has passed
     const now = Date.now();
-    if (logBuffer.length >= LOG_BUFFER_SIZE || (now - lastFlushTime) >= LOG_FLUSH_INTERVAL) {
+    if (
+      logBuffer.length >= LOG_BUFFER_SIZE ||
+      now - lastFlushTime >= LOG_FLUSH_INTERVAL
+    ) {
       flushLogBuffer();
     }
   } catch (error) {
@@ -233,7 +237,8 @@ process.on('SIGTERM', () => {
  */
 const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true';
 const BATCH_DELAY = parseInt(process.env.BATCH_DELAY) || 100; // Configurable delay between batches
-const PROGRESS_UPDATE_INTERVAL = parseInt(process.env.PROGRESS_UPDATE_INTERVAL) || 10; // Update progress every N items
+const PROGRESS_UPDATE_INTERVAL =
+  parseInt(process.env.PROGRESS_UPDATE_INTERVAL) || 10; // Update progress every N items
 
 const logVerbose = (message) => {
   if (VERBOSE_LOGGING) {
@@ -531,7 +536,9 @@ const uploadToSupabase = async (file, uploadPath) => {
  */
 const insertStatsToUploaderTable = async (files, options) => {
   if (!supabase) {
-    throw new Error('Supabase client not initialized. Stats mode requires Supabase connection.');
+    throw new Error(
+      'Supabase client not initialized. Stats mode requires Supabase connection.',
+    );
   }
 
   const detectionService = new FileDetectionService();
@@ -550,7 +557,9 @@ const insertStatsToUploaderTable = async (files, options) => {
       .limit(1);
 
     if (checkError) {
-      console.error(`❌ Error checking for existing record: ${checkError.message}`);
+      console.error(
+        `❌ Error checking for existing record: ${checkError.message}`,
+      );
       continue;
     }
 
@@ -569,7 +578,7 @@ const insertStatsToUploaderTable = async (files, options) => {
       arela_path: null,
       status: 'stats',
       rfc: null,
-      message: null
+      message: null,
     };
 
     // Try to detect document type for supported files
@@ -588,7 +597,9 @@ const insertStatsToUploaderTable = async (files, options) => {
           }
 
           // Extract RFC from fields if available
-          const rfcField = detection.fields.find(f => f.name === 'rfc' && f.found);
+          const rfcField = detection.fields.find(
+            (f) => f.name === 'rfc' && f.found,
+          );
           if (rfcField) {
             record.rfc = rfcField.value;
           }
@@ -616,7 +627,9 @@ const insertStatsToUploaderTable = async (files, options) => {
     return [];
   }
 
-  console.log(`💾 Inserting ${records.length} new records into uploader table...`);
+  console.log(
+    `💾 Inserting ${records.length} new records into uploader table...`,
+  );
 
   const { data, error } = await supabase
     .from('uploader')
@@ -637,7 +650,9 @@ const insertStatsToUploaderTable = async (files, options) => {
  */
 const insertStatsOnlyToUploaderTable = async (files, options) => {
   if (!supabase) {
-    throw new Error('Supabase client not initialized. Stats mode requires Supabase connection.');
+    throw new Error(
+      'Supabase client not initialized. Stats mode requires Supabase connection.',
+    );
   }
 
   const batchSize = 1000; // Large batch size for performance
@@ -650,7 +665,10 @@ const insertStatsOnlyToUploaderTable = async (files, options) => {
       // Use pre-computed stats if available, otherwise call fs.statSync
       const stats = file.stats || fs.statSync(file.path);
       const originalPath = options.clientPath || file.path;
-      const fileExtension = path.extname(file.path).toLowerCase().replace('.', '');
+      const fileExtension = path
+        .extname(file.path)
+        .toLowerCase()
+        .replace('.', '');
 
       const record = {
         document_type: null,
@@ -664,7 +682,7 @@ const insertStatsOnlyToUploaderTable = async (files, options) => {
         message: null,
         file_extension: fileExtension,
         created_at: new Date().toISOString(),
-        modified_at: stats.mtime.toISOString()
+        modified_at: stats.mtime.toISOString(),
       };
 
       allRecords.push(record);
@@ -678,7 +696,9 @@ const insertStatsOnlyToUploaderTable = async (files, options) => {
     return { totalInserted: 0, totalSkipped: 0, totalProcessed: 0 };
   }
 
-  console.log(`💾 Bulk inserting ${allRecords.length} file stats in batches of ${batchSize}...`);
+  console.log(
+    `💾 Bulk inserting ${allRecords.length} file stats in batches of ${batchSize}...`,
+  );
 
   let totalInserted = 0;
   let totalSkipped = 0;
@@ -689,16 +709,17 @@ const insertStatsOnlyToUploaderTable = async (files, options) => {
 
     try {
       // OPTIMIZED: Use upsert without select to avoid unnecessary data transfer
-      const { error, count } = await supabase
-        .from('uploader')
-        .upsert(batch, {
-          onConflict: 'original_path',
-          ignoreDuplicates: false,
-          count: 'exact'
-        });
+      const { error, count } = await supabase.from('uploader').upsert(batch, {
+        onConflict: 'original_path',
+        ignoreDuplicates: false,
+        count: 'exact',
+      });
 
       if (error) {
-        console.error(`❌ Error inserting batch ${Math.floor(i / batchSize) + 1}:`, error.message);
+        console.error(
+          `❌ Error inserting batch ${Math.floor(i / batchSize) + 1}:`,
+          error.message,
+        );
         continue;
       }
 
@@ -715,45 +736,65 @@ const insertStatsOnlyToUploaderTable = async (files, options) => {
       // This is reasonable since stats-only is typically run on new file sets
       totalInserted += affected;
 
-      console.log(`✅ Batch ${Math.floor(i / batchSize) + 1}: ${affected} rows processed`);
+      console.log(
+        `✅ Batch ${Math.floor(i / batchSize) + 1}: ${affected} rows processed`,
+      );
     } catch (error) {
-      console.error(`❌ Unexpected error in batch ${Math.floor(i / batchSize) + 1}:`, error.message);
+      console.error(
+        `❌ Unexpected error in batch ${Math.floor(i / batchSize) + 1}:`,
+        error.message,
+      );
     }
   }
 
   // Calculate skipped as difference between total records and inserted
   totalSkipped = allRecords.length - totalInserted;
 
-  console.log(`📊 Phase 1 Summary: ${totalInserted} records processed, estimated ${totalSkipped} were updates`);
+  console.log(
+    `📊 Phase 1 Summary: ${totalInserted} records processed, estimated ${totalSkipped} were updates`,
+  );
 
   return {
     totalInserted,
     totalSkipped,
-    totalProcessed: allRecords.length
+    totalProcessed: allRecords.length,
   };
 };
 
 /**
  * PHASE 2: Process PDF files for pedimento-simplificado detection
  * Only processes files with status 'fs-stats' and file_extension 'pdf'
+ * Processes records in chunks of 1000 to avoid loading all records into memory
  */
 const detectPedimentosInDatabase = async (options = {}) => {
   if (!supabase) {
     throw new Error('Supabase client not initialized.');
   }
 
-  console.log('🔍 Phase 2: Starting PDF detection for pedimento-simplificado documents...');
+  console.log(
+    '🔍 Phase 2: Starting PDF detection for pedimento-simplificado documents...',
+  );
+
+  const detectionService = new FileDetectionService();
+  const processingBatchSize = parseInt(options.batchSize) || 10; // Smaller batches for file I/O
+  const queryBatchSize = 1000; // Process 1000 records at a time
 
-  // Get all PDF files that need detection (status = 'fs-stats' and extension = 'pdf')
-  let allPdfRecords = [];
-  let hasMore = true;
+  let totalDetected = 0;
+  let totalProcessed = 0;
+  let totalErrors = 0;
   let offset = 0;
-  const queryBatchSize = 1000;
+  let chunkNumber = 1;
+
+  console.log('� Processing PDF files in chunks of 1000 records...');
 
-  console.log('📥 Fetching PDF files from database...');
+  // Process records in chunks of 1000
+  while (true) {
+    console.log(
+      `\n📥 Fetching chunk ${chunkNumber} (records ${offset + 1} to ${offset + queryBatchSize})...`,
+    );
 
-  while (hasMore) {
-    const { data: batch, error: queryError } = await supabase
+    // Fetch next chunk of PDF records
+    const { data: pdfRecords, error: queryError } = await supabase
       .from('uploader')
       .select('id, original_path, filename, file_extension, status')
       .eq('status', 'fs-stats')
@@ -762,131 +803,157 @@ const detectPedimentosInDatabase = async (options = {}) => {
       .range(offset, offset + queryBatchSize - 1);
 
     if (queryError) {
-      throw new Error(`Failed to fetch PDF records: ${queryError.message}`);
+      throw new Error(
+        `Failed to fetch PDF records chunk ${chunkNumber}: ${queryError.message}`,
+      );
     }
 
-    if (!batch || batch.length === 0) {
-      hasMore = false;
-    } else {
-      allPdfRecords.push(...batch);
-      offset += queryBatchSize;
-      console.log(`📄 Fetched ${batch.length} PDF records (total: ${allPdfRecords.length})`);
+    // If no records found, we're done
+    if (!pdfRecords || pdfRecords.length === 0) {
+      console.log(`📝 No more PDF files found. Processing completed.`);
+      break;
     }
-  }
 
-  if (allPdfRecords.length === 0) {
-    console.log('📝 No PDF files found for detection');
-    return { detectedCount: 0, processedCount: 0, errorCount: 0 };
-  }
+    console.log(
+      `� Processing chunk ${chunkNumber}: ${pdfRecords.length} PDF records`,
+    );
 
-  console.log(`🔍 Processing ${allPdfRecords.length} PDF files for detection...`);
+    // Create progress bar for this chunk
+    const progressBar = new cliProgress.SingleBar({
+      format: `🔍 Chunk ${chunkNumber} |{bar}| {percentage}% | {value}/{total} | Detected: {detected} | Errors: {errors}`,
+      barCompleteChar: '█',
+      barIncompleteChar: '░',
+      hideCursor: true,
+    });
 
-  const detectionService = new FileDetectionService();
-  const batchSize = parseInt(options.batchSize) || 10; // Smaller batches for file I/O
-  let totalDetected = 0;
-  let totalProcessed = 0;
-  let totalErrors = 0;
+    progressBar.start(pdfRecords.length, 0, { detected: 0, errors: 0 });
 
-  // Create progress bar
-  const progressBar = new cliProgress.SingleBar({
-    format: '🔍 PDF Detection |{bar}| {percentage}% | {value}/{total} | Detected: {detected} | Errors: {errors}',
-    barCompleteChar: '█',
-    barIncompleteChar: '░',
-    hideCursor: true,
-  });
+    let chunkDetected = 0;
+    let chunkProcessed = 0;
+    let chunkErrors = 0;
+
+    // Process files in smaller batches within this chunk
+    for (let i = 0; i < pdfRecords.length; i += processingBatchSize) {
+      const batch = pdfRecords.slice(i, i + processingBatchSize);
+      const updatePromises = [];
 
-  progressBar.start(allPdfRecords.length, 0, { detected: 0, errors: 0 });
+      for (const record of batch) {
+        try {
+          // Check if file still exists
+          if (!fs.existsSync(record.original_path)) {
+            updatePromises.push(
+              supabase
+                .from('uploader')
+                .update({
+                  status: 'file-not-found',
+                  message: 'File no longer exists at original path',
+                })
+                .eq('id', record.id),
+            );
+            chunkErrors++;
+            totalErrors++;
+            continue;
+          }
 
-  // Process files in smaller batches to avoid overwhelming the system
-  for (let i = 0; i < allPdfRecords.length; i += batchSize) {
-    const batch = allPdfRecords.slice(i, i + batchSize);
-    const updatePromises = [];
+          // Perform detection
+          const detection = await detectionService.detectFile(
+            record.original_path,
+          );
+          chunkProcessed++;
+          totalProcessed++;
+
+          const updateData = {
+            status: detection.detectedType ? 'detected' : 'not-detected',
+            document_type: detection.detectedType,
+            num_pedimento: detection.detectedPedimento,
+            arela_path: detection.arelaPath,
+            message: detection.error || null,
+          };
+
+          // Extract RFC from fields if available
+          if (detection.fields) {
+            const rfcField = detection.fields.find(
+              (f) => f.name === 'rfc' && f.found,
+            );
+            if (rfcField) {
+              updateData.rfc = rfcField.value;
+            }
+          }
+
+          if (detection.detectedType) {
+            chunkDetected++;
+            totalDetected++;
+          }
+
+          updatePromises.push(
+            supabase.from('uploader').update(updateData).eq('id', record.id),
+          );
+        } catch (error) {
+          console.error(
+            `❌ Error detecting ${record.filename}:`,
+            error.message,
+          );
+          chunkErrors++;
+          totalErrors++;
 
-    for (const record of batch) {
-      try {
-        // Check if file still exists
-        if (!fs.existsSync(record.original_path)) {
           updatePromises.push(
             supabase
               .from('uploader')
               .update({
-                status: 'file-not-found',
-                message: 'File no longer exists at original path'
+                status: 'detection-error',
+                message: error.message,
               })
-              .eq('id', record.id)
+              .eq('id', record.id),
           );
-          totalErrors++;
-          continue;
         }
+      }
 
-        // Perform detection
-        const detection = await detectionService.detectFile(record.original_path);
-        totalProcessed++;
-
-        const updateData = {
-          status: detection.detectedType ? 'detected' : 'not-detected',
-          document_type: detection.detectedType,
-          num_pedimento: detection.detectedPedimento,
-          arela_path: detection.arelaPath,
-          message: detection.error || null
-        };
-
-        // Extract RFC from fields if available
-        if (detection.fields) {
-          const rfcField = detection.fields.find(f => f.name === 'rfc' && f.found);
-          if (rfcField) {
-            updateData.rfc = rfcField.value;
-          }
-        }
-
-        if (detection.detectedType) {
-          totalDetected++;
-        }
-
-        updatePromises.push(
-          supabase
-            .from('uploader')
-            .update(updateData)
-            .eq('id', record.id)
-        );
-
+      // Execute all updates in parallel for this batch
+      try {
+        await Promise.all(updatePromises);
       } catch (error) {
-        console.error(`❌ Error detecting ${record.filename}:`, error.message);
-        totalErrors++;
-
-        updatePromises.push(
-          supabase
-            .from('uploader')
-            .update({
-              status: 'detection-error',
-              message: error.message
-            })
-            .eq('id', record.id)
+        console.error(
+          `❌ Error updating batch in chunk ${chunkNumber}:`,
+          error.message,
         );
       }
+
+      // Update progress for this chunk
+      progressBar.update(Math.min(i + processingBatchSize, pdfRecords.length), {
+        detected: chunkDetected,
+        errors: chunkErrors,
+      });
     }
 
-    // Execute all updates in parallel for this batch
-    try {
-      await Promise.all(updatePromises);
-    } catch (error) {
-      console.error(`❌ Error updating batch:`, error.message);
+    progressBar.stop();
+
+    console.log(
+      `✅ Chunk ${chunkNumber} completed: ${chunkDetected} detected, ${chunkProcessed} processed, ${chunkErrors} errors`,
+    );
+
+    // Move to next chunk
+    offset += queryBatchSize;
+    chunkNumber++;
+
+    // If we got fewer records than queryBatchSize, we've reached the end
+    if (pdfRecords.length < queryBatchSize) {
+      console.log(
+        `📝 Reached end of records (chunk had ${pdfRecords.length} records).`,
+      );
+      break;
     }
 
-    // Update progress
-    progressBar.update(Math.min(i + batchSize, allPdfRecords.length), {
-      detected: totalDetected,
-      errors: totalErrors
-    });
+    // Small delay between chunks to avoid overwhelming the database
+    await new Promise((resolve) => setTimeout(resolve, 500));
   }
 
-  progressBar.stop();
-
-  console.log(`📊 Phase 2 Summary: ${totalDetected} detected, ${totalProcessed} processed, ${totalErrors} errors`);
+  console.log(
+    `📊 Phase 2 Summary: ${totalDetected} detected, ${totalProcessed} processed, ${totalErrors} errors`,
+  );
   return {
     detectedCount: totalDetected,
     processedCount: totalProcessed,
-    errorCount: totalErrors
+    errorCount: totalErrors,
   };
 };
 
@@ -926,7 +993,9 @@ const processFilesInBatches = async (
 
   if (options.statsOnly) {
     // OPTIMIZED Stats-only mode - Only read filesystem stats, no file detection
-    console.log('📊 Phase 1: Processing files in optimized stats-only mode (no detection)...');
+    console.log(
+      '📊 Phase 1: Processing files in optimized stats-only mode (no detection)...',
+    );
 
     for (let i = 0; i < files.length; i += batchSize) {
       const batch = files.slice(i, i + batchSize);
@@ -934,7 +1003,7 @@ const processFilesInBatches = async (
       // OPTIMIZED: Batch read file stats to reduce I/O overhead
       const fileStatsResults = batchReadFileStats(batch);
       const statsFiles = fileStatsResults
-        .filter(result => result.stats !== null) // Only include files with valid stats
+        .filter((result) => result.stats !== null) // Only include files with valid stats
         .map((result) => {
           const originalFileName = path.basename(result.path);
 
@@ -946,16 +1015,23 @@ const processFilesInBatches = async (
         });
 
       // Log any files that couldn't be read
-      const failedFiles = fileStatsResults.filter(result => result.error !== null);
+      const failedFiles = fileStatsResults.filter(
+        (result) => result.error !== null,
+      );
       if (failedFiles.length > 0) {
-        console.log(`⚠️ Could not read stats for ${failedFiles.length} files in batch`);
-        failedFiles.forEach(failed => {
+        console.log(
+          `⚠️ Could not read stats for ${failedFiles.length} files in batch`,
+        );
+        failedFiles.forEach((failed) => {
           console.error(`   ❌ ${failed.path}: ${failed.error}`);
         });
       }
 
       try {
-        const result = await insertStatsOnlyToUploaderTable(statsFiles, options);
+        const result = await insertStatsOnlyToUploaderTable(
+          statsFiles,
+          options,
+        );
 
         totalUploaded += result.totalInserted;
         totalSkipped += result.totalSkipped;
@@ -966,7 +1042,6 @@ const processFilesInBatches = async (
           failureCount: totalErrors,
           skippedCount: totalSkipped,
         });
-
       } catch (error) {
         console.error(`❌ Error processing stats batch:`, error.message);
         totalErrors += batch.length;
@@ -1238,13 +1313,19 @@ const uploadFilesByRfc = async (options = {}) => {
   }
 
   if (!API_BASE_URL || !API_TOKEN) {
-    console.error('❌ Arela API configuration missing. Please set ARELA_API_URL and ARELA_API_TOKEN environment variables.');
+    console.error(
+      '❌ Arela API configuration missing. Please set ARELA_API_URL and ARELA_API_TOKEN environment variables.',
+    );
     process.exit(1);
   }
 
   if (!uploadRfcs || uploadRfcs.length === 0) {
-    console.error('❌ No RFCs specified. Please set UPLOAD_RFCS environment variable with pipe-separated RFC values.');
-    console.error('   Example: UPLOAD_RFCS="RFC123456789|RFC987654321|RFC555444333"');
+    console.error(
+      '❌ No RFCs specified. Please set UPLOAD_RFCS environment variable with pipe-separated RFC values.',
+    );
+    console.error(
+      '   Example: UPLOAD_RFCS="RFC123456789|RFC987654321|RFC555444333"',
+    );
     process.exit(1);
   }
 
@@ -1266,47 +1347,65 @@ const uploadFilesByRfc = async (options = {}) => {
 
   if (!rfcRecords || rfcRecords.length === 0) {
     console.log('ℹ️  No files found for the specified RFCs with arela_path');
-    console.log(`   Make sure files for RFCs [${uploadRfcs.join(', ')}] have been processed and have arela_path values`);
+    console.log(
+      `   Make sure files for RFCs [${uploadRfcs.join(', ')}] have been processed and have arela_path values`,
+    );
     return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
   }
 
   // Step 2: Get unique arela_paths from the RFC matches
-  const uniqueArelaPaths = [...new Set(rfcRecords.map(r => r.arela_path))];
-  console.log(`� Found ${uniqueArelaPaths.length} unique arela_path(s) for the specified RFCs`);
+  const uniqueArelaPaths = [...new Set(rfcRecords.map((r) => r.arela_path))];
+  console.log(
+    `� Found ${uniqueArelaPaths.length} unique arela_path(s) for the specified RFCs`,
+  );
 
   // Step 3: Get ALL files that have these arela_paths (including supporting documents)
-  // Use pagination to ensure we get all files, regardless of count
+  // Process arela_paths in smaller chunks to avoid URI length limits
   let allRelatedFiles = [];
-  let hasMore = true;
-  let offset = 0;
+  const arelaPathChunkSize = 50; // Process 50 arela_paths at a time to avoid URI limits
   const queryBatchSize = 1000;
 
-  console.log('📥 Fetching all related files (with pagination)...');
+  console.log('📥 Fetching all related files (processing arela_paths in chunks to avoid URI limits)...');
 
-  while (hasMore) {
-    const { data: batch, error: queryError } = await supabase
-      .from('uploader')
-      .select('id, original_path, arela_path, filename, rfc, document_type')
-      .in('arela_path', uniqueArelaPaths)
-      .not('original_path', 'is', null)
-      .range(offset, offset + queryBatchSize - 1);
+  // Process arela_paths in chunks
+  for (let i = 0; i < uniqueArelaPaths.length; i += arelaPathChunkSize) {
+    const arelaPathChunk = uniqueArelaPaths.slice(i, i + arelaPathChunkSize);
+    console.log(`   Processing arela_path chunk ${Math.floor(i / arelaPathChunkSize) + 1}/${Math.ceil(uniqueArelaPaths.length / arelaPathChunkSize)} (${arelaPathChunk.length} paths)`);
 
-    if (queryError) {
-      console.error('❌ Error fetching related files:', queryError.message);
-      return { processedCount: 0, uploadedCount: 0, errorCount: 1 };
-    }
+    // For each chunk of arela_paths, use pagination to get all related files
+    let hasMore = true;
+    let offset = 0;
 
-    if (!batch || batch.length === 0) {
-      hasMore = false;
-    } else {
-      allRelatedFiles = allRelatedFiles.concat(batch);
-      offset += queryBatchSize;
+    while (hasMore) {
+      const { data: batch, error: queryError } = await supabase
+        .from('uploader')
+        .select('id, original_path, arela_path, filename, rfc, document_type')
+        .in('arela_path', arelaPathChunk)
+        .not('original_path', 'is', null)
+        .range(offset, offset + queryBatchSize - 1);
+
+      if (queryError) {
+        console.error(`❌ Error fetching related files for chunk ${Math.floor(i / arelaPathChunkSize) + 1}:`, queryError.message);
+        return { processedCount: 0, uploadedCount: 0, errorCount: 1 };
+      }
 
-      // If we got less than queryBatchSize, we've reached the end
-      if (batch.length < queryBatchSize) {
+      if (!batch || batch.length === 0) {
         hasMore = false;
+      } else {
+        allRelatedFiles = allRelatedFiles.concat(batch);
+        offset += queryBatchSize;
+
+        // If we got less than queryBatchSize, we've reached the end for this chunk
+        if (batch.length < queryBatchSize) {
+          hasMore = false;
+        }
       }
     }
+
+    // Small delay between chunks to avoid overwhelming the database
+    if (i + arelaPathChunkSize < uniqueArelaPaths.length) {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    }
   }
 
   if (!allRelatedFiles || allRelatedFiles.length === 0) {
@@ -1314,7 +1413,9 @@ const uploadFilesByRfc = async (options = {}) => {
     return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
   }
 
-  console.log(`📁 Found ${allRelatedFiles.length} total files to upload (including supporting documents)`);
+  console.log(
+    `📁 Found ${allRelatedFiles.length} total files to upload (including supporting documents)`,
+  );
 
   // Group by RFC and arela_path for better organization
   const filesByRfc = allRelatedFiles.reduce((acc, record) => {
@@ -1328,8 +1429,12 @@ const uploadFilesByRfc = async (options = {}) => {
 
   console.log('📊 Files by RFC (including supporting documents):');
   for (const [rfc, files] of Object.entries(filesByRfc)) {
-    const documentTypes = [...new Set(files.map(f => f.document_type || 'Unknown'))];
-    console.log(`   ${rfc}: ${files.length} files (${documentTypes.join(', ')})`);
+    const documentTypes = [
+      ...new Set(files.map((f) => f.document_type || 'Unknown')),
+    ];
+    console.log(
+      `   ${rfc}: ${files.length} files (${documentTypes.join(', ')})`,
+    );
   }
 
   // Group by arela_path for upload organization
@@ -1354,7 +1459,8 @@ const uploadFilesByRfc = async (options = {}) => {
 
   // Create progress bar
   const progressBar = new cliProgress.SingleBar({
-    format: '🚀 Uploading files |{bar}| {percentage}% | {value}/{total} | Uploaded: {uploaded} | Errors: {errors} | Skipped: {skipped}',
+    format:
+      '🚀 Uploading files |{bar}| {percentage}% | {value}/{total} | Uploaded: {uploaded} | Errors: {errors} | Skipped: {skipped}',
     barCompleteChar: '█',
     barIncompleteChar: '░',
     hideCursor: true,
@@ -1377,7 +1483,9 @@ const uploadFilesByRfc = async (options = {}) => {
     const batchNumber = Math.floor(i / batchSize) + 1;
     const totalBatches = Math.ceil(allRelatedFiles.length / batchSize);
 
-    console.log(`\n📦 Processing batch ${batchNumber}/${totalBatches} (${batch.length} files)`);
+    console.log(
+      `\n📦 Processing batch ${batchNumber}/${totalBatches} (${batch.length} files)`,
+    );
 
     // Prepare files for upload
     const filesToUpload = [];
@@ -1407,9 +1515,11 @@ const uploadFilesByRfc = async (options = {}) => {
           rfc: record.rfc,
           documentType: record.document_type,
         });
-
       } catch (error) {
-        console.error(`   ❌ Error reading file ${record.original_path}:`, error.message);
+        console.error(
+          `   ❌ Error reading file ${record.original_path}:`,
+          error.message,
+        );
         totalErrors++;
       }
 
@@ -1425,7 +1535,9 @@ const uploadFilesByRfc = async (options = {}) => {
     // Upload the batch if we have files
     if (filesToUpload.length > 0) {
       try {
-        console.log(`   🚀 Uploading ${filesToUpload.length} files to Arela API...`);
+        console.log(
+          `   🚀 Uploading ${filesToUpload.length} files to Arela API...`,
+        );
 
         const formData = new FormData();
 
@@ -1461,7 +1573,9 @@ const uploadFilesByRfc = async (options = {}) => {
 
           // Set folder structure for this group - concatenate custom prefix with arela_path
           const folderStructure = options.folderStructure
-            ? `${options.folderStructure}/${arelaPath}`.replace(/\/+/g, '/').replace(/\/$/, '')
+            ? `${options.folderStructure}/${arelaPath}`
+              .replace(/\/+/g, '/')
+              .replace(/\/$/, '')
             : arelaPath;
           pathFormData.append('folderStructure', folderStructure);
           pathFormData.append('autoDetect', 'true');
@@ -1472,15 +1586,20 @@ const uploadFilesByRfc = async (options = {}) => {
             pathFormData.append('bucket', bucket);
           }
 
-          console.log(`     📁 Uploading ${pathFiles.length} files to: ${folderStructure}`);
+          console.log(
+            `     📁 Uploading ${pathFiles.length} files to: ${folderStructure}`,
+          );
 
-          const response = await fetch(`${API_BASE_URL}/api/storage/batch-upload-and-process`, {
-            method: 'POST',
-            headers: {
-              'x-api-key': API_TOKEN,
+          const response = await fetch(
+            `${API_BASE_URL}/api/storage/batch-upload-and-process`,
+            {
+              method: 'POST',
+              headers: {
+                'x-api-key': API_TOKEN,
+              },
+              body: pathFormData,
             },
-            body: pathFormData,
-          });
+          );
 
           if (!response.ok) {
             const errorText = await response.text();
@@ -1490,22 +1609,31 @@ const uploadFilesByRfc = async (options = {}) => {
           const result = await response.json();
 
           // Check if upload was successful based on stats rather than success field
-          const isSuccessful = result.stats && result.stats.uploadedCount > 0 && result.stats.errorCount === 0;
+          const isSuccessful =
+            result.stats &&
+            result.stats.uploadedCount > 0 &&
+            result.stats.errorCount === 0;
 
           if (isSuccessful) {
-            console.log(`     ✅ Group uploaded: ${result.stats.uploadedCount} files to ${folderStructure}`);
+            console.log(
+              `     ✅ Group uploaded: ${result.stats.uploadedCount} files to ${folderStructure}`,
+            );
             totalUploaded += result.stats.uploadedCount;
 
             if (result.stats.detectedCount > 0) {
-              console.log(`     🔍 Files detected: ${result.stats.detectedCount}`);
+              console.log(
+                `     🔍 Files detected: ${result.stats.detectedCount}`,
+              );
             }
             if (result.stats.organizedCount > 0) {
-              console.log(`     📁 Files organized: ${result.stats.organizedCount}`);
+              console.log(
+                `     📁 Files organized: ${result.stats.organizedCount}`,
+              );
             }
           } else {
             console.error(`     ❌ Upload failed for ${folderStructure}:`);
             if (result.errors && result.errors.length > 0) {
-              result.errors.forEach(error => {
+              result.errors.forEach((error) => {
                 console.error(`        - ${error.fileName}: ${error.error}`);
               });
             }
@@ -1513,18 +1641,20 @@ const uploadFilesByRfc = async (options = {}) => {
           }
 
           // Small delay between path groups
-          await new Promise(resolve => setTimeout(resolve, 100));
+          await new Promise((resolve) => setTimeout(resolve, 100));
         }
-
       } catch (error) {
-        console.error(`   ❌ Error uploading batch ${batchNumber}:`, error.message);
+        console.error(
+          `   ❌ Error uploading batch ${batchNumber}:`,
+          error.message,
+        );
         totalErrors += filesToUpload.length;
       }
     }
 
     // Small delay between batches
     if (i + batchSize < allRelatedFiles.length) {
-      await new Promise(resolve => setTimeout(resolve, BATCH_DELAY));
+      await new Promise((resolve) => setTimeout(resolve, BATCH_DELAY));
     }
   }
 
@@ -1568,7 +1698,10 @@ const propagateArelaPath = async (options = {}) => {
     .not('arela_path', 'is', null);
 
   if (pedimentoError) {
-    console.error('❌ Error fetching pedimento records:', pedimentoError.message);
+    console.error(
+      '❌ Error fetching pedimento records:',
+      pedimentoError.message,
+    );
     return { processedCount: 0, updatedCount: 0, errorCount: 1 };
   }
 
@@ -1577,7 +1710,9 @@ const propagateArelaPath = async (options = {}) => {
     return { processedCount: 0, updatedCount: 0, errorCount: 0 };
   }
 
-  console.log(`📋 Found ${pedimentoRecords.length} pedimento records with arela_path`);
+  console.log(
+    `📋 Found ${pedimentoRecords.length} pedimento records with arela_path`,
+  );
 
   let totalProcessed = 0;
   let totalUpdated = 0;
@@ -1585,7 +1720,8 @@ const propagateArelaPath = async (options = {}) => {
 
   // Create progress bar
   const progressBar = new cliProgress.SingleBar({
-    format: '🔄 Propagating paths |{bar}| {percentage}% | {value}/{total} | Updated: {updated} | Errors: {errors}',
+    format:
+      '🔄 Propagating paths |{bar}| {percentage}% | {value}/{total} | Updated: {updated} | Errors: {errors}',
     barCompleteChar: '█',
     barIncompleteChar: '░',
     hideCursor: true,
@@ -1611,9 +1747,11 @@ const propagateArelaPath = async (options = {}) => {
 
       // Extract folder part from existing arela_path by removing the filename
       const existingPath = pedimento.arela_path;
-      const folderArelaPath = existingPath.includes('/') ?
-        existingPath.substring(0, existingPath.lastIndexOf('/')) + '/' :
-        existingPath.endsWith('/') ? existingPath : existingPath + '/';
+      const folderArelaPath = existingPath.includes('/')
+        ? existingPath.substring(0, existingPath.lastIndexOf('/')) + '/'
+        : existingPath.endsWith('/')
+          ? existingPath
+          : existingPath + '/';
 
       console.log(`   🎯 Original arela path: ${existingPath}`);
       console.log(`   📁 Folder arela path: ${folderArelaPath}`);
@@ -1627,7 +1765,10 @@ const propagateArelaPath = async (options = {}) => {
         .neq('id', pedimento.id); // Exclude the pedimento itself
 
       if (relatedError) {
-        console.error(`❌ Error finding related files for ${pedimento.filename}:`, relatedError.message);
+        console.error(
+          `❌ Error finding related files for ${pedimento.filename}:`,
+          relatedError.message,
+        );
         totalErrors++;
         continue;
       }
@@ -1637,11 +1778,13 @@ const propagateArelaPath = async (options = {}) => {
         continue;
       }
 
-      console.log(`   📄 Found ${relatedFiles.length} related files to update:`);
+      console.log(
+        `   📄 Found ${relatedFiles.length} related files to update:`,
+      );
 
       // Show first 10 files, then indicate if there are more
       const filesToShow = relatedFiles.slice(0, 10);
-      filesToShow.forEach(file => {
+      filesToShow.forEach((file) => {
         console.log(`      - ${file.filename}`);
       });
 
@@ -1651,18 +1794,22 @@ const propagateArelaPath = async (options = {}) => {
 
       // Process files in batches to avoid URI length limitations
       const BATCH_SIZE = 50; // Process 50 files at a time
-      const fileIds = relatedFiles.map(f => f.id);
+      const fileIds = relatedFiles.map((f) => f.id);
       let batchErrors = 0;
       let batchUpdated = 0;
 
-      console.log(`   🔄 Processing ${relatedFiles.length} files in batches of ${BATCH_SIZE}...`);
+      console.log(
+        `   🔄 Processing ${relatedFiles.length} files in batches of ${BATCH_SIZE}...`,
+      );
 
       for (let i = 0; i < fileIds.length; i += BATCH_SIZE) {
         const batchIds = fileIds.slice(i, i + BATCH_SIZE);
         const batchNumber = Math.floor(i / BATCH_SIZE) + 1;
         const totalBatches = Math.ceil(fileIds.length / BATCH_SIZE);
 
-        console.log(`   📦 Batch ${batchNumber}/${totalBatches}: Updating ${batchIds.length} files...`);
+        console.log(
+          `   📦 Batch ${batchNumber}/${totalBatches}: Updating ${batchIds.length} files...`,
+        );
 
         try {
           const { error: updateError } = await supabase
@@ -1671,33 +1818,45 @@ const propagateArelaPath = async (options = {}) => {
             .in('id', batchIds);
 
           if (updateError) {
-            console.error(`   ❌ Error in batch ${batchNumber}:`, updateError.message);
+            console.error(
+              `   ❌ Error in batch ${batchNumber}:`,
+              updateError.message,
+            );
             batchErrors++;
           } else {
-            console.log(`   ✅ Batch ${batchNumber} completed: ${batchIds.length} files updated`);
+            console.log(
+              `   ✅ Batch ${batchNumber} completed: ${batchIds.length} files updated`,
+            );
             batchUpdated += batchIds.length;
           }
         } catch (error) {
-          console.error(`   ❌ Exception in batch ${batchNumber}:`, error.message);
+          console.error(
+            `   ❌ Exception in batch ${batchNumber}:`,
+            error.message,
+          );
           batchErrors++;
         }
 
         // Small delay between batches to avoid overwhelming the database
         if (i + BATCH_SIZE < fileIds.length) {
-          await new Promise(resolve => setTimeout(resolve, 100));
+          await new Promise((resolve) => setTimeout(resolve, 100));
         }
       }
 
       if (batchErrors > 0) {
-        console.error(`❌ ${batchErrors} batch(es) failed for ${pedimento.filename}`);
+        console.error(
+          `❌ ${batchErrors} batch(es) failed for ${pedimento.filename}`,
+        );
         totalErrors++;
       } else {
         console.log(`   🎯 Successfully updated ${batchUpdated} related files`);
         totalUpdated += batchUpdated;
       }
-
     } catch (error) {
-      console.error(`❌ Error processing ${pedimento.filename}:`, error.message);
+      console.error(
+        `❌ Error processing ${pedimento.filename}:`,
+        error.message,
+      );
       totalErrors++;
     }
 
@@ -1761,12 +1920,27 @@ program
     'Automatically detect year/pedimento from file paths',
   )
   .option('--client-path <path>', 'Client path for metadata tracking')
-  .option('--stats-only', 'Phase 1: Only read filesystem stats and insert to database (no file reading or detection)')
+  .option(
+    '--stats-only',
+    'Phase 1: Only read filesystem stats and insert to database (no file reading or detection)',
+  )
   .option('--no-detect', 'Disable document type detection in stats-only mode')
-  .option('--detect-pdfs', 'Phase 2: Process PDF files in database for pedimento-simplificado detection')
-  .option('--propagate-arela-path', 'Phase 3: Propagate arela_path from pedimento_simplificado records to related files with same base path')
-  .option('--upload-by-rfc', 'Phase 4: Upload files to Arela API based on RFC values from UPLOAD_RFCS environment variable')
-  .option('--run-all-phases', 'Run all 4 phases in sequence: stats → detect → propagate → upload')
+  .option(
+    '--detect-pdfs',
+    'Phase 2: Process PDF files in database for pedimento-simplificado detection',
+  )
+  .option(
+    '--propagate-arela-path',
+    'Phase 3: Propagate arela_path from pedimento_simplificado records to related files with same base path',
+  )
+  .option(
+    '--upload-by-rfc',
+    'Phase 4: Upload files to Arela API based on RFC values from UPLOAD_RFCS environment variable',
+  )
+  .option(
+    '--run-all-phases',
+    'Run all 4 phases in sequence: stats → detect → propagate → upload',
+  )
   .action(async (options) => {
     if (options.version) {
       console.log(packageVersion);
@@ -1782,7 +1956,9 @@ program
         batchSize: parseInt(options.batchSize) || 10,
       });
 
-      console.log(`✅ Phase 2 Complete: ${result.detectedCount} detected, ${result.errorCount} errors`);
+      console.log(
+        `✅ Phase 2 Complete: ${result.detectedCount} detected, ${result.errorCount} errors`,
+      );
       return;
     }
 
@@ -1822,8 +1998,12 @@ program
       // Also initialize Supabase for database queries
       if (!supabase) {
         if (!supabaseUrl || !supabaseKey) {
-          console.error('❌ RFC upload requires Supabase credentials for database queries.');
-          console.error('   Please set SUPABASE_URL and SUPABASE_KEY environment variables.');
+          console.error(
+            '❌ RFC upload requires Supabase credentials for database queries.',
+          );
+          console.error(
+            '   Please set SUPABASE_URL and SUPABASE_KEY environment variables.',
+          );
           process.exit(1);
         }
 
@@ -1857,7 +2037,9 @@ program
     const concurrency = parseInt(options.concurrency) || 10;
 
     if (options.statsOnly) {
-      console.log('📊 Mode: Stats Only - Reading file stats and inserting to uploader table');
+      console.log(
+        '📊 Mode: Stats Only - Reading file stats and inserting to uploader table',
+      );
       console.log('🚫 Files will NOT be uploaded');
       if (options.detect !== false) {
         console.log('🔍 Document type detection ENABLED for supported files');
@@ -1965,14 +2147,18 @@ program
         const detectionResult = await detectPedimentosInDatabase({
           batchSize: parseInt(options.batchSize) || 10,
         });
-        console.log(`✅ Phase 2 Complete: ${detectionResult.detectedCount} detected, ${detectionResult.errorCount} errors`);
+        console.log(
+          `✅ Phase 2 Complete: ${detectionResult.detectedCount} detected, ${detectionResult.errorCount} errors`,
+        );
 
         // Phase 3: Propagate arela_path
         console.log('\n📁 === PHASE 3: Propagate Arela Paths ===');
         const propagateResult = await propagateArelaPath({
           showProgress: options.showStats || true,
         });
-        console.log(`✅ Phase 3 Complete: ${propagateResult.updatedCount || 0} paths propagated`);
+        console.log(
+          `✅ Phase 3 Complete: ${propagateResult.updatedCount || 0} paths propagated`,
+        );
 
         // Phase 4: Upload by RFC
         if (uploadRfcs && uploadRfcs.length > 0) {
@@ -1989,18 +2175,22 @@ program
           console.log(`✅ Phase 4 Complete: Upload finished`);
         } else {
           console.log('\n⚠️ === PHASE 4: Upload by RFC ===');
-          console.log('⚠️ UPLOAD_RFCS environment variable not configured, skipping Phase 4');
+          console.log(
+            '⚠️ UPLOAD_RFCS environment variable not configured, skipping Phase 4',
+          );
         }
 
         console.log('\n🎉 All 4 phases completed successfully!');
-
       } catch (error) {
         console.error(`❌ Error in multi-phase execution:`, error.message);
         process.exit(1);
       }
     }
 
-    if (options.showStats && (sanitizationCache.size > 0 || pathDetectionCache.size > 0)) {
+    if (
+      options.showStats &&
+      (sanitizationCache.size > 0 || pathDetectionCache.size > 0)
+    ) {
       console.log(`📊 Performance Statistics:`);
       if (sanitizationCache.size > 0) {
         console.log(