@arela/uploader 0.2.4 → 0.2.6

package/src/services/DatabaseService.js

@@ -0,0 +1,1140 @@
+import fs from 'fs';
+import path from 'path';
+
+import appConfig from '../config/config.js';
+import { FileDetectionService } from '../file-detection.js';
+import logger from './LoggingService.js';
+import uploadServiceFactory from './upload/UploadServiceFactory.js';
+
+/**
+ * Database Service
+ * Handles all Supabase database operations for the uploader table
+ */
+export class DatabaseService {
+  constructor() {
+    this.detectionService = new FileDetectionService();
+  }
+
+  /**
+   * Get Supabase client
+   * @private
+   * @returns {Promise<Object>} Supabase client
+   */
+  async #getSupabaseClient() {
+    const supabaseService = uploadServiceFactory.getSupabaseService();
+    return await supabaseService.getClient();
+  }
+
+  /**
+   * Execute database query with retry logic and exponential backoff
+   * @private
+   * @param {Function} queryFn - Query function to execute
+   * @param {string} operation - Description of the operation for logging
+   * @param {number} maxRetries - Maximum number of retry attempts (default: 3)
+   * @returns {Promise<Object>} Query result
+   */
+  async #queryWithRetry(queryFn, operation, maxRetries = 3) {
+    let lastError;
+
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+      try {
+        const result = await queryFn();
+        if (attempt > 1) {
+          logger.info(`${operation} succeeded on attempt ${attempt}`);
+        }
+        return result;
+      } catch (error) {
+        lastError = error;
+
+        // Check if it's a timeout or connection error
+        const isRetriableError =
+          error.message?.includes('timeout') ||
+          error.message?.includes('canceling statement') ||
+          error.message?.includes('connection') ||
+          error.code === 'PGRST301'; // PostgREST timeout
+
+        if (!isRetriableError || attempt === maxRetries) {
+          throw error;
+        }
+
+        const backoffDelay = Math.min(1000 * Math.pow(2, attempt - 1), 30000); // Cap at 30 seconds
+        logger.warn(
+          `${operation} failed on attempt ${attempt}/${maxRetries}: ${error.message}`,
+        );
+        logger.info(`Retrying in ${backoffDelay}ms...`);
+
+        await new Promise((resolve) => setTimeout(resolve, backoffDelay));
+      }
+    }
+
+    throw lastError;
+  }
+
+  /**
+   * Insert file stats with document detection into uploader table
+   * @param {Array} files - Array of file objects
+   * @param {Object} options - Options including clientPath
+   * @returns {Promise<Array>} Inserted records
+   */
+  async insertStatsToUploaderTable(files, options) {
+    const supabase = await this.#getSupabaseClient();
+    const records = [];
+
+    for (const file of files) {
+      const stats = file.stats || fs.statSync(file.path);
+      const originalPath = options.clientPath || file.path;
+
+      // Check if record already exists
+      const { data: existingRecords, error: checkError } = await supabase
+        .from('uploader')
+        .select('id, original_path')
+        .eq('original_path', originalPath)
+        .limit(1);
+
+      if (checkError) {
+        logger.error(
+          `Error checking for existing record: ${checkError.message}`,
+        );
+        continue;
+      }
+
+      if (existingRecords && existingRecords.length > 0) {
+        logger.info(`Skipping duplicate: ${path.basename(file.path)}`);
+        continue;
+      }
+
+      // Initialize record with basic file stats
+      const record = {
+        document_type: null,
+        size: stats.size,
+        num_pedimento: null,
+        filename: file.originalName || path.basename(file.path),
+        original_path: originalPath,
+        arela_path: null,
+        status: 'stats',
+        rfc: null,
+        message: null,
+      };
+
+      // Try to detect document type for supported files
+      if (this.detectionService.isSupportedFileType(file.path)) {
+        try {
+          const detection = await this.detectionService.detectFile(file.path);
+
+          if (detection.detectedType) {
+            record.document_type = detection.detectedType;
+            record.num_pedimento = detection.detectedPedimento;
+            record.status = 'detected';
+
+            if (detection.arelaPath) {
+              record.arela_path = detection.arelaPath;
+            }
+
+            const rfcField = detection.fields.find(
+              (f) => f.name === 'rfc' && f.found,
+            );
+            if (rfcField) {
+              record.rfc = rfcField.value;
+            }
+          } else {
+            record.status = 'not-detected';
+            if (detection.error) {
+              record.message = detection.error;
+            }
+          }
+        } catch (error) {
+          logger.error(`Error detecting ${record.filename}: ${error.message}`);
+          record.status = 'detection-error';
+          record.message = error.message;
+        }
+      } else {
+        record.status = 'unsupported';
+        record.message = 'File type not supported for detection';
+      }
+
+      records.push(record);
+    }
+
+    if (records.length === 0) {
+      logger.info('No new records to insert (all were duplicates or errors)');
+      return [];
+    }
+
+    logger.info(
+      `Inserting ${records.length} new records into uploader table...`,
+    );
+
+    const { data, error } = await supabase
+      .from('uploader')
+      .insert(records)
+      .select();
+
+    if (error) {
+      throw new Error(`Failed to insert stats records: ${error.message}`);
+    }
+
+    return data;
+  }
+
+  /**
+   * Insert file stats only (no detection) into uploader table
+   * @param {Array} files - Array of file objects
+   * @param {Object} options - Options including clientPath
+   * @returns {Promise<Object>} Statistics about the operation
+   */
+  async insertStatsOnlyToUploaderTable(files, options) {
+    const supabase = await this.#getSupabaseClient();
+    const batchSize = 1000;
+    const allRecords = [];
+
+    logger.info('Collecting filesystem stats...');
+    for (const file of files) {
+      try {
+        const stats = file.stats || fs.statSync(file.path);
+        const originalPath = options.clientPath || file.path;
+        const fileExtension = path
+          .extname(file.path)
+          .toLowerCase()
+          .replace('.', '');
+
+        const record = {
+          document_type: null,
+          size: stats.size,
+          num_pedimento: null,
+          filename: file.originalName || path.basename(file.path),
+          original_path: originalPath,
+          arela_path: null,
+          status: 'fs-stats',
+          rfc: null,
+          message: null,
+          file_extension: fileExtension,
+          created_at: new Date().toISOString(),
+          modified_at: stats.mtime.toISOString(),
+        };
+
+        allRecords.push(record);
+      } catch (error) {
+        logger.error(`Error reading stats for ${file.path}: ${error.message}`);
+      }
+    }
+
+    if (allRecords.length === 0) {
+      logger.info('No file stats to insert');
+      return { totalInserted: 0, totalSkipped: 0, totalProcessed: 0 };
+    }
+
+    logger.info(
+      `Processing ${allRecords.length} file stats in batches of ${batchSize}...`,
+    );
+
+    let totalInserted = 0;
+    let totalUpdated = 0;
+
+    for (let i = 0; i < allRecords.length; i += batchSize) {
+      const batch = allRecords.slice(i, i + batchSize);
+
+      try {
+        // Check which records already exist
+        const originalPaths = batch.map((r) => r.original_path);
+        const { data: existingRecords, error: checkError } = await supabase
+          .from('uploader')
+          .select('original_path')
+          .in('original_path', originalPaths);
+
+        if (checkError) {
+          logger.error(
+            `Error checking existing records: ${checkError.message}`,
+          );
+          continue;
+        }
+
+        const existingPaths = new Set(
+          existingRecords?.map((r) => r.original_path) || [],
+        );
+        const newRecords = batch.filter(
+          (r) => !existingPaths.has(r.original_path),
+        );
+        const updateRecords = batch.filter((r) =>
+          existingPaths.has(r.original_path),
+        );
+
+        logger.info(
+          `Batch ${Math.floor(i / batchSize) + 1}: ${newRecords.length} new, ${updateRecords.length} updates`,
+        );
+
+        // Insert new records
+        if (newRecords.length > 0) {
+          const { error: insertError } = await supabase
+            .from('uploader')
+            .insert(newRecords);
+
+          if (insertError) {
+            logger.error(`Error inserting new records: ${insertError.message}`);
+          } else {
+            totalInserted += newRecords.length;
+            logger.success(`Inserted ${newRecords.length} new records`);
+          }
+        }
+
+        // Update existing records
+        if (updateRecords.length > 0) {
+          let batchUpdated = 0;
+          for (const record of updateRecords) {
+            const { error: updateError } = await supabase
+              .from('uploader')
+              .update({
+                size: record.size,
+                modified_at: record.modified_at,
+                filename: record.filename,
+                file_extension: record.file_extension,
+              })
+              .eq('original_path', record.original_path);
+
+            if (!updateError) {
+              batchUpdated++;
+            }
+          }
+          totalUpdated += batchUpdated;
+          logger.info(`Updated ${batchUpdated} existing records`);
+        }
+      } catch (error) {
+        logger.error(
+          `Unexpected error in batch ${Math.floor(i / batchSize) + 1}: ${error.message}`,
+        );
+      }
+    }
+
+    logger.success(
+      `Phase 1 Summary: ${totalInserted} new records inserted, ${totalUpdated} existing records updated`,
+    );
+
+    return {
+      totalInserted,
+      totalSkipped: totalUpdated,
+      totalProcessed: allRecords.length,
+    };
+  }
+
+  /**
+   * Process PDF files for pedimento-simplificado detection
+   * @param {Object} options - Processing options
+   * @returns {Promise<Object>} Processing result
+   */
+  async detectPedimentosInDatabase(options = {}) {
+    const supabase = await this.#getSupabaseClient();
+
+    logger.info(
+      'Phase 2: Starting PDF detection for pedimento-simplificado documents...',
+    );
+
+    const processingBatchSize = parseInt(options.batchSize) || 10;
+    // Reduced query batch size to avoid timeouts
+    const queryBatchSize = 500; // Reduced from 1000 to 500
+
+    let totalDetected = 0;
+    let totalProcessed = 0;
+    let totalErrors = 0;
+    let offset = 0;
+    let chunkNumber = 1;
+
+    logger.info(
+      `Processing PDF files in chunks of ${queryBatchSize} records...`,
+    );
+
+    while (true) {
+      logger.info(
+        `Fetching chunk ${chunkNumber} (records ${offset + 1} to ${offset + queryBatchSize})...`,
+      );
+
+      try {
+        // Split the query to make it more efficient with retry logic
+        const { data: pdfRecords, error: queryError } =
+          await this.#queryWithRetry(async () => {
+            return await supabase
+              .from('uploader')
+              .select('id, original_path, filename, file_extension, status')
+              .eq('status', 'fs-stats')
+              .eq('file_extension', 'pdf')
+              .ilike('filename', '%simp%')
+              .range(offset, offset + queryBatchSize - 1)
+              .order('id'); // Add explicit ordering for consistent pagination
+          }, `fetch PDF records chunk ${chunkNumber}`);
+
+        if (queryError) {
+          throw new Error(
+            `Failed to fetch PDF records chunk ${chunkNumber}: ${queryError.message}`,
+          );
+        }
+
+        if (!pdfRecords || pdfRecords.length === 0) {
+          logger.info('No more PDF files found. Processing completed.');
+          break;
+        }
+
+        logger.info(
+          `Processing chunk ${chunkNumber}: ${pdfRecords.length} PDF records`,
+        );
+
+        let chunkDetected = 0;
+        let chunkProcessed = 0;
+        let chunkErrors = 0;
+
+        // Process files in smaller batches
+        for (let i = 0; i < pdfRecords.length; i += processingBatchSize) {
+          const batch = pdfRecords.slice(i, i + processingBatchSize);
+          const updatePromises = [];
+
+          for (const record of batch) {
+            try {
+              if (!fs.existsSync(record.original_path)) {
+                logger.warn(
+                  `File not found: ${record.filename} at ${record.original_path}`,
+                );
+                updatePromises.push(
+                  supabase
+                    .from('uploader')
+                    .update({
+                      status: 'file-not-found',
+                      message: 'File no longer exists at original path',
+                    })
+                    .eq('id', record.id),
+                );
+                chunkErrors++;
+                totalErrors++;
+                continue;
+              }
+
+              const detection = await this.detectionService.detectFile(
+                record.original_path,
+              );
+              chunkProcessed++;
+              totalProcessed++;
+
+              const updateData = {
+                status: detection.detectedType ? 'detected' : 'not-detected',
+                document_type: detection.detectedType,
+                num_pedimento: detection.detectedPedimento,
+                arela_path: detection.arelaPath,
+                message: detection.error || null,
+              };
+
+              if (detection.fields) {
+                const rfcField = detection.fields.find(
+                  (f) => f.name === 'rfc' && f.found,
+                );
+                if (rfcField) {
+                  updateData.rfc = rfcField.value;
+                }
+              }
+
+              if (detection.detectedType) {
+                chunkDetected++;
+                totalDetected++;
+                logger.success(
+                  `Detected: ${record.filename} -> ${detection.detectedType} | Pedimento: ${detection.detectedPedimento || 'N/A'} | RFC: ${detection.fields?.rfc || 'N/A'}`,
+                );
+              } else {
+                logger.info(
+                  `Not detected: ${record.filename} - No pedimento-simplificado pattern found`,
+                );
+              }
+
+              updatePromises.push(
+                supabase
+                  .from('uploader')
+                  .update(updateData)
+                  .eq('id', record.id),
+              );
+            } catch (error) {
+              logger.error(
+                `Error detecting ${record.filename}: ${error.message}`,
+              );
+              chunkErrors++;
+              totalErrors++;
+
+              updatePromises.push(
+                supabase
+                  .from('uploader')
+                  .update({
+                    status: 'detection-error',
+                    message: error.message,
+                  })
+                  .eq('id', record.id),
+              );
+            }
+          }
+
+          try {
+            await Promise.all(updatePromises);
+          } catch (error) {
+            logger.error(
+              `Error updating batch in chunk ${chunkNumber}: ${error.message}`,
+            );
+          }
+        }
+
+        logger.success(
+          `Chunk ${chunkNumber} completed: ${chunkDetected} detected, ${chunkProcessed} processed, ${chunkErrors} errors`,
+        );
+
+        offset += queryBatchSize;
+        chunkNumber++;
+
+        if (pdfRecords.length < queryBatchSize) {
+          logger.info(
+            `Reached end of records (chunk had ${pdfRecords.length} records).`,
+          );
+          break;
+        }
+
+        // Small delay between chunks
+        await new Promise((resolve) => setTimeout(resolve, 500));
+      } catch (chunkError) {
+        logger.error(
+          `Error processing chunk ${chunkNumber}: ${chunkError.message}`,
+        );
+        // Continue to next chunk after error
+        offset += queryBatchSize;
+        chunkNumber++;
+        totalErrors++;
+        continue;
+      }
+    }
+
+    const result = {
+      detectedCount: totalDetected,
+      processedCount: totalProcessed,
+      errorCount: totalErrors,
+    };
+
+    logger.success(
+      `Phase 2 Summary: ${totalDetected} detected, ${totalProcessed} processed, ${totalErrors} errors`,
+    );
+
+    return result;
+  }
+
+  /**
+   * Propagate arela_path from pedimento_simplificado records to related files
+   * @param {Object} options - Options for propagation
+   * @returns {Promise<Object>} Processing result
+   */
+  async propagateArelaPath(options = {}) {
+    const supabase = await this.#getSupabaseClient();
+
+    logger.info('Phase 3: Starting arela_path propagation process...');
+    console.log('🔍 Finding pedimento_simplificado records with arela_path...');
+
+    // Get all pedimento_simplificado records that have arela_path
+    const { data: pedimentoRecords, error: pedimentoError } = await supabase
+      .from('uploader')
+      .select('id, original_path, arela_path, filename')
+      .eq('document_type', 'pedimento_simplificado')
+      .not('arela_path', 'is', null);
+
+    if (pedimentoError) {
+      const errorMsg = `Error fetching pedimento records: ${pedimentoError.message}`;
+      logger.error(errorMsg);
+      throw new Error(errorMsg);
+    }
+
+    if (!pedimentoRecords || pedimentoRecords.length === 0) {
+      logger.info('No pedimento_simplificado records with arela_path found');
+      console.log(
+        'ℹ️  No pedimento_simplificado records with arela_path found',
+      );
+      return { processedCount: 0, updatedCount: 0, errorCount: 0 };
+    }
+
+    console.log(
+      `📋 Found ${pedimentoRecords.length} pedimento records with arela_path`,
+    );
+    logger.info(
+      `Found ${pedimentoRecords.length} pedimento records with arela_path to process`,
+    );
+
+    let totalProcessed = 0;
+    let totalUpdated = 0;
+    let totalErrors = 0;
+    const BATCH_SIZE = 50; // Process files in batches
+
+    // Process each pedimento record
+    for (const pedimento of pedimentoRecords) {
+      try {
+        totalProcessed++;
+
+        // Extract base path from original_path (remove filename)
+        const basePath = path.dirname(pedimento.original_path);
+
+        logger.info(
+          `Processing pedimento: ${pedimento.filename} | Base path: ${basePath}`,
+        );
+
+        // Extract folder part from existing arela_path
+        const existingPath = pedimento.arela_path;
+        const folderArelaPath = existingPath.includes('/')
+          ? existingPath.substring(0, existingPath.lastIndexOf('/')) + '/'
+          : existingPath.endsWith('/')
+            ? existingPath
+            : existingPath + '/';
+
+        // Find all files with the same base path that don't have arela_path yet
+        const { data: relatedFiles, error: relatedError } = await supabase
+          .from('uploader')
+          .select('id, filename, original_path')
+          .like('original_path', `${basePath}%`)
+          .is('arela_path', null)
+          .neq('id', pedimento.id); // Exclude the pedimento itself
+
+        if (relatedError) {
+          logger.error(
+            `Error finding related files for ${pedimento.filename}: ${relatedError.message}`,
+          );
+          totalErrors++;
+          continue;
+        }
+
+        if (!relatedFiles || relatedFiles.length === 0) {
+          logger.info(`No related files found for ${pedimento.filename}`);
+          continue;
+        }
+
+        logger.info(
+          `Found ${relatedFiles.length} related files to update for ${pedimento.filename}`,
+        );
+
+        // Process files in batches
+        const fileIds = relatedFiles.map((f) => f.id);
+
+        for (let i = 0; i < fileIds.length; i += BATCH_SIZE) {
+          const batchIds = fileIds.slice(i, i + BATCH_SIZE);
+          const batchNumber = Math.floor(i / BATCH_SIZE) + 1;
+          const totalBatches = Math.ceil(fileIds.length / BATCH_SIZE);
+
+          logger.info(
+            `Batch ${batchNumber}/${totalBatches}: Updating ${batchIds.length} files...`,
+          );
+
+          try {
+            const { error: updateError } = await supabase
+              .from('uploader')
+              .update({ arela_path: folderArelaPath })
+              .in('id', batchIds);
+
+            if (updateError) {
+              logger.error(
+                `Error in batch ${batchNumber}: ${updateError.message}`,
+              );
+              totalErrors++;
+            } else {
+              totalUpdated += batchIds.length;
+              logger.info(
+                `Successfully updated batch ${batchNumber}: ${batchIds.length} files`,
+              );
+            }
+          } catch (batchError) {
+            logger.error(
+              `Exception in batch ${batchNumber}: ${batchError.message}`,
+            );
+            totalErrors++;
+          }
+        }
+      } catch (error) {
+        logger.error(
+          `Error processing pedimento ${pedimento.filename}: ${error.message}`,
+        );
+        totalErrors++;
+      }
+    }
+
+    const result = {
+      processedCount: totalProcessed,
+      updatedCount: totalUpdated,
+      errorCount: totalErrors,
+    };
+
+    logger.success(
+      `Phase 3 Summary: ${totalProcessed} pedimentos processed, ${totalUpdated} files updated, ${totalErrors} errors`,
+    );
+
+    return result;
+  }
+
+  /**
+   * Upload files to Arela API based on specific RFC values
+   * @param {Object} options - Upload options
+   * @returns {Promise<Object>} Processing result
+   */
+  async uploadFilesByRfc(options = {}) {
+    const supabase = await this.#getSupabaseClient();
+    const uploadService = await uploadServiceFactory.getUploadService();
+
+    // Get configuration
+    const appConfig = await import('../config/config.js').then(
+      (m) => m.appConfig,
+    );
+
+    if (!appConfig.upload.rfcs || appConfig.upload.rfcs.length === 0) {
+      const errorMsg =
+        'No RFCs specified. Please set UPLOAD_RFCS environment variable with pipe-separated RFC values.';
+      logger.error(errorMsg);
+      throw new Error(errorMsg);
+    }
+
+    logger.info('Phase 4: Starting RFC-based upload process...');
+    console.log('🎯 RFC-based Upload Mode');
+    console.log(`📋 Target RFCs: ${appConfig.upload.rfcs.join(', ')}`);
+    console.log('🔍 Searching for files to upload...');
+
+    // First, count total files for the RFCs to show filtering effect
+    const { count: totalRfcFiles, error: countError } = await supabase
+      .from('uploader')
+      .select('*', { count: 'exact', head: true })
+      .in('rfc', appConfig.upload.rfcs)
+      .not('arela_path', 'is', null);
+
+    if (countError) {
+      logger.warn(`Could not count total RFC files: ${countError.message}`);
+    } else {
+      console.log(`📊 Total files for specified RFCs: ${totalRfcFiles || 0}`);
+      logger.info(`Total files for specified RFCs: ${totalRfcFiles || 0}`);
+    }
+
+    // Step 1: Get all pedimento_simplificado records that match the specified RFCs and have arela_path
+    console.log(
+      '🎯 Finding pedimento_simplificado records for specified RFCs...',
+    );
+    const { data: pedimentoRfcRecords, error: pedimentoRfcError } =
+      await supabase
+        .from('uploader')
+        .select('arela_path')
+        .eq('document_type', 'pedimento_simplificado')
+        .in('rfc', appConfig.upload.rfcs)
+        .not('arela_path', 'is', null);
+
+    if (pedimentoRfcError) {
+      const errorMsg = `Error fetching pedimento RFC records: ${pedimentoRfcError.message}`;
+      logger.error(errorMsg);
+      throw new Error(errorMsg);
+    }
+
+    if (!pedimentoRfcRecords || pedimentoRfcRecords.length === 0) {
+      console.log(
+        'ℹ️  No pedimento_simplificado records found for the specified RFCs with arela_path',
+      );
+      logger.info('No pedimento_simplificado records found for specified RFCs');
+      return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
+    }
+
+    // Get unique arela_paths from pedimento records
+    const uniqueArelaPaths = [
+      ...new Set(pedimentoRfcRecords.map((r) => r.arela_path)),
+    ];
+    console.log(
+      `📋 Found ${pedimentoRfcRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths for specified RFCs`,
+    );
+    logger.info(
+      `Found ${pedimentoRfcRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths`,
+    );
+
+    // Step 2: Get all files with these arela_paths that haven't been uploaded yet
+    let rfcRecords = [];
+    const chunkSize = 50;
+
+    for (let i = 0; i < uniqueArelaPaths.length; i += chunkSize) {
+      const pathChunk = uniqueArelaPaths.slice(i, i + chunkSize);
+
+      const { data: chunkFiles, error: chunkError } = await supabase
+        .from('uploader')
+        .select('arela_path')
+        .in('arela_path', pathChunk)
+        .neq('status', 'file-uploaded')
+        .not('arela_path', 'is', null);
+
+      if (chunkError) {
+        const errorMsg = `Error fetching files for arela_paths chunk: ${chunkError.message}`;
+        logger.error(errorMsg);
+        throw new Error(errorMsg);
+      }
+
+      if (chunkFiles && chunkFiles.length > 0) {
+        rfcRecords = rfcRecords.concat(chunkFiles);
+      }
+    }
+
+    if (!rfcRecords || rfcRecords.length === 0) {
+      if (totalRfcFiles && totalRfcFiles > 0) {
+        console.log(
+          `ℹ️  All ${totalRfcFiles} files for the specified RFCs are already uploaded (status: file-uploaded)`,
+        );
+        console.log('   No new files to upload.');
+        logger.info(
+          `All ${totalRfcFiles} files for specified RFCs already uploaded`,
+        );
+      } else {
+        console.log(
+          'ℹ️  No files found for the specified RFCs with arela_path',
+        );
+        console.log(
+          `   Make sure files for RFCs [${appConfig.upload.rfcs.join(', ')}] have been processed and have arela_path values`,
+        );
+        logger.info('No files found for specified RFCs with arela_path');
+      }
+      return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
+    }
+
+    // Show filtering effect
+    const uploadableArelaPaths = [
+      ...new Set(rfcRecords.map((r) => r.arela_path)),
+    ];
+    const skipped = (totalRfcFiles || 0) - rfcRecords.length;
+    if (skipped > 0) {
+      console.log(
+        `📊 Found ${rfcRecords.length} files ready for upload (${skipped} already uploaded, skipped)`,
+      );
+    } else {
+      console.log(`📊 Found ${rfcRecords.length} files ready for upload`);
+    }
+    logger.info(
+      `Found ${rfcRecords.length} files ready for upload, ${skipped} skipped`,
+    );
+
+    // Step 3: Get ALL files that have these arela_paths (including supporting documents)
+    let allRelatedFiles = [];
+    const arelaPathChunkSize = 50;
+    const queryBatchSize = 1000;
+
+    console.log(
+      '📥 Fetching all related files (processing arela_paths in chunks to avoid URI limits)...',
+    );
+
+    // Process arela_paths in chunks
+    for (let i = 0; i < uploadableArelaPaths.length; i += arelaPathChunkSize) {
+      const arelaPathChunk = uploadableArelaPaths.slice(
+        i,
+        i + arelaPathChunkSize,
+      );
+      const chunkNumber = Math.floor(i / arelaPathChunkSize) + 1;
+      const totalChunks = Math.ceil(
+        uploadableArelaPaths.length / arelaPathChunkSize,
+      );
+
+      console.log(
+        `   Processing arela_path chunk ${chunkNumber}/${totalChunks} (${arelaPathChunk.length} paths)`,
+      );
+
+      // For each chunk of arela_paths, use pagination to get all related files
+      let hasMore = true;
+      let offset = 0;
+
+      while (hasMore) {
+        const { data: batch, error: queryError } = await supabase
+          .from('uploader')
+          .select('id, original_path, arela_path, filename, rfc, document_type')
+          .in('arela_path', arelaPathChunk)
+          .not('original_path', 'is', null)
+          .neq('status', 'file-uploaded')
+          .range(offset, offset + queryBatchSize - 1);
+
+        if (queryError) {
+          const errorMsg = `Error fetching related files for chunk ${chunkNumber}: ${queryError.message}`;
+          logger.error(errorMsg);
+          throw new Error(errorMsg);
+        }
+
+        if (batch && batch.length > 0) {
+          allRelatedFiles = allRelatedFiles.concat(batch);
+        }
+
+        hasMore = batch && batch.length === queryBatchSize;
+        offset += queryBatchSize;
+      }
+    }
+
+    if (!allRelatedFiles || allRelatedFiles.length === 0) {
+      console.log('ℹ️  No related files found to upload');
+      logger.info('No related files found to upload');
+      return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
+    }
+
+    console.log(`📋 Total files to upload: ${allRelatedFiles.length}`);
+    logger.info(`Total files to upload: ${allRelatedFiles.length}`);
+
+    // Step 4: Upload all related files
+    let totalProcessed = 0;
+    let totalUploaded = 0;
+    let totalErrors = 0;
+    const batchSize = parseInt(options.batchSize) || 10;
+
+    for (let i = 0; i < allRelatedFiles.length; i += batchSize) {
+      const batch = allRelatedFiles.slice(i, i + batchSize);
+
+      for (const file of batch) {
+        try {
+          totalProcessed++;
+
+          // Check if file exists
+          if (!fs.existsSync(file.original_path)) {
+            logger.warn(
+              `File not found: ${file.filename} at ${file.original_path}`,
+            );
+            await supabase
+              .from('uploader')
+              .update({
+                status: 'file-not-found',
+                message: 'File no longer exists at original path',
+              })
+              .eq('id', file.id);
+            totalErrors++;
+            continue;
+          }
+
+          // Upload the file (handle both API and Supabase services)
+          let uploadResult;
+          if (uploadService.getServiceName() === 'Supabase') {
+            // Supabase requires single file upload with uploadPath
+            let uploadPath;
+            if (options.folderStructure && file.arela_path) {
+              // Combine folder structure with arela_path: palco/RFC/Year/Patente/Aduana/Pedimento/filename
+              uploadPath = `uploads/${options.folderStructure}/${file.arela_path}${file.filename}`;
+            } else if (file.arela_path) {
+              // Use existing arela_path: RFC/Year/Patente/Aduana/Pedimento/filename
+              uploadPath = `uploads/${file.arela_path}${file.filename}`;
+            } else {
+              // Fallback to RFC folder
+              uploadPath = `uploads/${file.rfc}/${file.filename}`;
+            }
+
+            uploadResult = await uploadService.upload(
+              [
+                {
+                  path: file.original_path,
+                  name: file.filename,
+                  contentType: 'application/octet-stream',
+                },
+              ],
+              {
+                uploadPath: uploadPath,
+              },
+            );
+            uploadResult = { success: true, data: uploadResult };
+          } else {
+            // API service supports batch uploads and returns normalized response
+            let fullFolderStructure;
+            if (options.folderStructure && file.arela_path) {
+              // Combine folder structure with arela_path: palco/RFC/Year/Patente/Aduana/Pedimento/
+              fullFolderStructure = `${options.folderStructure}/${file.arela_path}`;
+            } else if (file.arela_path) {
+              // Use existing arela_path: RFC/Year/Patente/Aduana/Pedimento/
+              fullFolderStructure = file.arela_path;
+            } else {
+              // Fallback to RFC folder
+              fullFolderStructure = `${file.rfc}/`;
+            }
+
+            uploadResult = await uploadService.upload(
+              [
+                {
+                  path: file.original_path,
+                  name: file.filename,
+                  contentType: 'application/octet-stream',
+                },
+              ],
+              {
+                folderStructure: fullFolderStructure,
+              },
+            );
+          }
+
+          if (uploadResult.success) {
+            // Update database status
+            await supabase
+              .from('uploader')
+              .update({
+                status: 'file-uploaded',
+                message: 'Successfully uploaded to Arela API',
+              })
+              .eq('id', file.id);
+
+            totalUploaded++;
+            logger.info(`Uploaded: ${file.filename}`);
+          } else {
+            await supabase
+              .from('uploader')
+              .update({
+                status: 'upload-error',
+                message: uploadResult.error || 'Upload failed',
+              })
+              .eq('id', file.id);
+
+            totalErrors++;
+            logger.error(
+              `Upload failed: ${file.filename} - ${uploadResult.error}`,
+            );
+          }
+        } catch (error) {
+          totalErrors++;
+          logger.error(
+            `Error processing file ${file.filename}: ${error.message}`,
+          );
+
+          await supabase
+            .from('uploader')
+            .update({
+              status: 'upload-error',
+              message: `Processing error: ${error.message}`,
+            })
+            .eq('id', file.id);
+        }
+      }
+    }
+
+    const result = {
+      processedCount: totalProcessed,
+      uploadedCount: totalUploaded,
+      errorCount: totalErrors,
+    };
+
+    logger.success(
+      `Phase 4 Summary: ${totalProcessed} files processed, ${totalUploaded} uploaded, ${totalErrors} errors`,
+    );
+
+    return result;
+  }
+
+  /**
+   * Get processed file paths from log
+   * @returns {Set<string>} Set of processed file paths
+   */
+  getProcessedPaths() {
+    // This would need to be adapted to work with the LoggingService
+    // For now, return empty set
+    return new Set();
+  }
+
+  /**
+   * Query files that are ready for upload
+   * These are files that have been detected but not yet uploaded
+   * Uses the same RFC filtering logic as uploadFilesByRfc for consistency
+   * @param {Object} options - Query options
+   * @returns {Promise<Array>} Array of files ready for upload
+   */
+  async getFilesReadyForUpload(options = {}) {
+    const supabase = await this.#getSupabaseClient();
+
+    logger.info('Querying files ready for upload...');
+    console.log('🔍 Querying files ready for upload...');
+
+    // Check if UPLOAD_RFCS is configured
+    const uploadRfcs = appConfig.upload.rfcs;
+    if (!uploadRfcs || uploadRfcs.length === 0) {
+      console.log(
+        'ℹ️  No UPLOAD_RFCS configured. Please set UPLOAD_RFCS environment variable to see files ready for upload.',
+      );
+      console.log(
+        '   Example: UPLOAD_RFCS="RFC123456789|RFC987654321|RFC555444333"',
+      );
+      return [];
+    }
+
+    console.log(`🎯 Using RFC filter: ${uploadRfcs.join(', ')}`);
+
+    // Step 1: Find pedimento_simplificado documents for the specified RFCs that have arela_path
+    console.log(
+      '🎯 Finding pedimento_simplificado documents for specified RFCs with arela_path...',
+    );
+    const { data: pedimentoRecords, error: pedimentoError } = await supabase
+      .from('uploader')
+      .select('arela_path')
+      .eq('document_type', 'pedimento_simplificado')
+      .in('rfc', uploadRfcs)
+      .not('arela_path', 'is', null);
+
+    if (pedimentoError) {
+      throw new Error(
+        `Error querying pedimento_simplificado records: ${pedimentoError.message}`,
+      );
+    }
+
+    if (!pedimentoRecords || pedimentoRecords.length === 0) {
+      console.log(
+        'ℹ️  No pedimento_simplificado records with arela_path found',
+      );
+      return [];
+    }
+
+    // Get unique arela_paths
+    const uniqueArelaPaths = [
+      ...new Set(pedimentoRecords.map((r) => r.arela_path)),
+    ];
+    console.log(
+      `📋 Found ${pedimentoRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths`,
+    );
+
+    // Step 2: Find all related files with these arela_paths that haven't been uploaded yet
+    console.log('🔍 Finding all related files that need to be uploaded...');
+
+    // Process arela_paths in chunks to avoid URI length limits
+    let allReadyFiles = [];
+    const chunkSize = 50;
+
+    for (let i = 0; i < uniqueArelaPaths.length; i += chunkSize) {
+      const pathChunk = uniqueArelaPaths.slice(i, i + chunkSize);
+
+      const { data: chunkFiles, error: chunkError } = await supabase
+        .from('uploader')
+        .select(
+          'id, original_path, arela_path, filename, rfc, document_type, status',
+        )
+        .in('arela_path', pathChunk)
+        .neq('status', 'file-uploaded')
+        .not('original_path', 'is', null);
+
+      if (chunkError) {
+        throw new Error(
+          `Error querying files for arela_paths chunk: ${chunkError.message}`,
+        );
+      }
+
+      if (chunkFiles && chunkFiles.length > 0) {
+        allReadyFiles = allReadyFiles.concat(chunkFiles);
+      }
+    }
+
+    const readyFiles = allReadyFiles;
+
+    console.log(`📋 Found ${readyFiles?.length || 0} files ready for upload`);
+
+    if (readyFiles && readyFiles.length > 0) {
+      // Group by document type for summary
+      const byDocType = readyFiles.reduce((acc, file) => {
+        const docType = file.document_type || 'Unknown';
+        acc[docType] = (acc[docType] || 0) + 1;
+        return acc;
+      }, {});
+
+      console.log('📊 Files by document type:');
+      for (const [docType, count] of Object.entries(byDocType)) {
+        console.log(`   ${docType}: ${count} files`);
+      }
+
+      // Group by RFC
+      const byRfc = readyFiles.reduce((acc, file) => {
+        const rfc = file.rfc || 'No RFC';
+        acc[rfc] = (acc[rfc] || 0) + 1;
+        return acc;
+      }, {});
+
+      console.log('📊 Files by RFC:');
+      for (const [rfc, count] of Object.entries(byRfc)) {
+        console.log(`   ${rfc}: ${count} files`);
+      }
+    }
+
+    return readyFiles || [];
+  }
+}
+
+// Export singleton instance
+export const databaseService = new DatabaseService();
+export default databaseService;