@arela/uploader 1.1.3 → 1.2.0

package/.claude/worktrees/agent-multi-profile/src/services/DatabaseService.js

@@ -0,0 +1,2173 @@
+import fs from 'fs';
+import path from 'path';
+
+import appConfig from '../config/config.js';
+import { FileDetectionService } from '../file-detection.js';
+import logger from './LoggingService.js';
+import uploadServiceFactory from './upload/UploadServiceFactory.js';
+
+/**
+ * Database Service
+ * Handles all Supabase database operations for the uploader table
+ */
+export class DatabaseService {
+  constructor() {
+    this.detectionService = new FileDetectionService();
+  }
+
+  /**
+   * Get Supabase client
+   * @private
+   * @returns {Promise<Object>} Supabase client
+   */
+  async #getSupabaseClient() {
+    const supabaseService = uploadServiceFactory.getSupabaseService();
+    return await supabaseService.getClient();
+  }
+
+  /**
+   * Get Supabase client (public wrapper for dependency injection)
+   * Used by specialized services like SmartWatchDatabaseService
+   * @returns {Promise<Object>} Supabase client
+   */
+  async getSupabaseClient() {
+    return this.#getSupabaseClient();
+  }
+
+  /**
+   * Execute database query with retry logic and exponential backoff
+   * @private
+   * @param {Function} queryFn - Query function to execute
+   * @param {string} operation - Description of the operation for logging
+   * @param {number} maxRetries - Maximum number of retry attempts (default: 3)
+   * @returns {Promise<Object>} Query result
+   */
+  async #queryWithRetry(queryFn, operation, maxRetries = 3) {
+    let lastError;
+
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+      try {
+        const result = await queryFn();
+        if (attempt > 1) {
+          logger.info(`${operation} succeeded on attempt ${attempt}`);
+        }
+        return result;
+      } catch (error) {
+        lastError = error;
+
+        // Check if it's a timeout or connection error
+        const isRetriableError =
+          error.message?.includes('timeout') ||
+          error.message?.includes('canceling statement') ||
+          error.message?.includes('connection') ||
+          error.message?.includes('fetch failed') ||
+          error.code === 'PGRST301'; // PostgREST timeout
+
+        if (!isRetriableError || attempt === maxRetries) {
+          throw error;
+        }
+
+        const backoffDelay = Math.min(1000 * Math.pow(2, attempt - 1), 30000); // Cap at 30 seconds
+        logger.warn(
+          `${operation} failed on attempt ${attempt}/${maxRetries}: ${error.message}`,
+        );
+        logger.info(`Retrying in ${backoffDelay}ms...`);
+
+        await new Promise((resolve) => setTimeout(resolve, backoffDelay));
+      }
+    }
+
+    throw lastError;
+  }
+
+  /**
+   * Execute database query with retry logic (public wrapper for dependency injection)
+   * Used by specialized services like SmartWatchDatabaseService
+   * @param {Function} queryFn - Query function to execute
+   * @param {string} operation - Description of the operation for logging
+   * @param {number} maxRetries - Maximum number of retry attempts (default: 3)
+   * @returns {Promise<Object>} Query result
+   */
+  async queryWithRetry(queryFn, operation, maxRetries = 3) {
+    return this.#queryWithRetry(queryFn, operation, maxRetries);
+  }
+
+  /**
+   * Insert file stats with document detection into uploader table
+   * @param {Array} files - Array of file objects
+   * @param {Object} options - Options including clientPath
+   * @returns {Promise<Array>} Inserted records
+   */
+  async insertStatsToUploaderTable(files, options) {
+    const supabase = await this.#getSupabaseClient();
+    const records = [];
+
+    for (const file of files) {
+      const stats = file.stats || fs.statSync(file.path);
+      const originalPath = options.clientPath || file.path;
+
+      // Check if record already exists
+      const { data: existingRecords, error: checkError } = await supabase
+        .from('uploader')
+        .select('id, original_path')
+        .eq('original_path', originalPath)
+        .limit(1);
+
+      if (checkError) {
+        logger.error(
+          `Error checking for existing record: ${checkError.message}`,
+        );
+        continue;
+      }
+
+      if (existingRecords && existingRecords.length > 0) {
+        logger.info(`Skipping duplicate: ${path.basename(file.path)}`);
+        continue;
+      }
+
+      // Initialize record with basic file stats
+      const fileExtension = path
+        .extname(file.path)
+        .toLowerCase()
+        .replace('.', '');
+      const filename = file.originalName || path.basename(file.path);
+
+      const record = {
+        name: filename,
+        document_type: null,
+        size: stats.size,
+        num_pedimento: null,
+        filename: filename,
+        original_path: originalPath,
+        arela_path: null,
+        status: 'stats',
+        rfc: null,
+        message: null,
+        file_extension: fileExtension,
+        // Flag any PDF whose filename hints at a pedimento (simplificado,
+        // completo, or CoveFact). Column name preserved; semantics broadened.
+        is_like_simplificado: /(simp|pedim|covefact)/i.test(filename),
+        year: null,
+        created_at: new Date().toISOString(),
+        updated_at: new Date().toISOString(),
+        modified_at: stats.mtime.toISOString(),
+        // Queue/Processing columns (for arela-api)
+        processing_status: 'PENDING',
+        upload_attempts: 0,
+      };
+
+      // Try to detect document type for supported files
+      if (this.detectionService.isSupportedFileType(file.path)) {
+        try {
+          const detection = await this.detectionService.detectFile(file.path);
+
+          if (detection.detectedType) {
+            record.document_type = detection.detectedType;
+            record.num_pedimento = detection.detectedPedimento;
+            record.status = 'detected';
+
+            if (detection.arelaPath) {
+              record.arela_path = detection.arelaPath;
+            }
+
+            if (detection.detectedPedimentoYear) {
+              record.year = detection.detectedPedimentoYear;
+            }
+
+            if (detection.rfc) {
+              record.rfc = detection.rfc;
+            }
+          } else {
+            record.status = 'not-detected';
+            if (detection.error) {
+              record.message = detection.error;
+            }
+          }
+        } catch (error) {
+          logger.error(`Error detecting ${record.filename}: ${error.message}`);
+          record.status = 'detection-error';
+          record.message = error.message;
+        }
+      } else {
+        record.status = 'unsupported';
+        record.message = 'File type not supported for detection';
+      }
+
+      records.push(record);
+    }
+
+    if (records.length === 0) {
+      logger.info('No new records to insert (all were duplicates or errors)');
+      return [];
+    }
+
+    logger.info(
+      `Inserting ${records.length} new records into uploader table...`,
+    );
+
+    // Use upsert to handle duplicates gracefully
+    // This will insert new records or update existing ones (by original_path)
+    const { data, error } = await supabase
+      .from('uploader')
+      .upsert(records, { onConflict: 'original_path' })
+      .select();
+
+    if (error) {
+      throw new Error(`Failed to insert stats records: ${error.message}`);
+    }
+
+    // Propagate arela_path to related files in same folder
+    if (data && data.length > 0) {
+      await this.#propagateArelaPathToRelatedFiles(data, supabase);
+    }
+
+    return data;
+  }
+
+  /**
+   * Propagate arela_path from pedimento files to related files in the same directory
+   * When a pedimento_simplificado is detected, all files in its directory get the same arela_path
+   * Also checks database for existing files in the same directory that need the arela_path
+   * @private
+   * @param {Array} insertedRecords - Records that were just inserted
+   * @param {Object} supabase - Supabase client
+   * @returns {Promise<void>}
+   */
+  async #propagateArelaPathToRelatedFiles(insertedRecords, supabase) {
+    try {
+      // Group records by directory
+      const recordsByDir = {};
+
+      for (const record of insertedRecords) {
+        if (!record.original_path) continue;
+
+        const dirPath = path.dirname(record.original_path);
+        if (!recordsByDir[dirPath]) {
+          recordsByDir[dirPath] = {
+            pedimentos: [],
+            allFiles: [],
+          };
+        }
+
+        recordsByDir[dirPath].allFiles.push(record);
+
+        // Identify pedimento files
+        if (
+          record.document_type === 'pedimento_simplificado' &&
+          record.arela_path
+        ) {
+          recordsByDir[dirPath].pedimentos.push(record);
+        }
+      }
+
+      // For each directory with a pedimento, propagate its arela_path to related files
+      for (const [dirPath, dirData] of Object.entries(recordsByDir)) {
+        if (dirData.pedimentos.length === 0) continue;
+
+        // Use the first pedimento's arela_path (should be only one per directory typically)
+        const pedimentoRecord = dirData.pedimentos[0];
+        const arelaPath = pedimentoRecord.arela_path;
+
+        logger.info(
+          `📁 Propagating arela_path from pedimento to files in ${path.basename(dirPath)}/`,
+        );
+
+        // Step 1: Update newly inserted files in this directory
+        const fileIds = dirData.allFiles
+          .filter(
+            (f) =>
+              f.id &&
+              f.arela_path !== arelaPath &&
+              f.document_type !== 'pedimento_simplificado',
+          )
+          .map((f) => f.id);
+
+        if (fileIds.length > 0) {
+          const { error: updateError } = await supabase
+            .from('uploader')
+            .update({ arela_path: arelaPath })
+            .in('id', fileIds);
+
+          if (updateError) {
+            logger.warn(
+              `Could not propagate arela_path: ${updateError.message}`,
+            );
+          } else {
+            logger.info(
+              `✅ Updated ${fileIds.length} related files with arela_path: ${arelaPath}`,
+            );
+          }
+        }
+
+        // Step 2: Also find and update any existing files in the same directory that don't have arela_path
+        // This handles the case where files were detected earlier but the pedimento is detected now
+        try {
+          const dirPattern = dirPath.replace(/\\/g, '/'); // Normalize for SQL LIKE
+          const { data: existingFiles, error: fetchError } = await supabase
+            .from('uploader')
+            .select('id, original_path, document_type')
+            .like('original_path', `${dirPattern}/%`)
+            .is('arela_path', null)
+            .limit(1000); // Reasonable limit to avoid huge queries
+
+          if (fetchError) {
+            logger.warn(
+              `Could not fetch existing files in ${path.basename(dirPath)}: ${fetchError.message}`,
+            );
+          } else if (existingFiles && existingFiles.length > 0) {
+            const existingFileIds = existingFiles
+              .filter(
+                (f) => f.id && f.document_type !== 'pedimento_simplificado',
+              )
+              .map((f) => f.id);
+
+            if (existingFileIds.length > 0) {
+              const { error: existingError } = await supabase
+                .from('uploader')
+                .update({ arela_path: arelaPath })
+                .in('id', existingFileIds);
+
+              if (existingError) {
+                logger.warn(
+                  `Could not update existing files with arela_path: ${existingError.message}`,
+                );
+              } else {
+                logger.info(
+                  `✅ Updated ${existingFileIds.length} existing files in directory with arela_path: ${arelaPath}`,
+                );
+              }
+            }
+          }
+        } catch (existingFilesError) {
+          logger.warn(
+            `Error checking for existing files in ${path.basename(dirPath)}: ${existingFilesError.message}`,
+          );
+        }
+      }
+    } catch (error) {
+      logger.warn(
+        `Error propagating arela_path to related files: ${error.message}`,
+      );
+      // Don't throw - this is a non-critical operation
+    }
+  }
+
+  /**
+   * Insert file stats only (no detection) into uploader table
+   * @param {Array} files - Array of file objects
+   * @param {Object} options - Options including clientPath
+   * @returns {Promise<Object>} Statistics about the operation
+   */
+  async insertStatsOnlyToUploaderTable(files, options) {
+    const batchSize = 1000;
+    const quietMode = options?.quietMode || false;
+    const allRecords = [];
+
+    if (!quietMode) {
+      logger.info('Collecting filesystem stats...');
+    }
+
+    // Filter out system files and hidden files (macOS, Windows, Python, editors)
+    const systemFilePattern =
+      /^\.|__pycache__|\.pyc|\.swp|\.swo|Thumbs\.db|desktop\.ini|DS_Store|\$RECYCLE\.BIN|System Volume Information|~\$|\.tmp/i;
+
+    for (const file of files) {
+      try {
+        const fileName = file.originalName || path.basename(file.path);
+
+        // Skip system and hidden files
+        if (systemFilePattern.test(fileName)) {
+          if (!quietMode) {
+            logger.debug(`Skipping system file: ${fileName}`);
+          }
+          continue;
+        }
+
+        const stats = file.stats || fs.statSync(file.path);
+        const originalPath = options.clientPath || file.path;
+        const fileExtension = path
+          .extname(file.path)
+          .toLowerCase()
+          .replace('.', '');
+
+        const record = {
+          name: file.originalName || path.basename(file.path),
+          documentType: null,
+          size: stats.size,
+          numPedimento: null,
+          filename: file.originalName || path.basename(file.path),
+          originalPath: originalPath,
+          arelaPath: null,
+          status: 'fs-stats',
+          rfc: null,
+          message: null,
+          fileExtension: fileExtension,
+          modifiedAt: stats.mtime.toISOString(),
+          isLikeSimplificado:
+            fileExtension === 'pdf' &&
+            (file.originalName || path.basename(file.path))
+              .toLowerCase()
+              .includes('simp'),
+          year: null,
+          // Queue/Processing columns (for arela-api)
+          processingStatus: 'PENDING',
+          uploadAttempts: 0,
+        };
+
+        allRecords.push(record);
+      } catch (error) {
+        logger.error(`Error reading stats for ${file.path}: ${error.message}`);
+      }
+    }
+
+    if (allRecords.length === 0) {
+      if (!quietMode) {
+        logger.info('No file stats to insert');
+      }
+      return { totalInserted: 0, totalSkipped: 0, totalProcessed: 0 };
+    }
+
+    if (!quietMode) {
+      logger.info(
+        `Processing ${allRecords.length} file stats in batches of ${batchSize}...`,
+      );
+    }
+
+    let totalInserted = 0;
+    let totalUpdated = 0;
+
+    // Use API service for batch upsert
+    // If apiTarget is specified, use that specific API, otherwise use default
+    let uploadService;
+    if (options.apiTarget) {
+      uploadService = await uploadServiceFactory.getApiServiceForTarget(
+        options.apiTarget,
+      );
+    } else {
+      uploadService = await uploadServiceFactory.getUploadService(
+        options.forceSupabase,
+      );
+    }
+
+    // Use API service for batch upsert
+    for (let i = 0; i < allRecords.length; i += batchSize) {
+      const batch = allRecords.slice(i, i + batchSize);
+
+      try {
+        const result = await uploadService.batchUpsertStats(batch);
+
+        totalInserted += result.inserted || 0;
+        totalUpdated += result.updated || 0;
+
+        // Only log every 10th batch to reduce noise (skip in quiet mode)
+        if (
+          !quietMode &&
+          ((Math.floor(i / batchSize) + 1) % 10 === 0 ||
+            Math.floor(i / batchSize) + 1 === 1)
+        ) {
+          logger.info(
+            `Batch ${Math.floor(i / batchSize) + 1}: ${result.inserted || 0} new, ${result.updated || 0} updates`,
+          );
+        }
+      } catch (error) {
+        logger.error(
+          `Error in batch ${Math.floor(i / batchSize) + 1}: ${error.message}`,
+        );
+      }
+    }
+
+    if (!quietMode) {
+      logger.success(
+        `Phase 1 Summary: ${totalInserted} new records inserted, ${totalUpdated} existing records updated`,
+      );
+    }
+
+    return {
+      totalInserted,
+      totalSkipped: totalUpdated,
+      totalProcessed: allRecords.length,
+    };
+  }
+
+  /**
+   * Process PDF files for pedimento-simplificado detection
+   * @param {Object} options - Processing options
+   * @returns {Promise<Object>} Processing result
+   */
+  async detectPedimentosInDatabase(options = {}) {
+    logger.info(
+      'Phase 2: Starting PDF detection for pedimento-simplificado documents...',
+    );
+
+    const processingBatchSize = parseInt(options.batchSize) || 10;
+    // Query batch size for each API call
+    const queryBatchSize = 100;
+
+    let totalDetected = 0;
+    let totalProcessed = 0;
+    let totalErrors = 0;
+    let offset = 0;
+    let chunkNumber = 1;
+
+    // Get API service - use specific target if provided
+    let apiService;
+    if (options.apiTarget) {
+      apiService = await uploadServiceFactory.getApiServiceForTarget(
+        options.apiTarget,
+      );
+      logger.info(`Using API target: ${options.apiTarget}`);
+    } else {
+      apiService = await uploadServiceFactory.getUploadService();
+    }
+
+    if (apiService.getServiceName() !== 'Arela API') {
+      throw new Error(
+        'API service is required for PDF detection. Please configure ARELA_API_URL and ARELA_API_TOKEN.',
+      );
+    }
+
+    logger.info('Using API service for PDF detection...');
+
+    logger.info(
+      `Processing PDF files in chunks of ${queryBatchSize} records...`,
+    );
+
+    while (true) {
+      logger.info(
+        `Fetching chunk ${chunkNumber} (records ${offset + 1} to ${offset + queryBatchSize})...`,
+      );
+
+      try {
+        // Fetch records using API
+        const { data: pdfRecords, error: queryError } =
+          await apiService.fetchPdfRecordsForDetection({
+            offset,
+            limit: queryBatchSize,
+          });
+
+        if (queryError) {
+          throw new Error(
+            `Failed to fetch PDF records chunk ${chunkNumber}: ${queryError.message}`,
+          );
+        }
+
+        if (!pdfRecords || pdfRecords.length === 0) {
+          logger.info('No more PDF files found. Processing completed.');
+          break;
+        }
+
+        logger.info(
+          `Processing chunk ${chunkNumber}: ${pdfRecords.length} PDF records`,
+        );
+
+        let chunkDetected = 0;
+        let chunkProcessed = 0;
+        let chunkErrors = 0;
+
+        // Process files in smaller batches
+        const batchUpdates = [];
+
+        for (let i = 0; i < pdfRecords.length; i += processingBatchSize) {
+          const batch = pdfRecords.slice(i, i + processingBatchSize);
+
+          for (const record of batch) {
+            try {
+              if (!fs.existsSync(record.original_path)) {
+                logger.warn(
+                  `File not found: ${record.filename} at ${record.original_path}`,
+                );
+                batchUpdates.push({
+                  id: record.id,
+                  status: 'file-not-found',
+                  message: 'File no longer exists at original path',
+                });
+                chunkErrors++;
+                totalErrors++;
+                continue;
+              }
+
+              const detection = await this.detectionService.detectFile(
+                record.original_path,
+              );
+              chunkProcessed++;
+              totalProcessed++;
+
+              const updateData = {
+                id: record.id,
+                status: detection.detectedType ? 'detected' : 'not-detected',
+                documentType: detection.detectedType,
+                numPedimento: detection.detectedPedimento,
+                arelaPath: detection.arelaPath,
+                message: detection.error || null,
+                year: detection.detectedPedimentoYear || null,
+              };
+
+              if (detection.rfc) {
+                updateData.rfc = detection.rfc;
+              }
+
+              if (detection.detectedType) {
+                chunkDetected++;
+                totalDetected++;
+                logger.success(
+                  `Detected: ${record.filename} -> ${detection.detectedType} | Pedimento: ${detection.detectedPedimento || 'N/A'} | RFC: ${updateData.rfc || 'N/A'}`,
+                );
+              } else {
+                logger.info(
+                  `Not detected: ${record.filename} - No pedimento-simplificado pattern found`,
+                );
+              }
+
+              batchUpdates.push(updateData);
+            } catch (error) {
+              logger.error(
+                `Error detecting ${record.filename}: ${error.message}`,
+              );
+              chunkErrors++;
+              totalErrors++;
+
+              batchUpdates.push({
+                id: record.id,
+                status: 'detection-error',
+                message: error.message,
+              });
+            }
+          }
+        }
+
+        // Batch update using API
+        try {
+          if (batchUpdates.length > 0) {
+            const updateResult =
+              await apiService.batchUpdateDetectionResults(batchUpdates);
+            if (!updateResult.success) {
+              logger.error(
+                `Some updates failed in chunk ${chunkNumber}: ${updateResult.errors?.length || 0} errors`,
+              );
+            }
+          }
+        } catch (error) {
+          logger.error(
+            `Error updating batch in chunk ${chunkNumber}: ${error.message}`,
+          );
+        }
+
+        logger.success(
+          `Chunk ${chunkNumber} completed: ${chunkDetected} detected, ${chunkProcessed} processed, ${chunkErrors} errors`,
+        );
+
+        offset += queryBatchSize;
+        chunkNumber++;
+
+        if (pdfRecords.length < queryBatchSize) {
+          logger.info(
+            `Reached end of records (chunk had ${pdfRecords.length} records).`,
+          );
+          break;
+        }
+
+        // Small delay between chunks
+        await new Promise((resolve) => setTimeout(resolve, 500));
+      } catch (chunkError) {
+        logger.error(
+          `Error processing chunk ${chunkNumber}: ${chunkError.message}`,
+        );
+        // Continue to next chunk after error
+        offset += queryBatchSize;
+        chunkNumber++;
+        totalErrors++;
+        continue;
+      }
+    }
+
+    const result = {
+      detectedCount: totalDetected,
+      processedCount: totalProcessed,
+      errorCount: totalErrors,
+    };
+
+    logger.success(
+      `Phase 2 Summary: ${totalDetected} detected, ${totalProcessed} processed, ${totalErrors} errors`,
+    );
+
+    return result;
+  }
+
+  /**
+   * Propagate arela_path from pedimento_simplificado records to related files
+   * This operation is performed entirely on the backend for efficiency
+   * @param {Object} options - Options for propagation
+   * @returns {Promise<Object>} Processing result
+   */
+  async propagateArelaPath(options = {}) {
+    logger.info('Phase 3: Starting arela_path and year propagation process...');
+    console.log('🔍 Triggering backend propagation process...');
+
+    // Get API service - use specific target if provided
+    let apiService;
+    if (options.apiTarget) {
+      apiService = await uploadServiceFactory.getApiServiceForTarget(
+        options.apiTarget,
+      );
+      logger.info(`Using API target: ${options.apiTarget}`);
+    } else {
+      apiService = await uploadServiceFactory.getUploadService();
+    }
+
+    if (apiService.getServiceName() !== 'Arela API') {
+      throw new Error(
+        'API service is required for arela_path propagation. Please configure ARELA_API_URL and ARELA_API_TOKEN.',
+      );
+    }
+
+    logger.info('Using API service for arela_path propagation...');
+
+    // Log year filtering configuration
+    const years = appConfig.upload.years || [];
+    if (years.length > 0) {
+      logger.info(`🗓️  Year filter enabled: ${years.join(', ')}`);
+      console.log(`🗓️  Year filter enabled: ${years.join(', ')}`);
+    } else {
+      logger.info('🗓️  No year filter configured - processing all years');
+      console.log('🗓️  No year filter configured - processing all years');
+    }
+
+    console.log('⏳ Processing on backend... This may take a moment.');
+
+    // Trigger backend propagation - all logic runs server-side
+    const result = await apiService.propagateArelaPath({ years });
+
+    if (!result.success) {
+      const errorMsg = `Backend propagation failed: ${result.error || 'Unknown error'}`;
+      logger.error(errorMsg);
+      throw new Error(errorMsg);
+    }
+
+    // Display results
+    const { processedCount = 0, updatedCount = 0, errorCount = 0 } = result;
+
+    if (processedCount === 0) {
+      logger.info('No pedimento_simplificado records with arela_path found');
+      console.log(
+        'ℹ️  No pedimento_simplificado records with arela_path found',
+      );
+    } else {
+      console.log(`📋 Processed ${processedCount} pedimento records`);
+      logger.info(`Processed ${processedCount} pedimento records`);
+    }
+
+    logger.success(
+      `Phase 3 Summary: ${processedCount} pedimentos processed, ${updatedCount} files updated with arela_path and year, ${errorCount} errors`,
+    );
+
+    return {
+      processedCount,
+      updatedCount,
+      errorCount,
+    };
+  }
+
+  /**
+   * Upload files to Arela API based on specific RFC values
+   * Supports cross-tenant mode where source and target APIs can be different
+   * @param {Object} options - Upload options
+   * @param {string} options.sourceApi - Source API target for reading data (cross-tenant mode)
+   * @param {string} options.targetApi - Target API target for uploading files (cross-tenant mode)
+   * @param {string} options.apiTarget - Single API target for both reading and uploading (single API mode)
+   * @returns {Promise<Object>} Processing result
+   */
+  async uploadFilesByRfc(options = {}) {
+    // Get configuration
+    const appConfig = await import('../config/config.js').then(
+      (m) => m.appConfig,
+    );
+
+    // Determine if we're in cross-tenant mode
+    const isCrossTenant =
+      options.sourceApi &&
+      options.targetApi &&
+      options.sourceApi !== options.targetApi;
+
+    // Determine if we're in single API mode with specific target
+    const isSingleApiMode = !isCrossTenant && options.apiTarget;
+
+    let sourceService, targetService;
+
+    if (isCrossTenant) {
+      console.log('🔀 Cross-tenant upload mode enabled');
+      console.log(`   📖 Source API: ${options.sourceApi}`);
+      console.log(`   📝 Target API: ${options.targetApi}`);
+
+      // Get separate services for source and target
+      sourceService = await uploadServiceFactory.getApiServiceForTarget(
+        options.sourceApi,
+      );
+      targetService = await uploadServiceFactory.getApiServiceForTarget(
+        options.targetApi,
+      );
+
+      // Verify both services are available
+      if (!(await sourceService.isAvailable())) {
+        throw new Error(`Source API '${options.sourceApi}' is not available`);
+      }
+      if (!(await targetService.isAvailable())) {
+        throw new Error(`Target API '${options.targetApi}' is not available`);
+      }
+
+      console.log(`✅ Connected to source: ${sourceService.baseUrl}`);
+      console.log(`✅ Connected to target: ${targetService.baseUrl}`);
+    } else if (isSingleApiMode) {
+      // Single API mode with specific target - use the same service for both
+      console.log(`🎯 Single API mode: ${options.apiTarget}`);
+
+      const apiService = await uploadServiceFactory.getApiServiceForTarget(
+        options.apiTarget,
+      );
+
+      if (!(await apiService.isAvailable())) {
+        throw new Error(`API '${options.apiTarget}' is not available`);
+      }
+
+      console.log(`✅ Connected to: ${apiService.baseUrl}`);
+      sourceService = apiService;
+      targetService = apiService;
+    } else {
+      // Default mode - use the default service for both
+      const apiService = await uploadServiceFactory.getUploadService();
+
+      if (apiService.getServiceName() !== 'Arela API') {
+        throw new Error(
+          'API service is required for RFC-based upload. Please configure ARELA_API_URL and ARELA_API_TOKEN.',
+        );
+      }
+
+      sourceService = apiService;
+      targetService = apiService;
+    }
+
+    if (!appConfig.upload.rfcs || appConfig.upload.rfcs.length === 0) {
+      const errorMsg =
+        'No RFCs specified. Please set UPLOAD_RFCS environment variable with pipe-separated RFC values.';
+      logger.error(errorMsg);
+      throw new Error(errorMsg);
+    }
+
+    logger.info('Phase 4: Starting RFC-based upload process...');
+    logger.info(
+      `Using ${isCrossTenant ? 'cross-tenant' : 'standard'} API service for RFC-based upload...`,
+    );
+    console.log('🎯 RFC-based Upload Mode');
+    console.log(`📋 Target RFCs: ${appConfig.upload.rfcs.join(', ')}`);
+    console.log('🔍 Searching for files to upload...');
+
+    // First, count total files for the RFCs to show filtering effect
+    const { count: totalRfcFiles, error: countError } =
+      await sourceService.fetchRfcFileCount({
+        rfcs: appConfig.upload.rfcs,
+      });
+
+    if (countError) {
+      logger.warn(`Could not count total RFC files: ${countError.message}`);
+    } else {
+      console.log(`📊 Total files for specified RFCs: ${totalRfcFiles || 0}`);
+      logger.info(`Total files for specified RFCs: ${totalRfcFiles || 0}`);
+    }
+
+    // Step 1: Get all pedimento_simplificado records that match the specified RFCs and have arela_path using pagination
+    console.log(
+      '🎯 Finding pedimento_simplificado records for specified RFCs...',
+    );
+
+    let allPedimentoRecords = [];
+    let offset = 0;
+    const pageSize = 500; // Process pedimento records in pages
+    let hasMorePedimentoData = true;
+    let pageNumber = 1;
+
+    // Process pedimento records page by page for memory efficiency
+    while (hasMorePedimentoData) {
+      logger.info(
+        `Fetching pedimento records page ${pageNumber} (records ${offset + 1} to ${offset + pageSize})...`,
+      );
+
+      const { data: pedimentoPage, error: pedimentoError } =
+        await sourceService.fetchPedimentosByRfc({
+          rfcs: appConfig.upload.rfcs,
+          years: appConfig.upload.years || [],
+          offset,
+          limit: pageSize,
+        });
+
+      if (pedimentoError) {
+        const errorMsg = `Error fetching pedimento RFC records page ${pageNumber}: ${pedimentoError.message}`;
+        logger.error(errorMsg);
+        throw new Error(errorMsg);
+      }
+
+      if (!pedimentoPage || pedimentoPage.length === 0) {
+        hasMorePedimentoData = false;
+        logger.info('No more pedimento records found');
+        break;
+      }
+
+      logger.info(
+        `Processing pedimento page ${pageNumber}: ${pedimentoPage.length} pedimento records`,
+      );
+
+      // Add this page to our collection
+      allPedimentoRecords = allPedimentoRecords.concat(pedimentoPage);
+
+      // Check if we need to fetch the next page
+      if (pedimentoPage.length < pageSize) {
+        hasMorePedimentoData = false;
+        logger.info(
+          `Completed fetching pedimento records. Last page ${pageNumber} had ${pedimentoPage.length} records`,
+        );
+      } else {
+        offset += pageSize;
+        pageNumber++;
+        logger.info(
+          `Page ${pageNumber - 1} complete: ${pedimentoPage.length} records fetched, moving to next page...`,
+        );
+      }
+    }
+
+    if (!allPedimentoRecords || allPedimentoRecords.length === 0) {
+      console.log(
+        'ℹ️  No pedimento_simplificado records found for the specified RFCs with arela_path',
+      );
+      logger.info('No pedimento_simplificado records found for specified RFCs');
+      return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
+    }
+
+    // Get unique arela_paths from all pedimento records
+    const uniqueArelaPaths = [
+      ...new Set(allPedimentoRecords.map((r) => r.arela_path)),
+    ];
+    // pageNumber represents the current page (starts at 1)
+    const totalPages = pageNumber;
+    console.log(
+      `📋 Found ${allPedimentoRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths for specified RFCs across ${totalPages} page${totalPages !== 1 ? 's' : ''}`,
+    );
+    logger.info(
+      `Found ${allPedimentoRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths across ${totalPages} page${totalPages !== 1 ? 's' : ''}`,
+    );
+
+    // Step 2: Process files with optimized single query per chunk
+    let totalProcessed = 0;
+    let totalUploaded = 0;
+    let totalErrors = 0;
+    let globalFileCount = 0;
+    const arelaPathChunkSize = 50;
+    const batchSize = parseInt(options.batchSize) || 10;
+    const filePageSize = 1000; // Supabase limit per request
+
+    // Import performance configuration
+    const { performance: perfConfig } = appConfig;
+    const maxConcurrency = perfConfig?.maxApiConnections || 3;
+
+    console.log('📥 Processing files in chunks to avoid URI limits...');
+
+    // Process arela_paths in chunks and upload files as we fetch them
+    for (let i = 0; i < uniqueArelaPaths.length; i += arelaPathChunkSize) {
+      const arelaPathChunk = uniqueArelaPaths.slice(i, i + arelaPathChunkSize);
+      const chunkNumber = Math.floor(i / arelaPathChunkSize) + 1;
+      const totalChunks = Math.ceil(
+        uniqueArelaPaths.length / arelaPathChunkSize,
+      );
+
+      console.log(
+        `   Processing arela_path chunk ${chunkNumber}/${totalChunks} (${arelaPathChunk.length} paths)`,
+      );
+
+      // Fetch all files for this chunk with pagination to handle >1000 records
+      let allChunkFiles = [];
+      let fileOffset = 0;
+      let hasMoreFiles = true;
+      let filePageNum = 1;
+
+      while (hasMoreFiles) {
+        const { data: batch, error: queryError } =
+          await sourceService.fetchFilesForUpload({
+            arelaPaths: arelaPathChunk,
+            offset: fileOffset,
+            limit: filePageSize,
+          });
+
+        if (queryError) {
+          const errorMsg = `Error fetching files for chunk ${chunkNumber} page ${filePageNum}: ${queryError.message}`;
+          logger.error(errorMsg);
+          throw new Error(errorMsg);
+        }
+
+        if (!batch || batch.length === 0) {
+          hasMoreFiles = false;
+          if (filePageNum === 1) {
+            // No files found at all for this chunk
+            console.log(
+              `   ℹ️  Chunk ${chunkNumber}/${totalChunks}: No files to upload`,
+            );
+          }
+          break;
+        }
+
+        logger.debug(
+          `Chunk ${chunkNumber} page ${filePageNum}: fetched ${batch.length} files`,
+        );
+        allChunkFiles = allChunkFiles.concat(batch);
+
+        // Check if we need more pages
+        if (batch.length < filePageSize) {
+          hasMoreFiles = false;
+          logger.debug(
+            `Chunk ${chunkNumber}: Completed pagination with ${allChunkFiles.length} total files`,
+          );
+        } else {
+          fileOffset += filePageSize;
+          filePageNum++;
+        }
+      }
+
+      if (allChunkFiles.length === 0) {
+        continue;
+      }
+
+      const chunkFileCount = allChunkFiles.length;
+      globalFileCount += chunkFileCount;
+
+      console.log(
+        `   📦 Chunk ${chunkNumber}/${totalChunks}: Processing ${chunkFileCount} files`,
+      );
+
+      // Process this batch of files immediately using concurrent processing
+      // Split batch into upload batches
+      for (let j = 0; j < allChunkFiles.length; j += batchSize) {
+        const uploadBatch = allChunkFiles.slice(j, j + batchSize);
+        const batchNum = Math.floor(j / batchSize) + 1;
+        const totalBatches = Math.ceil(allChunkFiles.length / batchSize);
+
+        console.log(
+          `   📦 Processing upload batch ${batchNum}/${totalBatches} within chunk ${chunkNumber} (${uploadBatch.length} files)`,
+        );
+
+        // Process batch using concurrent processing similar to UploadCommand
+        // In cross-tenant mode: targetService for uploading, sourceService for reading
+        const batchResults = await this.#processRfcBatch(
+          uploadBatch,
+          targetService, // Used for uploading files
+          sourceService, // Used for reading metadata
+          options,
+          maxConcurrency,
+        );
+
+        totalProcessed += batchResults.processed;
+        totalUploaded += batchResults.uploaded;
+        totalErrors += batchResults.errors;
+
+        console.log(
+          `   📊 Batch complete - Progress: ${totalUploaded} uploaded, ${totalErrors} errors`,
+        );
+      }
+
+      console.log(
+        `   ✅ Chunk ${chunkNumber}/${totalChunks} complete: ${chunkFileCount} files processed`,
+      );
+    }
+
+    if (globalFileCount === 0) {
+      console.log('ℹ️  No related files found to upload');
+      logger.info('No related files found to upload');
+      return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
+    }
+
+    console.log(`📋 Total files processed: ${globalFileCount}`);
+    logger.info(`Total files processed: ${globalFileCount}`);
+
+    const result = {
+      processedCount: totalProcessed,
+      uploadedCount: totalUploaded,
+      errorCount: totalErrors,
+    };
+
+    logger.success(
+      `Phase 4 Summary: ${totalProcessed} files processed, ${totalUploaded} uploaded, ${totalErrors} errors`,
+    );
+
+    return result;
+  }
+
+  /**
+   * Get processed file paths from log
+   * @returns {Set<string>} Set of processed file paths
+   */
+  getProcessedPaths() {
+    // This would need to be adapted to work with the LoggingService
+    // For now, return empty set
+    return new Set();
+  }
+
+  /**
+   * Query files that are ready for upload
+   * These are files that have been detected but not yet uploaded
+   * Uses the same RFC filtering logic as uploadFilesByRfc for consistency
+   * @param {Object} options - Query options
+   * @returns {Promise<Array>} Array of files ready for upload
+   */
+  async getFilesReadyForUpload(options = {}) {
+    // Get API service
+    const apiService = await uploadServiceFactory.getUploadService();
+
+    if (apiService.getServiceName() !== 'Arela API') {
+      throw new Error(
+        'API service is required for querying files. Please configure ARELA_API_URL and ARELA_API_TOKEN.',
+      );
+    }
+
+    logger.info('Querying files ready for upload...');
+    console.log('🔍 Querying files ready for upload...');
+
+    // Check if UPLOAD_RFCS is configured
+    const uploadRfcs = appConfig.upload.rfcs;
+    if (!uploadRfcs || uploadRfcs.length === 0) {
+      console.log(
+        'ℹ️  No UPLOAD_RFCS configured. Please set UPLOAD_RFCS environment variable to see files ready for upload.',
+      );
+      console.log(
+        '   Example: UPLOAD_RFCS="RFC123456789|RFC987654321|RFC555444333"',
+      );
+      return [];
+    }
+
+    console.log(`🎯 Using RFC filter: ${uploadRfcs.join(', ')}`);
+
+    // Step 1: Find pedimento_simplificado documents for the specified RFCs that have arela_path
+    console.log(
+      '🎯 Finding pedimento_simplificado documents for specified RFCs with arela_path...',
+    );
+
+    const { data: pedimentoRecords, error: pedimentoError } =
+      await apiService.fetchPedimentosByRfc({
+        rfcs: uploadRfcs,
+        years: appConfig.upload.years || [],
+        offset: 0,
+        limit: 10000, // Fetch all pedimentos in one go for this query
+      });
+
+    if (pedimentoError) {
+      throw new Error(
+        `Error querying pedimento_simplificado records: ${pedimentoError.message}`,
+      );
+    }
+
+    if (!pedimentoRecords || pedimentoRecords.length === 0) {
+      console.log(
+        'ℹ️  No pedimento_simplificado records with arela_path found',
+      );
+      return [];
+    }
+
+    // Get unique arela_paths
+    const uniqueArelaPaths = [
+      ...new Set(pedimentoRecords.map((r) => r.arela_path)),
+    ];
+    console.log(
+      `📋 Found ${pedimentoRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths`,
+    );
+
+    // Step 2: Find all related files with these arela_paths that haven't been uploaded yet
+    console.log('🔍 Finding all related files that need to be uploaded...');
+
+    // Process arela_paths in chunks to avoid URI length limits
+    let allReadyFiles = [];
+    const chunkSize = 50;
+
+    for (let i = 0; i < uniqueArelaPaths.length; i += chunkSize) {
+      const pathChunk = uniqueArelaPaths.slice(i, i + chunkSize);
+
+      // Query with pagination to get all results
+      let chunkFiles = [];
+      let from = 0;
+      const pageSize = 1000;
+      let hasMoreData = true;
+
+      while (hasMoreData) {
+        const { data: pageData, error: chunkError } =
+          await apiService.fetchFilesForUpload({
+            arelaPaths: pathChunk,
+            offset: from,
+            limit: pageSize,
+          });
+
+        if (chunkError) {
+          throw new Error(
+            `Error querying files for arela_paths chunk: ${chunkError.message}`,
+          );
+        }
+
+        if (pageData && pageData.length > 0) {
+          chunkFiles = chunkFiles.concat(pageData);
+
+          // Check if we got a full page, indicating there might be more data
+          if (pageData.length < pageSize) {
+            hasMoreData = false;
+          } else {
+            from += pageSize;
+          }
+        } else {
+          hasMoreData = false;
+        }
+      }
+
+      if (chunkFiles.length > 0) {
+        allReadyFiles = allReadyFiles.concat(chunkFiles);
+      }
+    }
+
+    const readyFiles = allReadyFiles;
+
+    console.log(`📋 Found ${readyFiles?.length || 0} files ready for upload`);
+
+    if (readyFiles && readyFiles.length > 0) {
+      // Group by document type for summary
+      const byDocType = readyFiles.reduce((acc, file) => {
+        const docType = file.document_type || 'Unknown';
+        acc[docType] = (acc[docType] || 0) + 1;
+        return acc;
+      }, {});
+
+      console.log('📊 Files by document type:');
+      for (const [docType, count] of Object.entries(byDocType)) {
+        console.log(`   ${docType}: ${count} files`);
+      }
+
+      // Group by RFC
+      const byRfc = readyFiles.reduce((acc, file) => {
+        const rfc = file.rfc || 'No RFC';
+        acc[rfc] = (acc[rfc] || 0) + 1;
+        return acc;
+      }, {});
+
+      console.log('📊 Files by RFC:');
+      for (const [rfc, count] of Object.entries(byRfc)) {
+        console.log(`   ${rfc}: ${count} files`);
+      }
+    }
+
+    return readyFiles || [];
+  }
+
+  /**
+   * Process a batch of files using concurrent processing for RFC uploads
+   * @param {Array} files - Files to process in this batch
+   * @param {Object} uploadService - Upload service instance
+   * @param {Object} apiService - API service instance for database updates
+   * @param {Object} options - Upload options
+   * @param {number} maxConcurrency - Maximum concurrent operations
+   * @returns {Promise<Object>} Batch processing results
+   */
+  async #processRfcBatch(
+    files,
+    uploadService,
+    apiService,
+    options,
+    maxConcurrency,
+  ) {
+    const fs = (await import('fs')).default;
+
+    let processed = 0;
+    let uploaded = 0;
+    let errors = 0;
+
+    // For Supabase, process files individually (required by service)
+    if (uploadService.getServiceName() === 'Supabase') {
+      // Process files in concurrent chunks within the batch
+      const chunks = [];
+      for (let i = 0; i < files.length; i += maxConcurrency) {
+        chunks.push(files.slice(i, i + maxConcurrency));
+      }
+
+      // Process each chunk concurrently
+      for (const chunk of chunks) {
+        const chunkPromises = chunk.map(async (file) => {
+          return await this.#processRfcSingleFile(
+            file,
+            uploadService,
+            apiService,
+            options,
+            fs,
+          );
+        });
+
+        // Wait for all files in this chunk to complete
+        const chunkResults = await Promise.allSettled(chunkPromises);
+
+        // Count results
+        for (const result of chunkResults) {
+          processed++;
+          if (result.status === 'fulfilled' && result.value.success) {
+            uploaded++;
+          } else {
+            errors++;
+          }
+        }
+      }
+    } else {
+      // For API service, use true batch processing (multiple files per API call)
+      const apiChunks = [];
+      const apiChunkSize = Math.min(
+        5,
+        Math.ceil(files.length / maxConcurrency),
+      ); // 5 files per API call, or distribute evenly
+
+      for (let i = 0; i < files.length; i += apiChunkSize) {
+        apiChunks.push(files.slice(i, i + apiChunkSize));
+      }
+
+      console.log(
+        `   🚀 Processing ${apiChunks.length} API calls with ${apiChunkSize} files each (max ${maxConcurrency} concurrent)`,
+      );
+
+      // Process API chunks with controlled concurrency
+      const concurrentChunks = [];
+      for (let i = 0; i < apiChunks.length; i += maxConcurrency) {
+        concurrentChunks.push(apiChunks.slice(i, i + maxConcurrency));
+      }
+
+      for (const concurrentSet of concurrentChunks) {
+        const batchPromises = concurrentSet.map(async (chunk) => {
+          return await this.#processRfcApiBatch(
+            chunk,
+            uploadService,
+            apiService,
+            options,
+            fs,
+          );
+        });
+
+        // Wait for all concurrent batches to complete
+        const batchResults = await Promise.allSettled(batchPromises);
+
+        // Count results
+        for (const result of batchResults) {
+          if (result.status === 'fulfilled') {
+            processed += result.value.processed;
+            uploaded += result.value.uploaded;
+            errors += result.value.errors;
+          } else {
+            errors += result.value?.processed || 0;
+          }
+        }
+      }
+    }
+
+    return { processed, uploaded, errors };
+  }
+
+  /**
+   * Process a single file for RFC upload (Supabase mode)
+   */
+  async #processRfcSingleFile(file, uploadService, apiService, options, fs) {
+    try {
+      // Check if file exists
+      if (!fs.existsSync(file.original_path)) {
+        logger.warn(
+          `File not found: ${file.filename} at ${file.original_path}`,
+        );
+        await apiService.updateFileStatus([
+          {
+            id: file.id,
+            status: 'file-not-found',
+            message: 'File no longer exists at original path',
+          },
+        ]);
+        return { success: false, error: 'File not found' };
+      }
+
+      // Supabase requires single file upload with uploadPath
+      let uploadPath;
+      if (options.folderStructure && file.arela_path) {
+        uploadPath = `uploads/${options.folderStructure}/${file.arela_path}${file.filename}`;
+      } else if (file.arela_path) {
+        uploadPath = `uploads/${file.arela_path}${file.filename}`;
+      } else {
+        uploadPath = `uploads/${file.rfc}/${file.filename}`;
+      }
+
+      const uploadResult = await uploadService.upload(
+        [
+          {
+            path: file.original_path,
+            name: file.filename,
+            contentType: 'application/octet-stream',
+          },
+        ],
+        { uploadPath: uploadPath },
+      );
+
+      // Check upload result before updating database status
+      if (uploadResult.success) {
+        await apiService.updateFileStatus([
+          {
+            id: file.id,
+            status: 'file-uploaded',
+            message: 'Successfully uploaded to Supabase',
+            processing_status: 'UPLOADED',
+          },
+        ]);
+
+        logger.info(`✅ Uploaded: ${file.filename}`);
+        return { success: true, filename: file.filename };
+      } else {
+        await apiService.updateFileStatus([
+          {
+            id: file.id,
+            status: 'upload-error',
+            message: `Upload failed: ${uploadResult.error}`,
+          },
+        ]);
+
+        logger.error(
+          `❌ Upload failed: ${file.filename} - ${uploadResult.error}`,
+        );
+        return {
+          success: false,
+          error: uploadResult.error,
+          filename: file.filename,
+        };
+      }
+    } catch (error) {
+      logger.error(
+        `❌ Error processing file ${file.filename}: ${error.message}`,
+      );
+
+      await apiService.updateFileStatus([
+        {
+          id: file.id,
+          status: 'upload-error',
+          message: `Processing error: ${error.message}`,
+        },
+      ]);
+
+      return { success: false, error: error.message, filename: file.filename };
+    }
+  }
+
+  /**
+   * Process multiple files in a single API batch call (API service mode)
+   */
+  async #processRfcApiBatch(files, uploadService, apiService, options, fs) {
+    let processed = 0;
+    let uploaded = 0;
+    let errors = 0;
+
+    try {
+      // Prepare files for batch upload
+      const validFiles = [];
+      const invalidFiles = [];
+
+      for (const file of files) {
+        processed++;
+
+        if (!fs.existsSync(file.original_path)) {
+          logger.warn(
+            `File not found: ${file.filename} at ${file.original_path}`,
+          );
+          invalidFiles.push(file);
+          continue;
+        }
+
+        validFiles.push({
+          fileData: {
+            path: file.original_path,
+            name: file.filename,
+            contentType: 'application/octet-stream',
+          },
+          dbRecord: file,
+        });
+      }
+
+      // Update invalid files in database
+      if (invalidFiles.length > 0) {
+        await apiService.updateFileStatus(
+          invalidFiles.map((file) => ({
+            id: file.id,
+            status: 'file-not-found',
+            message: 'File no longer exists at original path',
+          })),
+        );
+        errors += invalidFiles.length;
+      }
+
+      // Process valid files in batch if any exist
+      if (validFiles.length > 0) {
+        // Determine folder structure (all files in this batch should have same arela_path)
+        const sampleFile = validFiles[0].dbRecord;
+        let fullFolderStructure;
+        if (options.folderStructure && sampleFile.arela_path) {
+          fullFolderStructure = `${options.folderStructure}/${sampleFile.arela_path}`;
+        } else if (sampleFile.arela_path) {
+          fullFolderStructure = sampleFile.arela_path;
+        } else {
+          fullFolderStructure = `${sampleFile.rfc}/`;
+        }
+
+        // Make single API call with multiple files
+        // Include RFC for multi-database routing (required for cross-tenant uploads)
+        const uploadResult = await uploadService.upload(
+          validFiles.map((f) => f.fileData),
+          {
+            folderStructure: fullFolderStructure,
+            rfc: sampleFile.rfc, // For cross-tenant: routes to correct client DB
+            autoDetect: true, // Enable detection on target API
+            autoOrganize: true, // Enable organization on target API
+          },
+        );
+
+        if (uploadResult.success && uploadResult.data) {
+          const apiResult = uploadResult.data;
+
+          logger.info(
+            `📋 Processing API response: ${apiResult.uploaded?.length || 0} uploaded, ${apiResult.errors?.length || 0} errors`,
+          );
+
+          // Debug logging to understand API response structure
+          logger.debug(
+            `🔍 API Response structure: ${JSON.stringify(apiResult, null, 2)}`,
+          );
+          if (apiResult.uploaded && apiResult.uploaded.length > 0) {
+            logger.debug(
+              `🔍 First uploaded file structure: ${JSON.stringify(apiResult.uploaded[0], null, 2)}`,
+            );
+          }
+
+          // Create filename to file mapping for quick lookup
+          const fileNameToRecord = new Map();
+          validFiles.forEach((f) => {
+            fileNameToRecord.set(f.fileData.name, f.dbRecord);
+          });
+
+          // Debug: Log expected filenames
+          logger.debug(
+            `🔍 Expected filenames: ${Array.from(fileNameToRecord.keys()).join(', ')}`,
+          );
+
+          // Prepare status updates
+          const statusUpdates = [];
+
+          // Handle successfully uploaded files
+          if (apiResult.uploaded && apiResult.uploaded.length > 0) {
+            const successfulFileIds = [];
+            const matchedFilenames = [];
+
+            apiResult.uploaded.forEach((uploadedFile) => {
+              // Try multiple possible property names for filename
+              const possibleFilename =
+                uploadedFile.fileName ||
+                uploadedFile.filename ||
+                uploadedFile.name ||
+                uploadedFile.file_name ||
+                uploadedFile.originalName ||
+                uploadedFile.original_name;
+
+              logger.debug(
+                `🔍 Trying to match uploaded file: ${JSON.stringify(uploadedFile)}`,
+              );
+
+              const dbRecord = fileNameToRecord.get(possibleFilename);
+              if (dbRecord) {
+                successfulFileIds.push(dbRecord.id);
+                matchedFilenames.push(possibleFilename);
+                logger.debug(`✅ Matched file: ${possibleFilename}`);
+
+                statusUpdates.push({
+                  id: dbRecord.id,
+                  status: 'file-uploaded',
+                  message: 'Successfully uploaded to Arela API (batch)',
+                  processing_status: 'UPLOADED',
+                });
+              } else {
+                logger.warn(
+                  `⚠️ Could not match uploaded file with any known filename: ${JSON.stringify(uploadedFile)}`,
+                );
+              }
+            });
+
+            // If no individual files matched but API indicates success, use fallback
+            if (
+              successfulFileIds.length === 0 &&
+              apiResult.uploaded.length > 0
+            ) {
+              logger.warn(
+                `🔄 Fallback: No individual file matches found, but API indicates ${apiResult.uploaded.length} uploads. Marking all ${validFiles.length} batch files as uploaded.`,
+              );
+              validFiles.forEach((f) => {
+                statusUpdates.push({
+                  id: f.dbRecord.id,
+                  status: 'file-uploaded',
+                  message: 'Successfully uploaded to Arela API (batch)',
+                  processing_status: 'UPLOADED',
+                });
+              });
+            }
+
+            uploaded += successfulFileIds.length || validFiles.length;
+          }
+
+          // Handle failed files
+          if (apiResult.errors && apiResult.errors.length > 0) {
+            apiResult.errors.forEach((errorInfo) => {
+              // Try multiple possible property names for filename in errors
+              const possibleFilename =
+                errorInfo.fileName ||
+                errorInfo.filename ||
+                errorInfo.name ||
+                errorInfo.file_name ||
+                errorInfo.originalName ||
+                errorInfo.original_name;
+
+              const dbRecord = fileNameToRecord.get(possibleFilename);
+              if (dbRecord) {
+                statusUpdates.push({
+                  id: dbRecord.id,
+                  status: 'upload-error',
+                  message: `Upload failed: ${errorInfo.error || 'Unknown error'}`,
+                });
+                errors++;
+              } else {
+                logger.warn(
+                  `⚠️ Could not match error file: ${JSON.stringify(errorInfo)}`,
+                );
+              }
+            });
+          }
+
+          // Handle any remaining files that weren't in uploaded or errors arrays
+          const extractFilename = (fileObj) => {
+            return (
+              fileObj.fileName ||
+              fileObj.filename ||
+              fileObj.name ||
+              fileObj.file_name ||
+              fileObj.originalName ||
+              fileObj.original_name
+            );
+          };
+
+          const processedFileNames = new Set([
+            ...(apiResult.uploaded || []).map(extractFilename).filter(Boolean),
+            ...(apiResult.errors || []).map(extractFilename).filter(Boolean),
+          ]);
+
+          const unprocessedFiles = validFiles.filter(
+            (f) => !processedFileNames.has(f.fileData.name),
+          );
+
+          if (unprocessedFiles.length > 0) {
+            const alreadyHandledCount = uploaded + errors;
+            const shouldMarkUnprocessed =
+              alreadyHandledCount < validFiles.length;
+
+            if (shouldMarkUnprocessed) {
+              unprocessedFiles.forEach((f) => {
+                statusUpdates.push({
+                  id: f.dbRecord.id,
+                  status: 'upload-error',
+                  message: 'File not found in API response',
+                });
+              });
+              errors += unprocessedFiles.length;
+
+              logger.warn(
+                `⚠️ Unprocessed files: ${unprocessedFiles.length} files not found in API response`,
+              );
+              logger.debug(
+                `🔍 API response uploaded array: ${JSON.stringify(apiResult.uploaded)}`,
+              );
+              logger.debug(
+                `🔍 Expected filenames: ${validFiles.map((f) => f.fileData.name).join(', ')}`,
+              );
+            } else {
+              logger.debug(
+                `✅ All files already handled (uploaded: ${uploaded}, errors: ${errors}), skipping unprocessed marking`,
+              );
+            }
+          }
+
+          // Batch update all status changes
+          if (statusUpdates.length > 0) {
+            const updateResult =
+              await apiService.updateFileStatus(statusUpdates);
+            if (!updateResult.success) {
+              logger.error(
+                `Some status updates failed: ${updateResult.errors?.length || 0} errors`,
+              );
+            } else {
+              logger.info(
+                `✅ Batch upload successful: ${uploaded} files uploaded`,
+              );
+            }
+          }
+        } else {
+          // Complete batch failure - mark all files as failed
+          const failureUpdates = validFiles.map((f) => ({
+            id: f.dbRecord.id,
+            status: 'upload-error',
+            message: uploadResult.error || 'Batch upload failed',
+          }));
+
+          await apiService.updateFileStatus(failureUpdates);
+
+          errors += validFiles.length;
+          logger.error(
+            `❌ Complete batch failure: ${validFiles.length} files - ${uploadResult.error}`,
+          );
+        }
+      }
+    } catch (error) {
+      logger.error(`❌ Error processing batch: ${error.message}`);
+
+      // Mark all files as failed
+      const failureUpdates = files.map((f) => ({
+        id: f.id,
+        status: 'upload-error',
+        message: `Batch processing error: ${error.message}`,
+      }));
+
+      await apiService.updateFileStatus(failureUpdates);
+
+      errors += files.length;
+    }
+
+    return { processed, uploaded, errors };
+  }
+
+  /**
+   * Insert upload session event into watch_uploads table
+   * @param {Object} uploadEvent - Upload event from LoggingService
+   * @param {string} sessionId - Session ID for tracking
+   * @returns {Promise<Object>} Inserted record
+   */
+  async insertUploadEvent(uploadEvent, sessionId) {
+    const supabase = await this.#getSupabaseClient();
+
+    const record = {
+      session_id: sessionId,
+      timestamp: uploadEvent.timestamp || new Date().toISOString(),
+      strategy: uploadEvent.strategy, // 'individual', 'batch', 'full-structure'
+      file_count: uploadEvent.fileCount || 0,
+      success_count: uploadEvent.successCount || 0,
+      failure_count: uploadEvent.failureCount || 0,
+      retry_count: uploadEvent.retryCount || 0,
+      duration_ms: uploadEvent.duration || 0,
+      status: uploadEvent.status || 'completed',
+      metadata: uploadEvent.metadata || null,
+    };
+
+    try {
+      const { data, error } = await this.#queryWithRetry(async () => {
+        return await supabase.from('watch_uploads').insert([record]).select();
+      }, `insert upload event for session ${sessionId}`);
+
+      if (error) {
+        logger.error(`Failed to insert upload event: ${error.message}`);
+        throw error;
+      }
+
+      return data[0];
+    } catch (error) {
+      logger.error(`Error inserting upload event: ${error.message}`);
+      throw error;
+    }
+  }
+
+  /**
+   * Insert retry event into watch_events table
+   * @param {string} uploadEventId - ID of the parent upload event
+   * @param {string} sessionId - Session ID for tracking
+   * @param {Object} retryEvent - Retry event from LoggingService
+   * @returns {Promise<Object>} Inserted record
+   */
+  async insertRetryEvent(uploadEventId, sessionId, retryEvent) {
+    const supabase = await this.#getSupabaseClient();
+
+    const record = {
+      upload_event_id: uploadEventId,
+      session_id: sessionId,
+      timestamp: retryEvent.timestamp || new Date().toISOString(),
+      attempt_number: retryEvent.attemptNumber || 0,
+      error_message: retryEvent.error || null,
+      backoff_ms: retryEvent.backoffMs || 0,
+      type: 'retry',
+    };
+
+    try {
+      const { data, error } = await this.#queryWithRetry(async () => {
+        return await supabase.from('watch_events').insert([record]).select();
+      }, `insert retry event for upload ${uploadEventId}`);
+
+      if (error) {
+        logger.error(`Failed to insert retry event: ${error.message}`);
+        throw error;
+      }
+
+      return data[0];
+    } catch (error) {
+      logger.error(`Error inserting retry event: ${error.message}`);
+      throw error;
+    }
+  }
+
+  /**
+   * Get upload history for a session
+   * @param {string} sessionId - Session ID to query
+   * @param {Object} options - Query options (limit, offset, strategy filter)
+   * @returns {Promise<Array>} Array of upload events
+   */
+  async getSessionUploadHistory(sessionId, options = {}) {
+    const supabase = await this.#getSupabaseClient();
+    const limit = options.limit || 100;
+    const offset = options.offset || 0;
+
+    try {
+      let query = supabase
+        .from('watch_uploads')
+        .select('*')
+        .eq('session_id', sessionId)
+        .order('timestamp', { ascending: false })
+        .range(offset, offset + limit - 1);
+
+      // Filter by strategy if provided
+      if (options.strategy) {
+        query = query.eq('strategy', options.strategy);
+      }
+
+      const { data, error } = await this.#queryWithRetry(async () => {
+        return await query;
+      }, `fetch upload history for session ${sessionId}`);
+
+      if (error) {
+        logger.error(`Failed to fetch upload history: ${error.message}`);
+        throw error;
+      }
+
+      return data || [];
+    } catch (error) {
+      logger.error(`Error fetching upload history: ${error.message}`);
+      return [];
+    }
+  }
+
+  /**
+   * Get retry history for an upload event
+   * @param {string} uploadEventId - Upload event ID to query
+   * @param {Object} options - Query options (limit, offset)
+   * @returns {Promise<Array>} Array of retry events
+   */
+  async getUploadRetryHistory(uploadEventId, options = {}) {
+    const supabase = await this.#getSupabaseClient();
+    const limit = options.limit || 100;
+    const offset = options.offset || 0;
+
+    try {
+      const { data, error } = await this.#queryWithRetry(async () => {
+        return await supabase
+          .from('watch_events')
+          .select('*')
+          .eq('upload_event_id', uploadEventId)
+          .eq('type', 'retry')
+          .order('timestamp', { ascending: true })
+          .range(offset, offset + limit - 1);
+      }, `fetch retry history for upload ${uploadEventId}`);
+
+      if (error) {
+        logger.error(`Failed to fetch retry history: ${error.message}`);
+        throw error;
+      }
+
+      return data || [];
+    } catch (error) {
+      logger.error(`Error fetching retry history: ${error.message}`);
+      return [];
+    }
+  }
+
+  /**
+   * Get session statistics
+   * @param {string} sessionId - Session ID to analyze
+   * @returns {Promise<Object>} Session statistics
+   */
+  async getSessionStatistics(sessionId) {
+    const supabase = await this.#getSupabaseClient();
+
+    try {
+      // Fetch all upload events for the session
+      const { data: uploads, error: uploadError } = await this.#queryWithRetry(
+        async () => {
+          return await supabase
+            .from('watch_uploads')
+            .select('*')
+            .eq('session_id', sessionId);
+        },
+        `fetch statistics for session ${sessionId}`,
+      );
+
+      if (uploadError) {
+        throw uploadError;
+      }
+
+      // Fetch all retry events for the session
+      const { data: retries, error: retryError } = await this.#queryWithRetry(
+        async () => {
+          return await supabase
+            .from('watch_events')
+            .select('*')
+            .eq('session_id', sessionId)
+            .eq('type', 'retry');
+        },
+        `fetch retry statistics for session ${sessionId}`,
+      );
+
+      if (retryError) {
+        throw retryError;
+      }
+
+      // Calculate statistics
+      const stats = {
+        sessionId,
+        totalUploadEvents: uploads?.length || 0,
+        totalRetryEvents: retries?.length || 0,
+        totalFileCount: 0,
+        totalSuccessCount: 0,
+        totalFailureCount: 0,
+        totalRetryCount: 0,
+        totalDuration: 0,
+        byStrategy: {
+          individual: {
+            uploadCount: 0,
+            totalFiles: 0,
+            totalSuccess: 0,
+            totalFailure: 0,
+            successRate: 0,
+            totalDuration: 0,
+          },
+          batch: {
+            uploadCount: 0,
+            totalFiles: 0,
+            totalSuccess: 0,
+            totalFailure: 0,
+            successRate: 0,
+            totalDuration: 0,
+          },
+          'full-structure': {
+            uploadCount: 0,
+            totalFiles: 0,
+            totalSuccess: 0,
+            totalFailure: 0,
+            successRate: 0,
+            totalDuration: 0,
+          },
+        },
+        retryStats: {
+          totalRetries: retries?.length || 0,
+          uniqueUploadsWithRetries: new Set(
+            retries?.map((r) => r.upload_event_id) || [],
+          ).size,
+          totalRetryDuration:
+            retries?.reduce((sum, r) => sum + (r.backoff_ms || 0), 0) || 0,
+        },
+      };
+
+      // Process upload events
+      if (uploads && uploads.length > 0) {
+        uploads.forEach((upload) => {
+          stats.totalFileCount += upload.file_count || 0;
+          stats.totalSuccessCount += upload.success_count || 0;
+          stats.totalFailureCount += upload.failure_count || 0;
+          stats.totalRetryCount += upload.retry_count || 0;
+          stats.totalDuration += upload.duration_ms || 0;
+
+          const strategyKey = upload.strategy || 'individual';
+          if (stats.byStrategy[strategyKey]) {
+            stats.byStrategy[strategyKey].uploadCount += 1;
+            stats.byStrategy[strategyKey].totalFiles += upload.file_count || 0;
+            stats.byStrategy[strategyKey].totalSuccess +=
+              upload.success_count || 0;
+            stats.byStrategy[strategyKey].totalFailure +=
+              upload.failure_count || 0;
+            stats.byStrategy[strategyKey].totalDuration +=
+              upload.duration_ms || 0;
+
+            // Calculate success rate
+            const totalFiles =
+              stats.byStrategy[strategyKey].totalSuccess +
+              stats.byStrategy[strategyKey].totalFailure;
+            if (totalFiles > 0) {
+              stats.byStrategy[strategyKey].successRate = (
+                (stats.byStrategy[strategyKey].totalSuccess / totalFiles) *
+                100
+              ).toFixed(2);
+            }
+          }
+        });
+      }
+
+      return stats;
+    } catch (error) {
+      logger.error(`Error calculating session statistics: ${error.message}`);
+      return null;
+    }
+  }
+
+  /**
+   * Delete old session data (cleanup)
+   * @param {number} daysOld - Delete sessions older than this many days
+   * @returns {Promise<Object>} Deletion results
+   */
+  async cleanupOldSessions(daysOld = 30) {
+    const supabase = await this.#getSupabaseClient();
+
+    try {
+      const cutoffDate = new Date();
+      cutoffDate.setDate(cutoffDate.getDate() - daysOld);
+
+      // Get sessions to delete
+      const { data: sessionsToDelete, error: fetchError } =
+        await this.#queryWithRetry(async () => {
+          return await supabase
+            .from('watch_uploads')
+            .select('session_id')
+            .lt('timestamp', cutoffDate.toISOString())
+            .distinct();
+        }, `fetch sessions older than ${daysOld} days`);
+
+      if (fetchError) {
+        throw fetchError;
+      }
+
+      let deletedUploads = 0;
+      let deletedEvents = 0;
+
+      if (sessionsToDelete && sessionsToDelete.length > 0) {
+        const sessionIds = sessionsToDelete.map((s) => s.session_id);
+
+        // Delete events
+        const { count: eventCount, error: eventError } =
+          await this.#queryWithRetry(async () => {
+            return await supabase
+              .from('watch_events')
+              .delete()
+              .in('session_id', sessionIds);
+          }, `delete events for old sessions`);
+
+        if (!eventError) {
+          deletedEvents = eventCount || 0;
+        }
+
+        // Delete uploads
+        const { count: uploadCount, error: uploadError } =
+          await this.#queryWithRetry(async () => {
+            return await supabase
+              .from('watch_uploads')
+              .delete()
+              .in('session_id', sessionIds);
+          }, `delete old session uploads`);
+
+        if (!uploadError) {
+          deletedUploads = uploadCount || 0;
+        }
+      }
+
+      return {
+        deletedUploads,
+        deletedEvents,
+        sessionsDeleted: sessionsToDelete?.length || 0,
+      };
+    } catch (error) {
+      logger.error(`Error cleaning up old sessions: ${error.message}`);
+      return { deletedUploads: 0, deletedEvents: 0, sessionsDeleted: 0 };
+    }
+  }
+
+  /**
+   * Cleanup database connections and resources
+   * Called during graceful shutdown
+   * @returns {Promise<Object>} Cleanup results
+   */
+  async cleanup() {
+    try {
+      logger.info('DatabaseService: Starting cleanup...');
+
+      // Commit any pending transactions
+      const transactionResult = await this.commitPendingTransactions();
+
+      // Close database connections
+      const closeResult = await this.closeConnections();
+
+      logger.info('DatabaseService: Cleanup complete');
+
+      return {
+        success: true,
+        transactionsCommitted: transactionResult.count,
+        connectionsClosedResult: closeResult,
+      };
+    } catch (error) {
+      logger.error(`DatabaseService: Error during cleanup: ${error.message}`);
+      return {
+        success: false,
+        error: error.message,
+      };
+    }
+  }
+
+  /**
+   * Commit any pending transactions before shutdown
+   * @private
+   * @returns {Promise<Object>} Results
+   */
+  async commitPendingTransactions() {
+    try {
+      logger.debug('DatabaseService: Committing pending transactions...');
+
+      // Note: This is a placeholder for actual transaction handling
+      // In a real implementation, you would track active transactions
+      // and ensure they are properly committed before shutdown
+
+      logger.debug('DatabaseService: Pending transactions committed');
+      return { count: 0, success: true };
+    } catch (error) {
+      logger.error(
+        `DatabaseService: Error committing transactions: ${error.message}`,
+      );
+      return { count: 0, success: false, error: error.message };
+    }
+  }
+
+  /**
+   * Close all database connections
+   * @private
+   * @returns {Promise<Object>} Results
+   */
+  async closeConnections() {
+    try {
+      logger.debug('DatabaseService: Closing database connections...');
+
+      // Close Supabase connection if available
+      if (this.supabase) {
+        // Supabase client will handle connection cleanup automatically
+        logger.debug('DatabaseService: Supabase connection cleanup initiated');
+      }
+
+      logger.info('DatabaseService: All database connections closed');
+      return { success: true };
+    } catch (error) {
+      logger.error(
+        `DatabaseService: Error closing connections: ${error.message}`,
+      );
+      return { success: false, error: error.message };
+    }
+  }
+}
+
+// Export singleton instance
+export const databaseService = new DatabaseService();
+export default databaseService;