@arela/uploader 0.2.13 → 1.0.0

package/src/services/DatabaseService.js

@@ -25,6 +25,15 @@ export class DatabaseService {
     return await supabaseService.getClient();
   }
 
+  /**
+   * Get Supabase client (public wrapper for dependency injection)
+   * Used by specialized services like SmartWatchDatabaseService
+   * @returns {Promise<Object>} Supabase client
+   */
+  async getSupabaseClient() {
+    return this.#getSupabaseClient();
+  }
+
   /**
    * Execute database query with retry logic and exponential backoff
    * @private
@@ -71,6 +80,18 @@ export class DatabaseService {
     throw lastError;
   }
 
+  /**
+   * Execute database query with retry logic (public wrapper for dependency injection)
+   * Used by specialized services like SmartWatchDatabaseService
+   * @param {Function} queryFn - Query function to execute
+   * @param {string} operation - Description of the operation for logging
+   * @param {number} maxRetries - Maximum number of retry attempts (default: 3)
+   * @returns {Promise<Object>} Query result
+   */
+  async queryWithRetry(queryFn, operation, maxRetries = 3) {
+    return this.#queryWithRetry(queryFn, operation, maxRetries);
+  }
+
   /**
    * Insert file stats with document detection into uploader table
    * @param {Array} files - Array of file objects
@@ -112,6 +133,7 @@ export class DatabaseService {
       const filename = file.originalName || path.basename(file.path);
 
       const record = {
+        name: filename,
         document_type: null,
         size: stats.size,
         num_pedimento: null,
@@ -122,11 +144,14 @@ export class DatabaseService {
         rfc: null,
         message: null,
         file_extension: fileExtension,
-        is_like_simplificado:
-          fileExtension === 'pdf' && filename.toLowerCase().includes('simp'),
+        is_like_simplificado: filename.toLowerCase().includes('simp'),
         year: null,
         created_at: new Date().toISOString(),
+        updated_at: new Date().toISOString(),
         modified_at: stats.mtime.toISOString(),
+        // Queue/Processing columns (for arela-api)
+        processing_status: 'PENDING',
+        upload_attempts: 0,
       };
 
       // Try to detect document type for supported files
@@ -147,11 +172,8 @@ export class DatabaseService {
               record.year = detection.detectedPedimentoYear;
             }
 
-            const rfcField = detection.fields.find(
-              (f) => f.name === 'rfc' && f.found,
-            );
-            if (rfcField) {
-              record.rfc = rfcField.value;
+            if (detection.rfc) {
+              record.rfc = detection.rfc;
             }
           } else {
             record.status = 'not-detected';
@@ -181,18 +203,153 @@ export class DatabaseService {
       `Inserting ${records.length} new records into uploader table...`,
     );
 
+    // Use upsert to handle duplicates gracefully
+    // This will insert new records or update existing ones (by original_path)
     const { data, error } = await supabase
       .from('uploader')
-      .insert(records)
+      .upsert(records, { onConflict: 'original_path' })
       .select();
 
     if (error) {
       throw new Error(`Failed to insert stats records: ${error.message}`);
     }
 
+    // Propagate arela_path to related files in same folder
+    if (data && data.length > 0) {
+      await this.#propagateArelaPathToRelatedFiles(data, supabase);
+    }
+
     return data;
   }
 
+  /**
+   * Propagate arela_path from pedimento files to related files in the same directory
+   * When a pedimento_simplificado is detected, all files in its directory get the same arela_path
+   * Also checks database for existing files in the same directory that need the arela_path
+   * @private
+   * @param {Array} insertedRecords - Records that were just inserted
+   * @param {Object} supabase - Supabase client
+   * @returns {Promise<void>}
+   */
+  async #propagateArelaPathToRelatedFiles(insertedRecords, supabase) {
+    try {
+      // Group records by directory
+      const recordsByDir = {};
+
+      for (const record of insertedRecords) {
+        if (!record.original_path) continue;
+
+        const dirPath = path.dirname(record.original_path);
+        if (!recordsByDir[dirPath]) {
+          recordsByDir[dirPath] = {
+            pedimentos: [],
+            allFiles: [],
+          };
+        }
+
+        recordsByDir[dirPath].allFiles.push(record);
+
+        // Identify pedimento files
+        if (
+          record.document_type === 'pedimento_simplificado' &&
+          record.arela_path
+        ) {
+          recordsByDir[dirPath].pedimentos.push(record);
+        }
+      }
+
+      // For each directory with a pedimento, propagate its arela_path to related files
+      for (const [dirPath, dirData] of Object.entries(recordsByDir)) {
+        if (dirData.pedimentos.length === 0) continue;
+
+        // Use the first pedimento's arela_path (should be only one per directory typically)
+        const pedimentoRecord = dirData.pedimentos[0];
+        const arelaPath = pedimentoRecord.arela_path;
+
+        logger.info(
+          `📁 Propagating arela_path from pedimento to files in ${path.basename(dirPath)}/`,
+        );
+
+        // Step 1: Update newly inserted files in this directory
+        const fileIds = dirData.allFiles
+          .filter(
+            (f) =>
+              f.id &&
+              f.arela_path !== arelaPath &&
+              f.document_type !== 'pedimento_simplificado',
+          )
+          .map((f) => f.id);
+
+        if (fileIds.length > 0) {
+          const { error: updateError } = await supabase
+            .from('uploader')
+            .update({ arela_path: arelaPath })
+            .in('id', fileIds);
+
+          if (updateError) {
+            logger.warn(
+              `Could not propagate arela_path: ${updateError.message}`,
+            );
+          } else {
+            logger.info(
+              `✅ Updated ${fileIds.length} related files with arela_path: ${arelaPath}`,
+            );
+          }
+        }
+
+        // Step 2: Also find and update any existing files in the same directory that don't have arela_path
+        // This handles the case where files were detected earlier but the pedimento is detected now
+        try {
+          const dirPattern = dirPath.replace(/\\/g, '/'); // Normalize for SQL LIKE
+          const { data: existingFiles, error: fetchError } = await supabase
+            .from('uploader')
+            .select('id, original_path, document_type')
+            .like('original_path', `${dirPattern}/%`)
+            .is('arela_path', null)
+            .limit(1000); // Reasonable limit to avoid huge queries
+
+          if (fetchError) {
+            logger.warn(
+              `Could not fetch existing files in ${path.basename(dirPath)}: ${fetchError.message}`,
+            );
+          } else if (existingFiles && existingFiles.length > 0) {
+            const existingFileIds = existingFiles
+              .filter(
+                (f) => f.id && f.document_type !== 'pedimento_simplificado',
+              )
+              .map((f) => f.id);
+
+            if (existingFileIds.length > 0) {
+              const { error: existingError } = await supabase
+                .from('uploader')
+                .update({ arela_path: arelaPath })
+                .in('id', existingFileIds);
+
+              if (existingError) {
+                logger.warn(
+                  `Could not update existing files with arela_path: ${existingError.message}`,
+                );
+              } else {
+                logger.info(
+                  `✅ Updated ${existingFileIds.length} existing files in directory with arela_path: ${arelaPath}`,
+                );
+              }
+            }
+          }
+        } catch (existingFilesError) {
+          logger.warn(
+            `Error checking for existing files in ${path.basename(dirPath)}: ${existingFilesError.message}`,
+          );
+        }
+      }
+    } catch (error) {
+      logger.warn(
+        `Error propagating arela_path to related files: ${error.message}`,
+      );
+      // Don't throw - this is a non-critical operation
+    }
+  }
+
   /**
    * Insert file stats only (no detection) into uploader table
    * @param {Array} files - Array of file objects
@@ -200,13 +357,30 @@ export class DatabaseService {
    * @returns {Promise<Object>} Statistics about the operation
    */
   async insertStatsOnlyToUploaderTable(files, options) {
-    const supabase = await this.#getSupabaseClient();
     const batchSize = 1000;
+    const quietMode = options?.quietMode || false;
     const allRecords = [];
 
-    logger.info('Collecting filesystem stats...');
+    if (!quietMode) {
+      logger.info('Collecting filesystem stats...');
+    }
+
+    // Filter out system files and hidden files (macOS, Windows, Python, editors)
+    const systemFilePattern =
+      /^\.|__pycache__|\.pyc|\.swp|\.swo|Thumbs\.db|desktop\.ini|DS_Store|\$RECYCLE\.BIN|System Volume Information|~\$|\.tmp/i;
+
     for (const file of files) {
       try {
+        const fileName = file.originalName || path.basename(file.path);
+
+        // Skip system and hidden files
+        if (systemFilePattern.test(fileName)) {
+          if (!quietMode) {
+            logger.debug(`Skipping system file: ${fileName}`);
+          }
+          continue;
+        }
+
         const stats = file.stats || fs.statSync(file.path);
         const originalPath = options.clientPath || file.path;
         const fileExtension = path
@@ -215,24 +389,27 @@ export class DatabaseService {
           .replace('.', '');
 
         const record = {
-          document_type: null,
+          name: file.originalName || path.basename(file.path),
+          documentType: null,
           size: stats.size,
-          num_pedimento: null,
+          numPedimento: null,
           filename: file.originalName || path.basename(file.path),
-          original_path: originalPath,
-          arela_path: null,
+          originalPath: originalPath,
+          arelaPath: null,
           status: 'fs-stats',
           rfc: null,
           message: null,
-          file_extension: fileExtension,
-          created_at: new Date().toISOString(),
-          modified_at: stats.mtime.toISOString(),
-          is_like_simplificado:
+          fileExtension: fileExtension,
+          modifiedAt: stats.mtime.toISOString(),
+          isLikeSimplificado:
             fileExtension === 'pdf' &&
             (file.originalName || path.basename(file.path))
               .toLowerCase()
               .includes('simp'),
           year: null,
+          // Queue/Processing columns (for arela-api)
+          processingStatus: 'PENDING',
+          uploadAttempts: 0,
         };
 
         allRecords.push(record);
@@ -242,104 +419,66 @@ export class DatabaseService {
     }
 
     if (allRecords.length === 0) {
-      logger.info('No file stats to insert');
+      if (!quietMode) {
+        logger.info('No file stats to insert');
+      }
       return { totalInserted: 0, totalSkipped: 0, totalProcessed: 0 };
     }
 
-    logger.info(
-      `Processing ${allRecords.length} file stats in batches of ${batchSize}...`,
-    );
+    if (!quietMode) {
+      logger.info(
+        `Processing ${allRecords.length} file stats in batches of ${batchSize}...`,
+      );
+    }
 
     let totalInserted = 0;
     let totalUpdated = 0;
 
+    // Use API service for batch upsert
+    // If apiTarget is specified, use that specific API, otherwise use default
+    let uploadService;
+    if (options.apiTarget) {
+      uploadService = await uploadServiceFactory.getApiServiceForTarget(
+        options.apiTarget,
+      );
+    } else {
+      uploadService = await uploadServiceFactory.getUploadService(
+        options.forceSupabase,
+      );
+    }
+
+    // Use API service for batch upsert
     for (let i = 0; i < allRecords.length; i += batchSize) {
       const batch = allRecords.slice(i, i + batchSize);
 
       try {
-        // Check which records already exist
-        const originalPaths = batch.map((r) => r.original_path);
-        const { data: existingRecords, error: checkError } = await supabase
-          .from('uploader')
-          .select('original_path')
-          .in('original_path', originalPaths);
-
-        if (checkError) {
-          logger.error(
-            `Error checking existing records: ${checkError.message}`,
-          );
-          continue;
-        }
+        const result = await uploadService.batchUpsertStats(batch);
 
-        const existingPaths = new Set(
-          existingRecords?.map((r) => r.original_path) || [],
-        );
-        const newRecords = batch.filter(
-          (r) => !existingPaths.has(r.original_path),
-        );
-        const updateRecords = batch.filter((r) =>
-          existingPaths.has(r.original_path),
-        );
+        totalInserted += result.inserted || 0;
+        totalUpdated += result.updated || 0;
 
-        // Only log every 10th batch to reduce noise
+        // Only log every 10th batch to reduce noise (skip in quiet mode)
         if (
-          (Math.floor(i / batchSize) + 1) % 10 === 0 ||
-          Math.floor(i / batchSize) + 1 === 1
+          !quietMode &&
+          ((Math.floor(i / batchSize) + 1) % 10 === 0 ||
+            Math.floor(i / batchSize) + 1 === 1)
         ) {
           logger.info(
-            `Batch ${Math.floor(i / batchSize) + 1}: ${newRecords.length} new, ${updateRecords.length} updates`,
+            `Batch ${Math.floor(i / batchSize) + 1}: ${result.inserted || 0} new, ${result.updated || 0} updates`,
           );
         }
-
-        // Insert new records
-        if (newRecords.length > 0) {
-          const { error: insertError } = await supabase
-            .from('uploader')
-            .insert(newRecords);
-
-          if (insertError) {
-            logger.error(`Error inserting new records: ${insertError.message}`);
-          } else {
-            totalInserted += newRecords.length;
-            // Only log the batch insertion, not the summary (which comes at the end)
-          }
-        }
-
-        // Update existing records
-        if (updateRecords.length > 0) {
-          let batchUpdated = 0;
-          for (const record of updateRecords) {
-            const { error: updateError } = await supabase
-              .from('uploader')
-              .update({
-                size: record.size,
-                modified_at: record.modified_at,
-                filename: record.filename,
-                file_extension: record.file_extension,
-                is_like_simplificado: record.is_like_simplificado,
-              })
-              .eq('original_path', record.original_path);
-
-            if (!updateError) {
-              batchUpdated++;
-            }
-          }
-          totalUpdated += batchUpdated;
-          // Reduce logging noise - only log when there are updates
-          if (batchUpdated > 0) {
-            logger.info(`Updated ${batchUpdated} existing records`);
-          }
-        }
       } catch (error) {
         logger.error(
-          `Unexpected error in batch ${Math.floor(i / batchSize) + 1}: ${error.message}`,
+          `Error in batch ${Math.floor(i / batchSize) + 1}: ${error.message}`,
         );
       }
     }
 
-    logger.success(
-      `Phase 1 Summary: ${totalInserted} new records inserted, ${totalUpdated} existing records updated`,
-    );
+    if (!quietMode) {
+      logger.success(
+        `Phase 1 Summary: ${totalInserted} new records inserted, ${totalUpdated} existing records updated`,
+      );
+    }
 
     return {
       totalInserted,
@@ -354,8 +493,6 @@ export class DatabaseService {
    * @returns {Promise<Object>} Processing result
    */
   async detectPedimentosInDatabase(options = {}) {
-    const supabase = await this.#getSupabaseClient();
-
     logger.info(
       'Phase 2: Starting PDF detection for pedimento-simplificado documents...',
     );
@@ -370,6 +507,25 @@ export class DatabaseService {
     let offset = 0;
     let chunkNumber = 1;
 
+    // Get API service - use specific target if provided
+    let apiService;
+    if (options.apiTarget) {
+      apiService = await uploadServiceFactory.getApiServiceForTarget(
+        options.apiTarget,
+      );
+      logger.info(`Using API target: ${options.apiTarget}`);
+    } else {
+      apiService = await uploadServiceFactory.getUploadService();
+    }
+
+    if (apiService.getServiceName() !== 'Arela API') {
+      throw new Error(
+        'API service is required for PDF detection. Please configure ARELA_API_URL and ARELA_API_TOKEN.',
+      );
+    }
+
+    logger.info('Using API service for PDF detection...');
+
     logger.info(
       `Processing PDF files in chunks of ${queryBatchSize} records...`,
     );
@@ -380,18 +536,12 @@ export class DatabaseService {
       );
 
       try {
-        // Split the query to make it more efficient with retry logic
+        // Fetch records using API
         const { data: pdfRecords, error: queryError } =
-          await this.#queryWithRetry(async () => {
-            return await supabase
-              .from('uploader')
-              .select('id, original_path, filename, file_extension, status')
-              .eq('status', 'fs-stats')
-              .eq('file_extension', 'pdf')
-              .eq('is_like_simplificado', true)
-              .range(offset, offset + queryBatchSize - 1)
-              .order('created_at');
-          }, `fetch PDF records chunk ${chunkNumber}`);
+          await apiService.fetchPdfRecordsForDetection({
+            offset,
+            limit: queryBatchSize,
+          });
 
         if (queryError) {
           throw new Error(
@@ -413,9 +563,10 @@ export class DatabaseService {
         let chunkErrors = 0;
 
         // Process files in smaller batches
+        const batchUpdates = [];
+
         for (let i = 0; i < pdfRecords.length; i += processingBatchSize) {
           const batch = pdfRecords.slice(i, i + processingBatchSize);
-          const updatePromises = [];
 
           for (const record of batch) {
             try {
@@ -423,15 +574,11 @@ export class DatabaseService {
                 logger.warn(
                   `File not found: ${record.filename} at ${record.original_path}`,
                 );
-                updatePromises.push(
-                  supabase
-                    .from('uploader')
-                    .update({
-                      status: 'file-not-found',
-                      message: 'File no longer exists at original path',
-                    })
-                    .eq('id', record.id),
-                );
+                batchUpdates.push({
+                  id: record.id,
+                  status: 'file-not-found',
+                  message: 'File no longer exists at original path',
+                });
                 chunkErrors++;
                 totalErrors++;
                 continue;
@@ -444,28 +591,24 @@ export class DatabaseService {
               totalProcessed++;
 
               const updateData = {
+                id: record.id,
                 status: detection.detectedType ? 'detected' : 'not-detected',
-                document_type: detection.detectedType,
-                num_pedimento: detection.detectedPedimento,
-                arela_path: detection.arelaPath,
+                documentType: detection.detectedType,
+                numPedimento: detection.detectedPedimento,
+                arelaPath: detection.arelaPath,
                 message: detection.error || null,
                 year: detection.detectedPedimentoYear || null,
               };
 
-              if (detection.fields) {
-                const rfcField = detection.fields.find(
-                  (f) => f.name === 'rfc' && f.found,
-                );
-                if (rfcField) {
-                  updateData.rfc = rfcField.value;
-                }
+              if (detection.rfc) {
+                updateData.rfc = detection.rfc;
               }
 
               if (detection.detectedType) {
                 chunkDetected++;
                 totalDetected++;
                 logger.success(
-                  `Detected: ${record.filename} -> ${detection.detectedType} | Pedimento: ${detection.detectedPedimento || 'N/A'} | RFC: ${detection.fields?.rfc || 'N/A'}`,
+                  `Detected: ${record.filename} -> ${detection.detectedType} | Pedimento: ${detection.detectedPedimento || 'N/A'} | RFC: ${updateData.rfc || 'N/A'}`,
                 );
               } else {
                 logger.info(
@@ -473,12 +616,7 @@ export class DatabaseService {
                 );
               }
 
-              updatePromises.push(
-                supabase
-                  .from('uploader')
-                  .update(updateData)
-                  .eq('id', record.id),
-              );
+              batchUpdates.push(updateData);
             } catch (error) {
               logger.error(
                 `Error detecting ${record.filename}: ${error.message}`,
@@ -486,25 +624,30 @@ export class DatabaseService {
               chunkErrors++;
               totalErrors++;
 
-              updatePromises.push(
-                supabase
-                  .from('uploader')
-                  .update({
-                    status: 'detection-error',
-                    message: error.message,
-                  })
-                  .eq('id', record.id),
-              );
+              batchUpdates.push({
+                id: record.id,
+                status: 'detection-error',
+                message: error.message,
+              });
             }
           }
+        }
 
-          try {
-            await Promise.all(updatePromises);
-          } catch (error) {
-            logger.error(
-              `Error updating batch in chunk ${chunkNumber}: ${error.message}`,
-            );
+        // Batch update using API
+        try {
+          if (batchUpdates.length > 0) {
+            const updateResult =
+              await apiService.batchUpdateDetectionResults(batchUpdates);
+            if (!updateResult.success) {
+              logger.error(
+                `Some updates failed in chunk ${chunkNumber}: ${updateResult.errors?.length || 0} errors`,
+              );
+            }
           }
+        } catch (error) {
+          logger.error(
+            `Error updating batch in chunk ${chunkNumber}: ${error.message}`,
+          );
         }
 
         logger.success(
@@ -550,276 +693,156 @@ export class DatabaseService {
 
   /**
    * Propagate arela_path from pedimento_simplificado records to related files
+   * This operation is performed entirely on the backend for efficiency
    * @param {Object} options - Options for propagation
    * @returns {Promise<Object>} Processing result
    */
   async propagateArelaPath(options = {}) {
-    const supabase = await this.#getSupabaseClient();
-
     logger.info('Phase 3: Starting arela_path and year propagation process...');
-    console.log('🔍 Processing pedimento_simplificado records page by page...');
+    console.log('🔍 Triggering backend propagation process...');
 
-    // Log year filtering configuration
-    if (appConfig.upload.years && appConfig.upload.years.length > 0) {
-      logger.info(
-        `🗓️  Year filter enabled: ${appConfig.upload.years.join(', ')}`,
-      );
-      console.log(
-        `🗓️  Year filter enabled: ${appConfig.upload.years.join(', ')}`,
+    // Get API service - use specific target if provided
+    let apiService;
+    if (options.apiTarget) {
+      apiService = await uploadServiceFactory.getApiServiceForTarget(
+        options.apiTarget,
       );
+      logger.info(`Using API target: ${options.apiTarget}`);
     } else {
-      logger.info('🗓️  No year filter configured - processing all years');
-      console.log('🗓️  No year filter configured - processing all years');
+      apiService = await uploadServiceFactory.getUploadService();
     }
 
-    let totalProcessed = 0;
-    let totalUpdated = 0;
-    let totalErrors = 0;
-    let offset = 0;
-    const pageSize = 50;
-    let hasMoreData = true;
-    let pageNumber = 1;
-    const BATCH_SIZE = 50; // Process files in batches
-
-    // Process pedimento records page by page for memory efficiency
-    while (hasMoreData) {
-      logger.info(
-        `Fetching and processing pedimento records page ${pageNumber} (records ${offset + 1} to ${offset + pageSize})...`,
-      );
-
-      let query = supabase
-        .from('uploader')
-        .select('id, original_path, arela_path, filename, year')
-        .eq('document_type', 'pedimento_simplificado')
-        .not('arela_path', 'is', null);
-
-      // Add year filter if UPLOAD_YEARS is configured
-      if (appConfig.upload.years && appConfig.upload.years.length > 0) {
-        query = query.in('year', appConfig.upload.years);
-      }
-
-      const { data: pedimentoPage, error: pedimentoError } = await query
-        .range(offset, offset + pageSize - 1)
-        .order('created_at');
-
-      if (pedimentoError) {
-        const errorMsg = `Error fetching pedimento records page ${pageNumber}: ${pedimentoError.message}`;
-        logger.error(errorMsg);
-        throw new Error(errorMsg);
-      }
-
-      if (!pedimentoPage || pedimentoPage.length === 0) {
-        hasMoreData = false;
-        logger.info('No more pedimento records found');
-        break;
-      }
-
-      logger.info(
-        `Processing page ${pageNumber}: ${pedimentoPage.length} pedimento records`,
+    if (apiService.getServiceName() !== 'Arela API') {
+      throw new Error(
+        'API service is required for arela_path propagation. Please configure ARELA_API_URL and ARELA_API_TOKEN.',
       );
+    }
 
-      // Process each pedimento record in the current page
-      for (const pedimento of pedimentoPage) {
-        try {
-          totalProcessed++;
-
-          // Extract base path from original_path (remove filename)
-          const basePath = path.dirname(pedimento.original_path);
-
-          logger.info(
-            `Processing pedimento: ${pedimento.filename} | Base path: ${basePath} | Year: ${pedimento.year || 'N/A'}`,
-          );
-
-          // Extract folder part from existing arela_path
-          const existingPath = pedimento.arela_path;
-          const folderArelaPath = existingPath.includes('/')
-            ? existingPath.substring(0, existingPath.lastIndexOf('/')) + '/'
-            : existingPath.endsWith('/')
-              ? existingPath
-              : existingPath + '/';
-
-          // Process related files page by page for memory efficiency
-          let relatedFilesFrom = 0;
-          const relatedFilesPageSize = 50;
-          let hasMoreRelatedFiles = true;
-          let relatedFilesPageNumber = 1;
-          let totalRelatedFilesProcessed = 0;
-
-          logger.info(
-            `Searching and processing related files in base path: ${basePath}`,
-          );
-
-          while (hasMoreRelatedFiles) {
-            const { data: relatedFilesPage, error: relatedError } =
-              await this.#queryWithRetry(async () => {
-                return await supabase
-                  .from('uploader')
-                  .select('id, filename, original_path')
-                  .like('original_path', `${basePath}%`)
-                  .is('arela_path', null)
-                  .neq('id', pedimento.id) // Exclude the pedimento itself
-                  .range(
-                    relatedFilesFrom,
-                    relatedFilesFrom + relatedFilesPageSize - 1,
-                  );
-              }, `query related files for ${pedimento.filename} (page ${relatedFilesPageNumber})`);
-
-            if (relatedError) {
-              logger.error(
-                `Error finding related files for ${pedimento.filename}: ${relatedError.message}`,
-              );
-              totalErrors++;
-              break;
-            }
-            // console.log(`query by basePath: ${basePath} count: [${relatedFilesPage.length}]`);
-
-            if (!relatedFilesPage || relatedFilesPage.length === 0) {
-              hasMoreRelatedFiles = false;
-              if (totalRelatedFilesProcessed === 0) {
-                logger.info(`No related files found for ${pedimento.filename}`);
-              }
-              break;
-            }
-
-            logger.info(
-              `Processing related files page ${relatedFilesPageNumber}: ${relatedFilesPage.length} files for ${pedimento.filename}`,
-            );
-
-            // Track if any updates occurred in this page
-            let updatesOccurred = false;
-            // Process this page of related files in batches
-            const pageFileIds = relatedFilesPage.map((f) => f.id);
+    logger.info('Using API service for arela_path propagation...');
 
-            for (let i = 0; i < pageFileIds.length; i += BATCH_SIZE) {
-              const batchIds = pageFileIds.slice(i, i + BATCH_SIZE);
-              const batchNumber =
-                Math.floor(relatedFilesFrom / BATCH_SIZE) +
-                Math.floor(i / BATCH_SIZE) +
-                1;
+    // Log year filtering configuration
+    const years = appConfig.upload.years || [];
+    if (years.length > 0) {
+      logger.info(`🗓️  Year filter enabled: ${years.join(', ')}`);
+      console.log(`🗓️  Year filter enabled: ${years.join(', ')}`);
+    } else {
+      logger.info('🗓️  No year filter configured - processing all years');
+      console.log('🗓️  No year filter configured - processing all years');
+    }
 
-              logger.info(
-                `Updating batch ${batchNumber}: ${batchIds.length} files with arela_path and year...`,
-              );
+    console.log('⏳ Processing on backend... This may take a moment.');
 
-              try {
-                const { error: updateError } = await supabase
-                  .from('uploader')
-                  .update({
-                    arela_path: folderArelaPath,
-                    year: pedimento.year,
-                  })
-                  .in('id', batchIds);
-
-                if (updateError) {
-                  logger.error(
-                    `Error in batch ${batchNumber}: ${updateError.message}`,
-                  );
-                  totalErrors++;
-                } else {
-                  totalUpdated += batchIds.length;
-                  totalRelatedFilesProcessed += batchIds.length;
-                  updatesOccurred = true; // Mark that updates occurred
-                  logger.info(
-                    `Successfully updated batch ${batchNumber}: ${batchIds.length} files with arela_path and year`,
-                  );
-                }
-              } catch (batchError) {
-                logger.error(
-                  `Exception in batch ${batchNumber}: ${batchError.message}`,
-                );
-                totalErrors++;
-              }
-            }
+    // Trigger backend propagation - all logic runs server-side
+    const result = await apiService.propagateArelaPath({ years });
 
-            // Check if we need to fetch the next page of related files
-            if (relatedFilesPage.length < relatedFilesPageSize) {
-              logger.info('No more related files for', basePath);
-              hasMoreRelatedFiles = false;
-              logger.info(
-                `Completed processing related files for ${pedimento.filename}. Total processed: ${totalRelatedFilesProcessed}`,
-              );
-            } else {
-              // If updates occurred, reset pagination to start from beginning
-              // since records that matched the query may no longer match after update
-              if (updatesOccurred) {
-                relatedFilesFrom = 0;
-                logger.info(
-                  `Page ${relatedFilesPageNumber} complete with updates: ${relatedFilesPage.length} files processed, restarting pagination from beginning due to query condition changes...`,
-                );
-              } else {
-                relatedFilesFrom += relatedFilesPageSize - 1;
-                logger.info(
-                  `Page ${relatedFilesPageNumber} complete: ${relatedFilesPage.length} files processed, continuing to next page...`,
-                );
-              }
-              relatedFilesPageNumber++;
-            }
-          }
-        } catch (error) {
-          logger.error(
-            `Error processing pedimento ${pedimento.filename}: ${error.message}`,
-          );
-          totalErrors++;
-        }
-      }
-
-      // Check if we need to fetch the next page
-      if (pedimentoPage.length < pageSize) {
-        hasMoreData = false;
-        logger.info(
-          `Completed processing. Last page ${pageNumber} had ${pedimentoPage.length} records`,
-        );
-      } else {
-        offset += pageSize;
-        pageNumber++;
-        logger.info(
-          `Page ${pageNumber - 1} complete: ${pedimentoPage.length} records processed, moving to next page...`,
-        );
-      }
+    if (!result.success) {
+      const errorMsg = `Backend propagation failed: ${result.error || 'Unknown error'}`;
+      logger.error(errorMsg);
+      throw new Error(errorMsg);
     }
 
-    // Final summary
-    if (totalProcessed === 0) {
+    // Display results
+    const { processedCount = 0, updatedCount = 0, errorCount = 0 } = result;
+
+    if (processedCount === 0) {
       logger.info('No pedimento_simplificado records with arela_path found');
       console.log(
         'ℹ️  No pedimento_simplificado records with arela_path found',
       );
     } else {
-      console.log(
-        `📋 Processed ${totalProcessed} pedimento records across ${pageNumber - 1} pages`,
-      );
-      logger.info(
-        `Processed ${totalProcessed} pedimento records across ${pageNumber - 1} pages`,
-      );
+      console.log(`📋 Processed ${processedCount} pedimento records`);
+      logger.info(`Processed ${processedCount} pedimento records`);
     }
 
-    const result = {
-      processedCount: totalProcessed,
-      updatedCount: totalUpdated,
-      errorCount: totalErrors,
-    };
-
     logger.success(
-      `Phase 3 Summary: ${totalProcessed} pedimentos processed, ${totalUpdated} files updated with arela_path and year, ${totalErrors} errors`,
+      `Phase 3 Summary: ${processedCount} pedimentos processed, ${updatedCount} files updated with arela_path and year, ${errorCount} errors`,
     );
 
-    return result;
+    return {
+      processedCount,
+      updatedCount,
+      errorCount,
+    };
   }
 
   /**
    * Upload files to Arela API based on specific RFC values
+   * Supports cross-tenant mode where source and target APIs can be different
    * @param {Object} options - Upload options
+   * @param {string} options.sourceApi - Source API target for reading data (cross-tenant mode)
+   * @param {string} options.targetApi - Target API target for uploading files (cross-tenant mode)
+   * @param {string} options.apiTarget - Single API target for both reading and uploading (single API mode)
    * @returns {Promise<Object>} Processing result
    */
   async uploadFilesByRfc(options = {}) {
-    const supabase = await this.#getSupabaseClient();
-    const uploadService = await uploadServiceFactory.getUploadService();
-
     // Get configuration
     const appConfig = await import('../config/config.js').then(
       (m) => m.appConfig,
     );
 
+    // Determine if we're in cross-tenant mode
+    const isCrossTenant =
+      options.sourceApi &&
+      options.targetApi &&
+      options.sourceApi !== options.targetApi;
+
+    // Determine if we're in single API mode with specific target
+    const isSingleApiMode = !isCrossTenant && options.apiTarget;
+
+    let sourceService, targetService;
+
+    if (isCrossTenant) {
+      console.log('🔀 Cross-tenant upload mode enabled');
+      console.log(`   📖 Source API: ${options.sourceApi}`);
+      console.log(`   📝 Target API: ${options.targetApi}`);
+
+      // Get separate services for source and target
+      sourceService = await uploadServiceFactory.getApiServiceForTarget(
+        options.sourceApi,
+      );
+      targetService = await uploadServiceFactory.getApiServiceForTarget(
+        options.targetApi,
+      );
+
+      // Verify both services are available
+      if (!(await sourceService.isAvailable())) {
+        throw new Error(`Source API '${options.sourceApi}' is not available`);
+      }
+      if (!(await targetService.isAvailable())) {
+        throw new Error(`Target API '${options.targetApi}' is not available`);
+      }
+
+      console.log(`✅ Connected to source: ${sourceService.baseUrl}`);
+      console.log(`✅ Connected to target: ${targetService.baseUrl}`);
+    } else if (isSingleApiMode) {
+      // Single API mode with specific target - use the same service for both
+      console.log(`🎯 Single API mode: ${options.apiTarget}`);
+
+      const apiService = await uploadServiceFactory.getApiServiceForTarget(
+        options.apiTarget,
+      );
+
+      if (!(await apiService.isAvailable())) {
+        throw new Error(`API '${options.apiTarget}' is not available`);
+      }
+
+      console.log(`✅ Connected to: ${apiService.baseUrl}`);
+      sourceService = apiService;
+      targetService = apiService;
+    } else {
+      // Default mode - use the default service for both
+      const apiService = await uploadServiceFactory.getUploadService();
+
+      if (apiService.getServiceName() !== 'Arela API') {
+        throw new Error(
+          'API service is required for RFC-based upload. Please configure ARELA_API_URL and ARELA_API_TOKEN.',
+        );
+      }
+
+      sourceService = apiService;
+      targetService = apiService;
+    }
+
     if (!appConfig.upload.rfcs || appConfig.upload.rfcs.length === 0) {
       const errorMsg =
         'No RFCs specified. Please set UPLOAD_RFCS environment variable with pipe-separated RFC values.';
@@ -828,16 +851,18 @@ export class DatabaseService {
     }
 
     logger.info('Phase 4: Starting RFC-based upload process...');
+    logger.info(
+      `Using ${isCrossTenant ? 'cross-tenant' : 'standard'} API service for RFC-based upload...`,
+    );
     console.log('🎯 RFC-based Upload Mode');
     console.log(`📋 Target RFCs: ${appConfig.upload.rfcs.join(', ')}`);
     console.log('🔍 Searching for files to upload...');
 
     // First, count total files for the RFCs to show filtering effect
-    const { count: totalRfcFiles, error: countError } = await supabase
-      .from('uploader')
-      .select('*', { count: 'exact', head: true })
-      .in('rfc', appConfig.upload.rfcs)
-      .not('arela_path', 'is', null);
+    const { count: totalRfcFiles, error: countError } =
+      await sourceService.fetchRfcFileCount({
+        rfcs: appConfig.upload.rfcs,
+      });
 
     if (countError) {
       logger.warn(`Could not count total RFC files: ${countError.message}`);
@@ -863,22 +888,13 @@ export class DatabaseService {
         `Fetching pedimento records page ${pageNumber} (records ${offset + 1} to ${offset + pageSize})...`,
       );
 
-      let pedimentoQuery = supabase
-        .from('uploader')
-        .select('arela_path')
-        .eq('document_type', 'pedimento_simplificado')
-        .in('rfc', appConfig.upload.rfcs)
-        .not('arela_path', 'is', null);
-
-      // Add year filter if UPLOAD_YEARS is configured
-      if (appConfig.upload.years && appConfig.upload.years.length > 0) {
-        pedimentoQuery = pedimentoQuery.in('year', appConfig.upload.years);
-      }
-
       const { data: pedimentoPage, error: pedimentoError } =
-        await pedimentoQuery
-          .range(offset, offset + pageSize - 1)
-          .order('created_at');
+        await sourceService.fetchPedimentosByRfc({
+          rfcs: appConfig.upload.rfcs,
+          years: appConfig.upload.years || [],
+          offset,
+          limit: pageSize,
+        });
 
       if (pedimentoError) {
         const errorMsg = `Error fetching pedimento RFC records page ${pageNumber}: ${pedimentoError.message}`;
@@ -926,11 +942,13 @@ export class DatabaseService {
     const uniqueArelaPaths = [
       ...new Set(allPedimentoRecords.map((r) => r.arela_path)),
     ];
+    // pageNumber represents the current page (starts at 1)
+    const totalPages = pageNumber;
     console.log(
-      `📋 Found ${allPedimentoRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths for specified RFCs across ${pageNumber - 1} pages`,
+      `📋 Found ${allPedimentoRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths for specified RFCs across ${totalPages} page${totalPages !== 1 ? 's' : ''}`,
     );
     logger.info(
-      `Found ${allPedimentoRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths across ${pageNumber - 1} pages`,
+      `Found ${allPedimentoRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths across ${totalPages} page${totalPages !== 1 ? 's' : ''}`,
     );
 
     // Step 2: Process files with optimized single query per chunk
@@ -940,6 +958,7 @@ export class DatabaseService {
     let globalFileCount = 0;
     const arelaPathChunkSize = 50;
     const batchSize = parseInt(options.batchSize) || 10;
+    const filePageSize = 1000; // Supabase limit per request
 
     // Import performance configuration
     const { performance: perfConfig } = appConfig;
@@ -959,28 +978,59 @@ export class DatabaseService {
         `   Processing arela_path chunk ${chunkNumber}/${totalChunks} (${arelaPathChunk.length} paths)`,
       );
 
-      // Fetch all files for this chunk in a single query
-      const { data: batch, error: queryError } = await supabase
-        .from('uploader')
-        .select('id, original_path, arela_path, filename, rfc, document_type')
-        .in('arela_path', arelaPathChunk)
-        .neq('status', 'file-uploaded')
-        .order('created_at');
+      // Fetch all files for this chunk with pagination to handle >1000 records
+      let allChunkFiles = [];
+      let fileOffset = 0;
+      let hasMoreFiles = true;
+      let filePageNum = 1;
+
+      while (hasMoreFiles) {
+        const { data: batch, error: queryError } =
+          await sourceService.fetchFilesForUpload({
+            arelaPaths: arelaPathChunk,
+            offset: fileOffset,
+            limit: filePageSize,
+          });
 
-      if (queryError) {
-        const errorMsg = `Error fetching files for chunk ${chunkNumber}: ${queryError.message}`;
-        logger.error(errorMsg);
-        throw new Error(errorMsg);
-      }
+        if (queryError) {
+          const errorMsg = `Error fetching files for chunk ${chunkNumber} page ${filePageNum}: ${queryError.message}`;
+          logger.error(errorMsg);
+          throw new Error(errorMsg);
+        }
 
-      if (!batch || batch.length === 0) {
-        console.log(
-          `   ℹ️  Chunk ${chunkNumber}/${totalChunks}: No files to upload`,
+        if (!batch || batch.length === 0) {
+          hasMoreFiles = false;
+          if (filePageNum === 1) {
+            // No files found at all for this chunk
+            console.log(
+              `   ℹ️  Chunk ${chunkNumber}/${totalChunks}: No files to upload`,
+            );
+          }
+          break;
+        }
+
+        logger.debug(
+          `Chunk ${chunkNumber} page ${filePageNum}: fetched ${batch.length} files`,
         );
+        allChunkFiles = allChunkFiles.concat(batch);
+
+        // Check if we need more pages
+        if (batch.length < filePageSize) {
+          hasMoreFiles = false;
+          logger.debug(
+            `Chunk ${chunkNumber}: Completed pagination with ${allChunkFiles.length} total files`,
+          );
+        } else {
+          fileOffset += filePageSize;
+          filePageNum++;
+        }
+      }
+
+      if (allChunkFiles.length === 0) {
         continue;
       }
 
-      const chunkFileCount = batch.length;
+      const chunkFileCount = allChunkFiles.length;
       globalFileCount += chunkFileCount;
 
       console.log(
@@ -989,20 +1039,21 @@ export class DatabaseService {
 
       // Process this batch of files immediately using concurrent processing
       // Split batch into upload batches
-      for (let j = 0; j < batch.length; j += batchSize) {
-        const uploadBatch = batch.slice(j, j + batchSize);
+      for (let j = 0; j < allChunkFiles.length; j += batchSize) {
+        const uploadBatch = allChunkFiles.slice(j, j + batchSize);
         const batchNum = Math.floor(j / batchSize) + 1;
-        const totalBatches = Math.ceil(batch.length / batchSize);
+        const totalBatches = Math.ceil(allChunkFiles.length / batchSize);
 
         console.log(
           `   📦 Processing upload batch ${batchNum}/${totalBatches} within chunk ${chunkNumber} (${uploadBatch.length} files)`,
         );
 
         // Process batch using concurrent processing similar to UploadCommand
+        // In cross-tenant mode: targetService for uploading, sourceService for reading
         const batchResults = await this.#processRfcBatch(
           uploadBatch,
-          uploadService,
-          supabase,
+          targetService, // Used for uploading files
+          sourceService, // Used for reading metadata
           options,
           maxConcurrency,
         );
@@ -1061,7 +1112,14 @@ export class DatabaseService {
    * @returns {Promise<Array>} Array of files ready for upload
    */
   async getFilesReadyForUpload(options = {}) {
-    const supabase = await this.#getSupabaseClient();
+    // Get API service
+    const apiService = await uploadServiceFactory.getUploadService();
+
+    if (apiService.getServiceName() !== 'Arela API') {
+      throw new Error(
+        'API service is required for querying files. Please configure ARELA_API_URL and ARELA_API_TOKEN.',
+      );
+    }
 
     logger.info('Querying files ready for upload...');
     console.log('🔍 Querying files ready for upload...');
@@ -1085,23 +1143,13 @@ export class DatabaseService {
       '🎯 Finding pedimento_simplificado documents for specified RFCs with arela_path...',
     );
 
-    let pedimentoReadyQuery = supabase
-      .from('uploader')
-      .select('arela_path')
-      .eq('document_type', 'pedimento_simplificado')
-      .in('rfc', uploadRfcs)
-      .not('arela_path', 'is', null);
-
-    // Add year filter if UPLOAD_YEARS is configured
-    if (appConfig.upload.years && appConfig.upload.years.length > 0) {
-      pedimentoReadyQuery = pedimentoReadyQuery.in(
-        'year',
-        appConfig.upload.years,
-      );
-    }
-
     const { data: pedimentoRecords, error: pedimentoError } =
-      await pedimentoReadyQuery;
+      await apiService.fetchPedimentosByRfc({
+        rfcs: uploadRfcs,
+        years: appConfig.upload.years || [],
+        offset: 0,
+        limit: 10000, // Fetch all pedimentos in one go for this query
+      });
 
     if (pedimentoError) {
       throw new Error(
@@ -1134,22 +1182,19 @@ export class DatabaseService {
     for (let i = 0; i < uniqueArelaPaths.length; i += chunkSize) {
       const pathChunk = uniqueArelaPaths.slice(i, i + chunkSize);
 
-      // Query with pagination to get all results (Supabase default limit is 1000)
+      // Query with pagination to get all results
       let chunkFiles = [];
       let from = 0;
       const pageSize = 1000;
       let hasMoreData = true;
 
       while (hasMoreData) {
-        const { data: pageData, error: chunkError } = await supabase
-          .from('uploader')
-          .select(
-            'id, original_path, arela_path, filename, rfc, document_type, status',
-          )
-          .in('arela_path', pathChunk)
-          .neq('status', 'file-uploaded')
-          .not('original_path', 'is', null)
-          .range(from, from + pageSize - 1);
+        const { data: pageData, error: chunkError } =
+          await apiService.fetchFilesForUpload({
+            arelaPaths: pathChunk,
+            offset: from,
+            limit: pageSize,
+          });
 
         if (chunkError) {
           throw new Error(
@@ -1213,7 +1258,7 @@ export class DatabaseService {
    * Process a batch of files using concurrent processing for RFC uploads
    * @param {Array} files - Files to process in this batch
    * @param {Object} uploadService - Upload service instance
-   * @param {Object} supabase - Supabase client
+   * @param {Object} apiService - API service instance for database updates
    * @param {Object} options - Upload options
    * @param {number} maxConcurrency - Maximum concurrent operations
    * @returns {Promise<Object>} Batch processing results
@@ -1221,7 +1266,7 @@ export class DatabaseService {
   async #processRfcBatch(
     files,
     uploadService,
-    supabase,
+    apiService,
     options,
     maxConcurrency,
   ) {
@@ -1245,7 +1290,7 @@ export class DatabaseService {
           return await this.#processRfcSingleFile(
             file,
             uploadService,
-            supabase,
+            apiService,
             options,
             fs,
           );
@@ -1291,7 +1336,7 @@ export class DatabaseService {
           return await this.#processRfcApiBatch(
             chunk,
             uploadService,
-            supabase,
+            apiService,
             options,
             fs,
           );
@@ -1319,20 +1364,20 @@ export class DatabaseService {
   /**
    * Process a single file for RFC upload (Supabase mode)
    */
-  async #processRfcSingleFile(file, uploadService, supabase, options, fs) {
+  async #processRfcSingleFile(file, uploadService, apiService, options, fs) {
     try {
       // Check if file exists
       if (!fs.existsSync(file.original_path)) {
         logger.warn(
           `File not found: ${file.filename} at ${file.original_path}`,
         );
-        await supabase
-          .from('uploader')
-          .update({
+        await apiService.updateFileStatus([
+          {
+            id: file.id,
             status: 'file-not-found',
             message: 'File no longer exists at original path',
-          })
-          .eq('id', file.id);
+          },
+        ]);
         return { success: false, error: 'File not found' };
       }
 
@@ -1359,24 +1404,25 @@ export class DatabaseService {
 
       // Check upload result before updating database status
       if (uploadResult.success) {
-        await supabase
-          .from('uploader')
-          .update({
+        await apiService.updateFileStatus([
+          {
+            id: file.id,
             status: 'file-uploaded',
             message: 'Successfully uploaded to Supabase',
-          })
-          .eq('id', file.id);
+            processing_status: 'UPLOADED',
+          },
+        ]);
 
         logger.info(`✅ Uploaded: ${file.filename}`);
         return { success: true, filename: file.filename };
       } else {
-        await supabase
-          .from('uploader')
-          .update({
+        await apiService.updateFileStatus([
+          {
+            id: file.id,
             status: 'upload-error',
             message: `Upload failed: ${uploadResult.error}`,
-          })
-          .eq('id', file.id);
+          },
+        ]);
 
         logger.error(
           `❌ Upload failed: ${file.filename} - ${uploadResult.error}`,
@@ -1392,13 +1438,13 @@ export class DatabaseService {
         `❌ Error processing file ${file.filename}: ${error.message}`,
       );
 
-      await supabase
-        .from('uploader')
-        .update({
+      await apiService.updateFileStatus([
+        {
+          id: file.id,
           status: 'upload-error',
           message: `Processing error: ${error.message}`,
-        })
-        .eq('id', file.id);
+        },
+      ]);
 
       return { success: false, error: error.message, filename: file.filename };
     }
@@ -1407,7 +1453,7 @@ export class DatabaseService {
   /**
    * Process multiple files in a single API batch call (API service mode)
    */
-  async #processRfcApiBatch(files, uploadService, supabase, options, fs) {
+  async #processRfcApiBatch(files, uploadService, apiService, options, fs) {
     let processed = 0;
     let uploaded = 0;
     let errors = 0;
@@ -1439,15 +1485,15 @@ export class DatabaseService {
       }
 
       // Update invalid files in database
-      for (const file of invalidFiles) {
-        await supabase
-          .from('uploader')
-          .update({
+      if (invalidFiles.length > 0) {
+        await apiService.updateFileStatus(
+          invalidFiles.map((file) => ({
+            id: file.id,
             status: 'file-not-found',
             message: 'File no longer exists at original path',
-          })
-          .eq('id', file.id);
-        errors++;
+          })),
+        );
+        errors += invalidFiles.length;
       }
 
       // Process valid files in batch if any exist
@@ -1464,9 +1510,15 @@ export class DatabaseService {
         }
 
         // Make single API call with multiple files
+        // Include RFC for multi-database routing (required for cross-tenant uploads)
         const uploadResult = await uploadService.upload(
           validFiles.map((f) => f.fileData),
-          { folderStructure: fullFolderStructure },
+          {
+            folderStructure: fullFolderStructure,
+            rfc: sampleFile.rfc, // For cross-tenant: routes to correct client DB
+            autoDetect: true, // Enable detection on target API
+            autoOrganize: true, // Enable organization on target API
+          },
         );
 
         if (uploadResult.success && uploadResult.data) {
@@ -1497,6 +1549,9 @@ export class DatabaseService {
             `🔍 Expected filenames: ${Array.from(fileNameToRecord.keys()).join(', ')}`,
           );
 
+          // Prepare status updates
+          const statusUpdates = [];
+
           // Handle successfully uploaded files
           if (apiResult.uploaded && apiResult.uploaded.length > 0) {
             const successfulFileIds = [];
@@ -1521,6 +1576,13 @@ export class DatabaseService {
                 successfulFileIds.push(dbRecord.id);
                 matchedFilenames.push(possibleFilename);
                 logger.debug(`✅ Matched file: ${possibleFilename}`);
+
+                statusUpdates.push({
+                  id: dbRecord.id,
+                  status: 'file-uploaded',
+                  message: 'Successfully uploaded to Arela API (batch)',
+                  processing_status: 'UPLOADED',
+                });
               } else {
                 logger.warn(
                   `⚠️ Could not match uploaded file with any known filename: ${JSON.stringify(uploadedFile)}`,
@@ -1536,29 +1598,21 @@ export class DatabaseService {
               logger.warn(
                 `🔄 Fallback: No individual file matches found, but API indicates ${apiResult.uploaded.length} uploads. Marking all ${validFiles.length} batch files as uploaded.`,
               );
-              validFiles.forEach((f) => successfulFileIds.push(f.dbRecord.id));
-            }
-
-            if (successfulFileIds.length > 0) {
-              await supabase
-                .from('uploader')
-                .update({
+              validFiles.forEach((f) => {
+                statusUpdates.push({
+                  id: f.dbRecord.id,
                   status: 'file-uploaded',
                   message: 'Successfully uploaded to Arela API (batch)',
-                })
-                .in('id', successfulFileIds);
-
-              uploaded += successfulFileIds.length;
-              logger.info(
-                `✅ Batch upload successful: ${successfulFileIds.length} files uploaded`,
-              );
+                  processing_status: 'UPLOADED',
+                });
+              });
             }
+
+            uploaded += successfulFileIds.length || validFiles.length;
           }
 
           // Handle failed files
           if (apiResult.errors && apiResult.errors.length > 0) {
-            const failedFileIds = [];
-
             apiResult.errors.forEach((errorInfo) => {
               // Try multiple possible property names for filename in errors
               const possibleFilename =
@@ -1571,32 +1625,21 @@ export class DatabaseService {
 
               const dbRecord = fileNameToRecord.get(possibleFilename);
               if (dbRecord) {
-                failedFileIds.push(dbRecord.id);
+                statusUpdates.push({
+                  id: dbRecord.id,
+                  status: 'upload-error',
+                  message: `Upload failed: ${errorInfo.error || 'Unknown error'}`,
+                });
+                errors++;
               } else {
                 logger.warn(
                   `⚠️ Could not match error file: ${JSON.stringify(errorInfo)}`,
                 );
               }
             });
-
-            if (failedFileIds.length > 0) {
-              await supabase
-                .from('uploader')
-                .update({
-                  status: 'upload-error',
-                  message: `Upload failed: ${apiResult.errors[0].error}`,
-                })
-                .in('id', failedFileIds);
-
-              errors += failedFileIds.length;
-              logger.error(
-                `❌ Batch upload errors: ${failedFileIds.length} files failed`,
-              );
-            }
           }
 
           // Handle any remaining files that weren't in uploaded or errors arrays
-          // Use robust filename extraction for both uploaded and error files
           const extractFilename = (fileObj) => {
             return (
               fileObj.fileName ||
@@ -1616,26 +1659,24 @@ export class DatabaseService {
           const unprocessedFiles = validFiles.filter(
             (f) => !processedFileNames.has(f.fileData.name),
           );
+
           if (unprocessedFiles.length > 0) {
-            // Only mark as unprocessed if we haven't already handled all files through fallback logic
-            // If we used fallback (all files marked as uploaded), don't mark any as unprocessed
             const alreadyHandledCount = uploaded + errors;
             const shouldMarkUnprocessed =
               alreadyHandledCount < validFiles.length;
 
             if (shouldMarkUnprocessed) {
-              const unprocessedIds = unprocessedFiles.map((f) => f.dbRecord.id);
-              await supabase
-                .from('uploader')
-                .update({
+              unprocessedFiles.forEach((f) => {
+                statusUpdates.push({
+                  id: f.dbRecord.id,
                   status: 'upload-error',
                   message: 'File not found in API response',
-                })
-                .in('id', unprocessedIds);
+                });
+              });
+              errors += unprocessedFiles.length;
 
-              errors += unprocessedIds.length;
               logger.warn(
-                `⚠️ Unprocessed files: ${unprocessedIds.length} files not found in API response`,
+                `⚠️ Unprocessed files: ${unprocessedFiles.length} files not found in API response`,
               );
               logger.debug(
                 `🔍 API response uploaded array: ${JSON.stringify(apiResult.uploaded)}`,
@@ -1649,16 +1690,30 @@ export class DatabaseService {
               );
             }
           }
+
+          // Batch update all status changes
+          if (statusUpdates.length > 0) {
+            const updateResult =
+              await apiService.updateFileStatus(statusUpdates);
+            if (!updateResult.success) {
+              logger.error(
+                `Some status updates failed: ${updateResult.errors?.length || 0} errors`,
+              );
+            } else {
+              logger.info(
+                `✅ Batch upload successful: ${uploaded} files uploaded`,
+              );
+            }
+          }
         } else {
           // Complete batch failure - mark all files as failed
-          const fileIds = validFiles.map((f) => f.dbRecord.id);
-          await supabase
-            .from('uploader')
-            .update({
-              status: 'upload-error',
-              message: uploadResult.error || 'Batch upload failed',
-            })
-            .in('id', fileIds);
+          const failureUpdates = validFiles.map((f) => ({
+            id: f.dbRecord.id,
+            status: 'upload-error',
+            message: uploadResult.error || 'Batch upload failed',
+          }));
+
+          await apiService.updateFileStatus(failureUpdates);
 
           errors += validFiles.length;
           logger.error(
@@ -1670,20 +1725,445 @@ export class DatabaseService {
       logger.error(`❌ Error processing batch: ${error.message}`);
 
       // Mark all files as failed
-      const fileIds = files.map((f) => f.id);
-      await supabase
-        .from('uploader')
-        .update({
-          status: 'upload-error',
-          message: `Batch processing error: ${error.message}`,
-        })
-        .in('id', fileIds);
+      const failureUpdates = files.map((f) => ({
+        id: f.id,
+        status: 'upload-error',
+        message: `Batch processing error: ${error.message}`,
+      }));
+
+      await apiService.updateFileStatus(failureUpdates);
 
       errors += files.length;
     }
 
     return { processed, uploaded, errors };
   }
+
+  /**
+   * Insert upload session event into watch_uploads table
+   * @param {Object} uploadEvent - Upload event from LoggingService
+   * @param {string} sessionId - Session ID for tracking
+   * @returns {Promise<Object>} Inserted record
+   */
+  async insertUploadEvent(uploadEvent, sessionId) {
+    const supabase = await this.#getSupabaseClient();
+
+    const record = {
+      session_id: sessionId,
+      timestamp: uploadEvent.timestamp || new Date().toISOString(),
+      strategy: uploadEvent.strategy, // 'individual', 'batch', 'full-structure'
+      file_count: uploadEvent.fileCount || 0,
+      success_count: uploadEvent.successCount || 0,
+      failure_count: uploadEvent.failureCount || 0,
+      retry_count: uploadEvent.retryCount || 0,
+      duration_ms: uploadEvent.duration || 0,
+      status: uploadEvent.status || 'completed',
+      metadata: uploadEvent.metadata || null,
+    };
+
+    try {
+      const { data, error } = await this.#queryWithRetry(async () => {
+        return await supabase.from('watch_uploads').insert([record]).select();
+      }, `insert upload event for session ${sessionId}`);
+
+      if (error) {
+        logger.error(`Failed to insert upload event: ${error.message}`);
+        throw error;
+      }
+
+      return data[0];
+    } catch (error) {
+      logger.error(`Error inserting upload event: ${error.message}`);
+      throw error;
+    }
+  }
+
+  /**
+   * Insert retry event into watch_events table
+   * @param {string} uploadEventId - ID of the parent upload event
+   * @param {string} sessionId - Session ID for tracking
+   * @param {Object} retryEvent - Retry event from LoggingService
+   * @returns {Promise<Object>} Inserted record
+   */
+  async insertRetryEvent(uploadEventId, sessionId, retryEvent) {
+    const supabase = await this.#getSupabaseClient();
+
+    const record = {
+      upload_event_id: uploadEventId,
+      session_id: sessionId,
+      timestamp: retryEvent.timestamp || new Date().toISOString(),
+      attempt_number: retryEvent.attemptNumber || 0,
+      error_message: retryEvent.error || null,
+      backoff_ms: retryEvent.backoffMs || 0,
+      type: 'retry',
+    };
+
+    try {
+      const { data, error } = await this.#queryWithRetry(async () => {
+        return await supabase.from('watch_events').insert([record]).select();
+      }, `insert retry event for upload ${uploadEventId}`);
+
+      if (error) {
+        logger.error(`Failed to insert retry event: ${error.message}`);
+        throw error;
+      }
+
+      return data[0];
+    } catch (error) {
+      logger.error(`Error inserting retry event: ${error.message}`);
+      throw error;
+    }
+  }
+
+  /**
+   * Get upload history for a session
+   * @param {string} sessionId - Session ID to query
+   * @param {Object} options - Query options (limit, offset, strategy filter)
+   * @returns {Promise<Array>} Array of upload events
+   */
+  async getSessionUploadHistory(sessionId, options = {}) {
+    const supabase = await this.#getSupabaseClient();
+    const limit = options.limit || 100;
+    const offset = options.offset || 0;
+
+    try {
+      let query = supabase
+        .from('watch_uploads')
+        .select('*')
+        .eq('session_id', sessionId)
+        .order('timestamp', { ascending: false })
+        .range(offset, offset + limit - 1);
+
+      // Filter by strategy if provided
+      if (options.strategy) {
+        query = query.eq('strategy', options.strategy);
+      }
+
+      const { data, error } = await this.#queryWithRetry(async () => {
+        return await query;
+      }, `fetch upload history for session ${sessionId}`);
+
+      if (error) {
+        logger.error(`Failed to fetch upload history: ${error.message}`);
+        throw error;
+      }
+
+      return data || [];
+    } catch (error) {
+      logger.error(`Error fetching upload history: ${error.message}`);
+      return [];
+    }
+  }
+
+  /**
+   * Get retry history for an upload event
+   * @param {string} uploadEventId - Upload event ID to query
+   * @param {Object} options - Query options (limit, offset)
+   * @returns {Promise<Array>} Array of retry events
+   */
+  async getUploadRetryHistory(uploadEventId, options = {}) {
+    const supabase = await this.#getSupabaseClient();
+    const limit = options.limit || 100;
+    const offset = options.offset || 0;
+
+    try {
+      const { data, error } = await this.#queryWithRetry(async () => {
+        return await supabase
+          .from('watch_events')
+          .select('*')
+          .eq('upload_event_id', uploadEventId)
+          .eq('type', 'retry')
+          .order('timestamp', { ascending: true })
+          .range(offset, offset + limit - 1);
+      }, `fetch retry history for upload ${uploadEventId}`);
+
+      if (error) {
+        logger.error(`Failed to fetch retry history: ${error.message}`);
+        throw error;
+      }
+
+      return data || [];
+    } catch (error) {
+      logger.error(`Error fetching retry history: ${error.message}`);
+      return [];
+    }
+  }
+
+  /**
+   * Get session statistics
+   * @param {string} sessionId - Session ID to analyze
+   * @returns {Promise<Object>} Session statistics
+   */
+  async getSessionStatistics(sessionId) {
+    const supabase = await this.#getSupabaseClient();
+
+    try {
+      // Fetch all upload events for the session
+      const { data: uploads, error: uploadError } = await this.#queryWithRetry(
+        async () => {
+          return await supabase
+            .from('watch_uploads')
+            .select('*')
+            .eq('session_id', sessionId);
+        },
+        `fetch statistics for session ${sessionId}`,
+      );
+
+      if (uploadError) {
+        throw uploadError;
+      }
+
+      // Fetch all retry events for the session
+      const { data: retries, error: retryError } = await this.#queryWithRetry(
+        async () => {
+          return await supabase
+            .from('watch_events')
+            .select('*')
+            .eq('session_id', sessionId)
+            .eq('type', 'retry');
+        },
+        `fetch retry statistics for session ${sessionId}`,
+      );
+
+      if (retryError) {
+        throw retryError;
+      }
+
+      // Calculate statistics
+      const stats = {
+        sessionId,
+        totalUploadEvents: uploads?.length || 0,
+        totalRetryEvents: retries?.length || 0,
+        totalFileCount: 0,
+        totalSuccessCount: 0,
+        totalFailureCount: 0,
+        totalRetryCount: 0,
+        totalDuration: 0,
+        byStrategy: {
+          individual: {
+            uploadCount: 0,
+            totalFiles: 0,
+            totalSuccess: 0,
+            totalFailure: 0,
+            successRate: 0,
+            totalDuration: 0,
+          },
+          batch: {
+            uploadCount: 0,
+            totalFiles: 0,
+            totalSuccess: 0,
+            totalFailure: 0,
+            successRate: 0,
+            totalDuration: 0,
+          },
+          'full-structure': {
+            uploadCount: 0,
+            totalFiles: 0,
+            totalSuccess: 0,
+            totalFailure: 0,
+            successRate: 0,
+            totalDuration: 0,
+          },
+        },
+        retryStats: {
+          totalRetries: retries?.length || 0,
+          uniqueUploadsWithRetries: new Set(
+            retries?.map((r) => r.upload_event_id) || [],
+          ).size,
+          totalRetryDuration:
+            retries?.reduce((sum, r) => sum + (r.backoff_ms || 0), 0) || 0,
+        },
+      };
+
+      // Process upload events
+      if (uploads && uploads.length > 0) {
+        uploads.forEach((upload) => {
+          stats.totalFileCount += upload.file_count || 0;
+          stats.totalSuccessCount += upload.success_count || 0;
+          stats.totalFailureCount += upload.failure_count || 0;
+          stats.totalRetryCount += upload.retry_count || 0;
+          stats.totalDuration += upload.duration_ms || 0;
+
+          const strategyKey = upload.strategy || 'individual';
+          if (stats.byStrategy[strategyKey]) {
+            stats.byStrategy[strategyKey].uploadCount += 1;
+            stats.byStrategy[strategyKey].totalFiles += upload.file_count || 0;
+            stats.byStrategy[strategyKey].totalSuccess +=
+              upload.success_count || 0;
+            stats.byStrategy[strategyKey].totalFailure +=
+              upload.failure_count || 0;
+            stats.byStrategy[strategyKey].totalDuration +=
+              upload.duration_ms || 0;
+
+            // Calculate success rate
+            const totalFiles =
+              stats.byStrategy[strategyKey].totalSuccess +
+              stats.byStrategy[strategyKey].totalFailure;
+            if (totalFiles > 0) {
+              stats.byStrategy[strategyKey].successRate = (
+                (stats.byStrategy[strategyKey].totalSuccess / totalFiles) *
+                100
+              ).toFixed(2);
+            }
+          }
+        });
+      }
+
+      return stats;
+    } catch (error) {
+      logger.error(`Error calculating session statistics: ${error.message}`);
+      return null;
+    }
+  }
+
+  /**
+   * Delete old session data (cleanup)
+   * @param {number} daysOld - Delete sessions older than this many days
+   * @returns {Promise<Object>} Deletion results
+   */
+  async cleanupOldSessions(daysOld = 30) {
+    const supabase = await this.#getSupabaseClient();
+
+    try {
+      const cutoffDate = new Date();
+      cutoffDate.setDate(cutoffDate.getDate() - daysOld);
+
+      // Get sessions to delete
+      const { data: sessionsToDelete, error: fetchError } =
+        await this.#queryWithRetry(async () => {
+          return await supabase
+            .from('watch_uploads')
+            .select('session_id')
+            .lt('timestamp', cutoffDate.toISOString())
+            .distinct();
+        }, `fetch sessions older than ${daysOld} days`);
+
+      if (fetchError) {
+        throw fetchError;
+      }
+
+      let deletedUploads = 0;
+      let deletedEvents = 0;
+
+      if (sessionsToDelete && sessionsToDelete.length > 0) {
+        const sessionIds = sessionsToDelete.map((s) => s.session_id);
+
+        // Delete events
+        const { count: eventCount, error: eventError } =
+          await this.#queryWithRetry(async () => {
+            return await supabase
+              .from('watch_events')
+              .delete()
+              .in('session_id', sessionIds);
+          }, `delete events for old sessions`);
+
+        if (!eventError) {
+          deletedEvents = eventCount || 0;
+        }
+
+        // Delete uploads
+        const { count: uploadCount, error: uploadError } =
+          await this.#queryWithRetry(async () => {
+            return await supabase
+              .from('watch_uploads')
+              .delete()
+              .in('session_id', sessionIds);
+          }, `delete old session uploads`);
+
+        if (!uploadError) {
+          deletedUploads = uploadCount || 0;
+        }
+      }
+
+      return {
+        deletedUploads,
+        deletedEvents,
+        sessionsDeleted: sessionsToDelete?.length || 0,
+      };
+    } catch (error) {
+      logger.error(`Error cleaning up old sessions: ${error.message}`);
+      return { deletedUploads: 0, deletedEvents: 0, sessionsDeleted: 0 };
+    }
+  }
+
+  /**
+   * Cleanup database connections and resources
+   * Called during graceful shutdown
+   * @returns {Promise<Object>} Cleanup results
+   */
+  async cleanup() {
+    try {
+      logger.info('DatabaseService: Starting cleanup...');
+
+      // Commit any pending transactions
+      const transactionResult = await this.commitPendingTransactions();
+
+      // Close database connections
+      const closeResult = await this.closeConnections();
+
+      logger.info('DatabaseService: Cleanup complete');
+
+      return {
+        success: true,
+        transactionsCommitted: transactionResult.count,
+        connectionsClosedResult: closeResult,
+      };
+    } catch (error) {
+      logger.error(`DatabaseService: Error during cleanup: ${error.message}`);
+      return {
+        success: false,
+        error: error.message,
+      };
+    }
+  }
+
+  /**
+   * Commit any pending transactions before shutdown
+   * @private
+   * @returns {Promise<Object>} Results
+   */
+  async commitPendingTransactions() {
+    try {
+      logger.debug('DatabaseService: Committing pending transactions...');
+
+      // Note: This is a placeholder for actual transaction handling
+      // In a real implementation, you would track active transactions
+      // and ensure they are properly committed before shutdown
+
+      logger.debug('DatabaseService: Pending transactions committed');
+      return { count: 0, success: true };
+    } catch (error) {
+      logger.error(
+        `DatabaseService: Error committing transactions: ${error.message}`,
+      );
+      return { count: 0, success: false, error: error.message };
+    }
+  }
+
+  /**
+   * Close all database connections
+   * @private
+   * @returns {Promise<Object>} Results
+   */
+  async closeConnections() {
+    try {
+      logger.debug('DatabaseService: Closing database connections...');
+
+      // Close Supabase connection if available
+      if (this.supabase) {
+        // Supabase client will handle connection cleanup automatically
+        logger.debug('DatabaseService: Supabase connection cleanup initiated');
+      }
+
+      logger.info('DatabaseService: All database connections closed');
+      return { success: true };
+    } catch (error) {
+      logger.error(
+        `DatabaseService: Error closing connections: ${error.message}`,
+      );
+      return { success: false, error: error.message };
+    }
+  }
 }
 
 // Export singleton instance