@arela/uploader 0.2.7 → 0.2.9

package/src/services/DatabaseService.js

@@ -109,7 +109,7 @@ export class DatabaseService {
         .toLowerCase()
         .replace('.', '');
       const filename = file.originalName || path.basename(file.path);
-      
+
       const record = {
         document_type: null,
         size: stats.size,
@@ -121,8 +121,8 @@ export class DatabaseService {
         rfc: null,
         message: null,
         file_extension: fileExtension,
-        is_like_simplificado: fileExtension === 'pdf' && 
-          filename.toLowerCase().includes('simp'),
+        is_like_simplificado:
+          fileExtension === 'pdf' && filename.toLowerCase().includes('simp'),
         year: null,
         created_at: new Date().toISOString(),
         modified_at: stats.mtime.toISOString(),
@@ -226,8 +226,11 @@ export class DatabaseService {
           file_extension: fileExtension,
           created_at: new Date().toISOString(),
           modified_at: stats.mtime.toISOString(),
-          is_like_simplificado: fileExtension === 'pdf' && 
-            (file.originalName || path.basename(file.path)).toLowerCase().includes('simp'),
+          is_like_simplificado:
+            fileExtension === 'pdf' &&
+            (file.originalName || path.basename(file.path))
+              .toLowerCase()
+              .includes('simp'),
           year: null,
         };
 
@@ -277,9 +280,15 @@ export class DatabaseService {
           existingPaths.has(r.original_path),
         );
 
-        logger.info(
-          `Batch ${Math.floor(i / batchSize) + 1}: ${newRecords.length} new, ${updateRecords.length} updates`,
-        );
+        // Only log every 10th batch to reduce noise
+        if (
+          (Math.floor(i / batchSize) + 1) % 10 === 0 ||
+          Math.floor(i / batchSize) + 1 === 1
+        ) {
+          logger.info(
+            `Batch ${Math.floor(i / batchSize) + 1}: ${newRecords.length} new, ${updateRecords.length} updates`,
+          );
+        }
 
         // Insert new records
         if (newRecords.length > 0) {
@@ -291,7 +300,7 @@ export class DatabaseService {
             logger.error(`Error inserting new records: ${insertError.message}`);
           } else {
             totalInserted += newRecords.length;
-            logger.success(`Inserted ${newRecords.length} new records`);
+            // Only log the batch insertion, not the summary (which comes at the end)
           }
         }
 
@@ -316,7 +325,10 @@ export class DatabaseService {
             }
           }
           totalUpdated += batchUpdated;
-          logger.info(`Updated ${batchUpdated} existing records`);
+          // Reduce logging noise - only log when there are updates
+          if (batchUpdated > 0) {
+            logger.info(`Updated ${batchUpdated} existing records`);
+          }
         }
       } catch (error) {
         logger.error(
@@ -545,7 +557,9 @@ export class DatabaseService {
     const supabase = await this.#getSupabaseClient();
 
     logger.info('Phase 3: Starting arela_path and year propagation process...');
-    console.log('🔍 Finding pedimento_simplificado records with arela_path and year...');
+    console.log(
+      '🔍 Finding pedimento_simplificado records with arela_path and year...',
+    );
 
     // Get all pedimento_simplificado records that have arela_path
     const { data: pedimentoRecords, error: pedimentoError } = await supabase
@@ -601,32 +615,66 @@ export class DatabaseService {
             : existingPath + '/';
 
         // Find all files with the same base path that don't have arela_path yet
-        const { data: relatedFiles, error: relatedError } = await supabase
-          .from('uploader')
-          .select('id, filename, original_path')
-          .like('original_path', `${basePath}%`)
-          .is('arela_path', null)
-          .neq('id', pedimento.id); // Exclude the pedimento itself
+        // Use pagination to handle cases with more than 1000 related files
+        let allRelatedFiles = [];
+        let from = 0;
+        const pageSize = 1000;
+        let hasMoreData = true;
+        let paginationError = null;
 
-        if (relatedError) {
-          logger.error(
-            `Error finding related files for ${pedimento.filename}: ${relatedError.message}`,
-          );
-          totalErrors++;
+        logger.info(`Searching for related files in base path: ${basePath}`);
+
+        while (hasMoreData) {
+          const { data: relatedFilesPage, error: relatedError } = await supabase
+            .from('uploader')
+            .select('id, filename, original_path')
+            .like('original_path', `${basePath}%`)
+            .is('arela_path', null)
+            .neq('id', pedimento.id) // Exclude the pedimento itself
+            .range(from, from + pageSize - 1);
+
+          if (relatedError) {
+            logger.error(
+              `Error finding related files for ${pedimento.filename}: ${relatedError.message}`,
+            );
+            paginationError = relatedError;
+            totalErrors++;
+            break;
+          }
+
+          if (relatedFilesPage && relatedFilesPage.length > 0) {
+            allRelatedFiles = allRelatedFiles.concat(relatedFilesPage);
+
+            // Check if we got a full page, indicating there might be more data
+            if (relatedFilesPage.length < pageSize) {
+              hasMoreData = false;
+            } else {
+              from += pageSize;
+              logger.info(
+                `Fetched ${relatedFilesPage.length} files, continuing pagination (total so far: ${allRelatedFiles.length})`,
+              );
+            }
+          } else {
+            hasMoreData = false;
+          }
+        }
+
+        // Continue with error handling if pagination failed
+        if (paginationError) {
           continue;
         }
 
-        if (!relatedFiles || relatedFiles.length === 0) {
+        if (!allRelatedFiles || allRelatedFiles.length === 0) {
           logger.info(`No related files found for ${pedimento.filename}`);
           continue;
         }
 
         logger.info(
-          `Found ${relatedFiles.length} related files to update for ${pedimento.filename}`,
+          `Found ${allRelatedFiles.length} related files to update for ${pedimento.filename}`,
         );
 
         // Process files in batches
-        const fileIds = relatedFiles.map((f) => f.id);
+        const fileIds = allRelatedFiles.map((f) => f.id);
 
         for (let i = 0; i < fileIds.length; i += BATCH_SIZE) {
           const batchIds = fileIds.slice(i, i + BATCH_SIZE);
@@ -640,9 +688,9 @@ export class DatabaseService {
           try {
             const { error: updateError } = await supabase
               .from('uploader')
-              .update({ 
+              .update({
                 arela_path: folderArelaPath,
-                year: pedimento.year 
+                year: pedimento.year,
               })
               .in('id', batchIds);
 
@@ -885,133 +933,54 @@ export class DatabaseService {
     console.log(`📋 Total files to upload: ${allRelatedFiles.length}`);
     logger.info(`Total files to upload: ${allRelatedFiles.length}`);
 
-    // Step 4: Upload all related files
+    // Step 4: Upload all related files using concurrent batch processing
     let totalProcessed = 0;
     let totalUploaded = 0;
     let totalErrors = 0;
     const batchSize = parseInt(options.batchSize) || 10;
 
-    for (let i = 0; i < allRelatedFiles.length; i += batchSize) {
-      const batch = allRelatedFiles.slice(i, i + batchSize);
-
-      for (const file of batch) {
-        try {
-          totalProcessed++;
+    // Import performance configuration
+    const { performance: perfConfig } = appConfig;
+    const maxConcurrency = perfConfig?.maxApiConnections || 3;
 
-          // Check if file exists
-          if (!fs.existsSync(file.original_path)) {
-            logger.warn(
-              `File not found: ${file.filename} at ${file.original_path}`,
-            );
-            await supabase
-              .from('uploader')
-              .update({
-                status: 'file-not-found',
-                message: 'File no longer exists at original path',
-              })
-              .eq('id', file.id);
-            totalErrors++;
-            continue;
-          }
-
-          // Upload the file (handle both API and Supabase services)
-          let uploadResult;
-          if (uploadService.getServiceName() === 'Supabase') {
-            // Supabase requires single file upload with uploadPath
-            let uploadPath;
-            if (options.folderStructure && file.arela_path) {
-              // Combine folder structure with arela_path: palco/RFC/Year/Patente/Aduana/Pedimento/filename
-              uploadPath = `uploads/${options.folderStructure}/${file.arela_path}${file.filename}`;
-            } else if (file.arela_path) {
-              // Use existing arela_path: RFC/Year/Patente/Aduana/Pedimento/filename
-              uploadPath = `uploads/${file.arela_path}${file.filename}`;
-            } else {
-              // Fallback to RFC folder
-              uploadPath = `uploads/${file.rfc}/${file.filename}`;
-            }
-
-            uploadResult = await uploadService.upload(
-              [
-                {
-                  path: file.original_path,
-                  name: file.filename,
-                  contentType: 'application/octet-stream',
-                },
-              ],
-              {
-                uploadPath: uploadPath,
-              },
-            );
-            uploadResult = { success: true, data: uploadResult };
-          } else {
-            // API service supports batch uploads and returns normalized response
-            let fullFolderStructure;
-            if (options.folderStructure && file.arela_path) {
-              // Combine folder structure with arela_path: palco/RFC/Year/Patente/Aduana/Pedimento/
-              fullFolderStructure = `${options.folderStructure}/${file.arela_path}`;
-            } else if (file.arela_path) {
-              // Use existing arela_path: RFC/Year/Patente/Aduana/Pedimento/
-              fullFolderStructure = file.arela_path;
-            } else {
-              // Fallback to RFC folder
-              fullFolderStructure = `${file.rfc}/`;
-            }
+    console.log(
+      `🚀 Starting batch upload: ${allRelatedFiles.length} files in batches of ${batchSize}`,
+    );
+    console.log(
+      `⚡ Concurrent processing: up to ${maxConcurrency} parallel operations`,
+    );
 
-            uploadResult = await uploadService.upload(
-              [
-                {
-                  path: file.original_path,
-                  name: file.filename,
-                  contentType: 'application/octet-stream',
-                },
-              ],
-              {
-                folderStructure: fullFolderStructure,
-              },
-            );
-          }
+    // Process files in batches with concurrent processing
+    for (let i = 0; i < allRelatedFiles.length; i += batchSize) {
+      const batch = allRelatedFiles.slice(i, i + batchSize);
+      const batchNum = Math.floor(i / batchSize) + 1;
+      const totalBatches = Math.ceil(allRelatedFiles.length / batchSize);
 
-          if (uploadResult.success) {
-            // Update database status
-            await supabase
-              .from('uploader')
-              .update({
-                status: 'file-uploaded',
-                message: 'Successfully uploaded to Arela API',
-              })
-              .eq('id', file.id);
+      console.log(
+        `📦 Processing batch ${batchNum}/${totalBatches} (${batch.length} files)`,
+      );
 
-            totalUploaded++;
-            logger.info(`Uploaded: ${file.filename}`);
-          } else {
-            await supabase
-              .from('uploader')
-              .update({
-                status: 'upload-error',
-                message: uploadResult.error || 'Upload failed',
-              })
-              .eq('id', file.id);
+      // Process batch using concurrent processing similar to UploadCommand
+      const batchResults = await this.#processRfcBatch(
+        batch,
+        uploadService,
+        supabase,
+        options,
+        maxConcurrency,
+      );
 
-            totalErrors++;
-            logger.error(
-              `Upload failed: ${file.filename} - ${uploadResult.error}`,
-            );
-          }
-        } catch (error) {
-          totalErrors++;
-          logger.error(
-            `Error processing file ${file.filename}: ${error.message}`,
-          );
+      totalProcessed += batchResults.processed;
+      totalUploaded += batchResults.uploaded;
+      totalErrors += batchResults.errors;
 
-          await supabase
-            .from('uploader')
-            .update({
-              status: 'upload-error',
-              message: `Processing error: ${error.message}`,
-            })
-            .eq('id', file.id);
-        }
-      }
+      // Progress update
+      const progress = (
+        ((i + batch.length) / allRelatedFiles.length) *
+        100
+      ).toFixed(1);
+      console.log(
+        `📊 Batch ${batchNum} complete - Progress: ${progress}% (${totalUploaded}/${allRelatedFiles.length} uploaded)`,
+      );
     }
 
     const result = {
@@ -1106,22 +1075,44 @@ export class DatabaseService {
     for (let i = 0; i < uniqueArelaPaths.length; i += chunkSize) {
       const pathChunk = uniqueArelaPaths.slice(i, i + chunkSize);
 
-      const { data: chunkFiles, error: chunkError } = await supabase
-        .from('uploader')
-        .select(
-          'id, original_path, arela_path, filename, rfc, document_type, status',
-        )
-        .in('arela_path', pathChunk)
-        .neq('status', 'file-uploaded')
-        .not('original_path', 'is', null);
+      // Query with pagination to get all results (Supabase default limit is 1000)
+      let chunkFiles = [];
+      let from = 0;
+      const pageSize = 1000;
+      let hasMoreData = true;
 
-      if (chunkError) {
-        throw new Error(
-          `Error querying files for arela_paths chunk: ${chunkError.message}`,
-        );
+      while (hasMoreData) {
+        const { data: pageData, error: chunkError } = await supabase
+          .from('uploader')
+          .select(
+            'id, original_path, arela_path, filename, rfc, document_type, status',
+          )
+          .in('arela_path', pathChunk)
+          .neq('status', 'file-uploaded')
+          .not('original_path', 'is', null)
+          .range(from, from + pageSize - 1);
+
+        if (chunkError) {
+          throw new Error(
+            `Error querying files for arela_paths chunk: ${chunkError.message}`,
+          );
+        }
+
+        if (pageData && pageData.length > 0) {
+          chunkFiles = chunkFiles.concat(pageData);
+
+          // Check if we got a full page, indicating there might be more data
+          if (pageData.length < pageSize) {
+            hasMoreData = false;
+          } else {
+            from += pageSize;
+          }
+        } else {
+          hasMoreData = false;
+        }
       }
 
-      if (chunkFiles && chunkFiles.length > 0) {
+      if (chunkFiles.length > 0) {
         allReadyFiles = allReadyFiles.concat(chunkFiles);
       }
     }
@@ -1158,6 +1149,482 @@ export class DatabaseService {
 
     return readyFiles || [];
   }
+
+  /**
+   * Process a batch of files using concurrent processing for RFC uploads
+   * @param {Array} files - Files to process in this batch
+   * @param {Object} uploadService - Upload service instance
+   * @param {Object} supabase - Supabase client
+   * @param {Object} options - Upload options
+   * @param {number} maxConcurrency - Maximum concurrent operations
+   * @returns {Promise<Object>} Batch processing results
+   */
+  async #processRfcBatch(
+    files,
+    uploadService,
+    supabase,
+    options,
+    maxConcurrency,
+  ) {
+    const fs = (await import('fs')).default;
+
+    let processed = 0;
+    let uploaded = 0;
+    let errors = 0;
+
+    // For Supabase, process files individually (required by service)
+    if (uploadService.getServiceName() === 'Supabase') {
+      // Process files in concurrent chunks within the batch
+      const chunks = [];
+      for (let i = 0; i < files.length; i += maxConcurrency) {
+        chunks.push(files.slice(i, i + maxConcurrency));
+      }
+
+      // Process each chunk concurrently
+      for (const chunk of chunks) {
+        const chunkPromises = chunk.map(async (file) => {
+          return await this.#processRfcSingleFile(
+            file,
+            uploadService,
+            supabase,
+            options,
+            fs,
+          );
+        });
+
+        // Wait for all files in this chunk to complete
+        const chunkResults = await Promise.allSettled(chunkPromises);
+
+        // Count results
+        for (const result of chunkResults) {
+          processed++;
+          if (result.status === 'fulfilled' && result.value.success) {
+            uploaded++;
+          } else {
+            errors++;
+          }
+        }
+      }
+    } else {
+      // For API service, use true batch processing (multiple files per API call)
+      const apiChunks = [];
+      const apiChunkSize = Math.min(
+        5,
+        Math.ceil(files.length / maxConcurrency),
+      ); // 5 files per API call, or distribute evenly
+
+      for (let i = 0; i < files.length; i += apiChunkSize) {
+        apiChunks.push(files.slice(i, i + apiChunkSize));
+      }
+
+      console.log(
+        `   🚀 Processing ${apiChunks.length} API calls with ${apiChunkSize} files each (max ${maxConcurrency} concurrent)`,
+      );
+
+      // Process API chunks with controlled concurrency
+      const concurrentChunks = [];
+      for (let i = 0; i < apiChunks.length; i += maxConcurrency) {
+        concurrentChunks.push(apiChunks.slice(i, i + maxConcurrency));
+      }
+
+      for (const concurrentSet of concurrentChunks) {
+        const batchPromises = concurrentSet.map(async (chunk) => {
+          return await this.#processRfcApiBatch(
+            chunk,
+            uploadService,
+            supabase,
+            options,
+            fs,
+          );
+        });
+
+        // Wait for all concurrent batches to complete
+        const batchResults = await Promise.allSettled(batchPromises);
+
+        // Count results
+        for (const result of batchResults) {
+          if (result.status === 'fulfilled') {
+            processed += result.value.processed;
+            uploaded += result.value.uploaded;
+            errors += result.value.errors;
+          } else {
+            errors += result.value?.processed || 0;
+          }
+        }
+      }
+    }
+
+    return { processed, uploaded, errors };
+  }
+
+  /**
+   * Process a single file for RFC upload (Supabase mode)
+   */
+  async #processRfcSingleFile(file, uploadService, supabase, options, fs) {
+    try {
+      // Check if file exists
+      if (!fs.existsSync(file.original_path)) {
+        logger.warn(
+          `File not found: ${file.filename} at ${file.original_path}`,
+        );
+        await supabase
+          .from('uploader')
+          .update({
+            status: 'file-not-found',
+            message: 'File no longer exists at original path',
+          })
+          .eq('id', file.id);
+        return { success: false, error: 'File not found' };
+      }
+
+      // Supabase requires single file upload with uploadPath
+      let uploadPath;
+      if (options.folderStructure && file.arela_path) {
+        uploadPath = `uploads/${options.folderStructure}/${file.arela_path}${file.filename}`;
+      } else if (file.arela_path) {
+        uploadPath = `uploads/${file.arela_path}${file.filename}`;
+      } else {
+        uploadPath = `uploads/${file.rfc}/${file.filename}`;
+      }
+
+      const uploadResult = await uploadService.upload(
+        [
+          {
+            path: file.original_path,
+            name: file.filename,
+            contentType: 'application/octet-stream',
+          },
+        ],
+        { uploadPath: uploadPath },
+      );
+
+      // Check upload result before updating database status
+      if (uploadResult.success) {
+        await supabase
+          .from('uploader')
+          .update({
+            status: 'file-uploaded',
+            message: 'Successfully uploaded to Supabase',
+          })
+          .eq('id', file.id);
+
+        logger.info(`✅ Uploaded: ${file.filename}`);
+        return { success: true, filename: file.filename };
+      } else {
+        await supabase
+          .from('uploader')
+          .update({
+            status: 'upload-error',
+            message: `Upload failed: ${uploadResult.error}`,
+          })
+          .eq('id', file.id);
+
+        logger.error(
+          `❌ Upload failed: ${file.filename} - ${uploadResult.error}`,
+        );
+        return {
+          success: false,
+          error: uploadResult.error,
+          filename: file.filename,
+        };
+      }
+    } catch (error) {
+      logger.error(
+        `❌ Error processing file ${file.filename}: ${error.message}`,
+      );
+
+      await supabase
+        .from('uploader')
+        .update({
+          status: 'upload-error',
+          message: `Processing error: ${error.message}`,
+        })
+        .eq('id', file.id);
+
+      return { success: false, error: error.message, filename: file.filename };
+    }
+  }
+
+  /**
+   * Process multiple files in a single API batch call (API service mode)
+   */
+  async #processRfcApiBatch(files, uploadService, supabase, options, fs) {
+    let processed = 0;
+    let uploaded = 0;
+    let errors = 0;
+
+    try {
+      // Prepare files for batch upload
+      const validFiles = [];
+      const invalidFiles = [];
+
+      for (const file of files) {
+        processed++;
+
+        if (!fs.existsSync(file.original_path)) {
+          logger.warn(
+            `File not found: ${file.filename} at ${file.original_path}`,
+          );
+          invalidFiles.push(file);
+          continue;
+        }
+
+        validFiles.push({
+          fileData: {
+            path: file.original_path,
+            name: file.filename,
+            contentType: 'application/octet-stream',
+          },
+          dbRecord: file,
+        });
+      }
+
+      // Update invalid files in database
+      for (const file of invalidFiles) {
+        await supabase
+          .from('uploader')
+          .update({
+            status: 'file-not-found',
+            message: 'File no longer exists at original path',
+          })
+          .eq('id', file.id);
+        errors++;
+      }
+
+      // Process valid files in batch if any exist
+      if (validFiles.length > 0) {
+        // Determine folder structure (all files in this batch should have same arela_path)
+        const sampleFile = validFiles[0].dbRecord;
+        let fullFolderStructure;
+        if (options.folderStructure && sampleFile.arela_path) {
+          fullFolderStructure = `${options.folderStructure}/${sampleFile.arela_path}`;
+        } else if (sampleFile.arela_path) {
+          fullFolderStructure = sampleFile.arela_path;
+        } else {
+          fullFolderStructure = `${sampleFile.rfc}/`;
+        }
+
+        // Make single API call with multiple files
+        const uploadResult = await uploadService.upload(
+          validFiles.map((f) => f.fileData),
+          { folderStructure: fullFolderStructure },
+        );
+
+        if (uploadResult.success && uploadResult.data) {
+          const apiResult = uploadResult.data;
+
+          logger.info(
+            `📋 Processing API response: ${apiResult.uploaded?.length || 0} uploaded, ${apiResult.errors?.length || 0} errors`,
+          );
+
+          // Debug logging to understand API response structure
+          logger.debug(
+            `🔍 API Response structure: ${JSON.stringify(apiResult, null, 2)}`,
+          );
+          if (apiResult.uploaded && apiResult.uploaded.length > 0) {
+            logger.debug(
+              `🔍 First uploaded file structure: ${JSON.stringify(apiResult.uploaded[0], null, 2)}`,
+            );
+          }
+
+          // Create filename to file mapping for quick lookup
+          const fileNameToRecord = new Map();
+          validFiles.forEach((f) => {
+            fileNameToRecord.set(f.fileData.name, f.dbRecord);
+          });
+
+          // Debug: Log expected filenames
+          logger.debug(
+            `🔍 Expected filenames: ${Array.from(fileNameToRecord.keys()).join(', ')}`,
+          );
+
+          // Handle successfully uploaded files
+          if (apiResult.uploaded && apiResult.uploaded.length > 0) {
+            const successfulFileIds = [];
+            const matchedFilenames = [];
+
+            apiResult.uploaded.forEach((uploadedFile) => {
+              // Try multiple possible property names for filename
+              const possibleFilename =
+                uploadedFile.fileName ||
+                uploadedFile.filename ||
+                uploadedFile.name ||
+                uploadedFile.file_name ||
+                uploadedFile.originalName ||
+                uploadedFile.original_name;
+
+              logger.debug(
+                `🔍 Trying to match uploaded file: ${JSON.stringify(uploadedFile)}`,
+              );
+
+              const dbRecord = fileNameToRecord.get(possibleFilename);
+              if (dbRecord) {
+                successfulFileIds.push(dbRecord.id);
+                matchedFilenames.push(possibleFilename);
+                logger.debug(`✅ Matched file: ${possibleFilename}`);
+              } else {
+                logger.warn(
+                  `⚠️ Could not match uploaded file with any known filename: ${JSON.stringify(uploadedFile)}`,
+                );
+              }
+            });
+
+            // If no individual files matched but API indicates success, use fallback
+            if (
+              successfulFileIds.length === 0 &&
+              apiResult.uploaded.length > 0
+            ) {
+              logger.warn(
+                `🔄 Fallback: No individual file matches found, but API indicates ${apiResult.uploaded.length} uploads. Marking all ${validFiles.length} batch files as uploaded.`,
+              );
+              validFiles.forEach((f) => successfulFileIds.push(f.dbRecord.id));
+            }
+
+            if (successfulFileIds.length > 0) {
+              await supabase
+                .from('uploader')
+                .update({
+                  status: 'file-uploaded',
+                  message: 'Successfully uploaded to Arela API (batch)',
+                })
+                .in('id', successfulFileIds);
+
+              uploaded += successfulFileIds.length;
+              logger.info(
+                `✅ Batch upload successful: ${successfulFileIds.length} files uploaded`,
+              );
+            }
+          }
+
+          // Handle failed files
+          if (apiResult.errors && apiResult.errors.length > 0) {
+            const failedFileIds = [];
+
+            apiResult.errors.forEach((errorInfo) => {
+              // Try multiple possible property names for filename in errors
+              const possibleFilename =
+                errorInfo.fileName ||
+                errorInfo.filename ||
+                errorInfo.name ||
+                errorInfo.file_name ||
+                errorInfo.originalName ||
+                errorInfo.original_name;
+
+              const dbRecord = fileNameToRecord.get(possibleFilename);
+              if (dbRecord) {
+                failedFileIds.push(dbRecord.id);
+              } else {
+                logger.warn(
+                  `⚠️ Could not match error file: ${JSON.stringify(errorInfo)}`,
+                );
+              }
+            });
+
+            if (failedFileIds.length > 0) {
+              await supabase
+                .from('uploader')
+                .update({
+                  status: 'upload-error',
+                  message: `Upload failed: ${apiResult.errors[0].error}`,
+                })
+                .in('id', failedFileIds);
+
+              errors += failedFileIds.length;
+              logger.error(
+                `❌ Batch upload errors: ${failedFileIds.length} files failed`,
+              );
+            }
+          }
+
+          // Handle any remaining files that weren't in uploaded or errors arrays
+          // Use robust filename extraction for both uploaded and error files
+          const extractFilename = (fileObj) => {
+            return (
+              fileObj.fileName ||
+              fileObj.filename ||
+              fileObj.name ||
+              fileObj.file_name ||
+              fileObj.originalName ||
+              fileObj.original_name
+            );
+          };
+
+          const processedFileNames = new Set([
+            ...(apiResult.uploaded || []).map(extractFilename).filter(Boolean),
+            ...(apiResult.errors || []).map(extractFilename).filter(Boolean),
+          ]);
+
+          const unprocessedFiles = validFiles.filter(
+            (f) => !processedFileNames.has(f.fileData.name),
+          );
+          if (unprocessedFiles.length > 0) {
+            // Only mark as unprocessed if we haven't already handled all files through fallback logic
+            // If we used fallback (all files marked as uploaded), don't mark any as unprocessed
+            const alreadyHandledCount = uploaded + errors;
+            const shouldMarkUnprocessed =
+              alreadyHandledCount < validFiles.length;
+
+            if (shouldMarkUnprocessed) {
+              const unprocessedIds = unprocessedFiles.map((f) => f.dbRecord.id);
+              await supabase
+                .from('uploader')
+                .update({
+                  status: 'upload-error',
+                  message: 'File not found in API response',
+                })
+                .in('id', unprocessedIds);
+
+              errors += unprocessedIds.length;
+              logger.warn(
+                `⚠️ Unprocessed files: ${unprocessedIds.length} files not found in API response`,
+              );
+              logger.debug(
+                `🔍 API response uploaded array: ${JSON.stringify(apiResult.uploaded)}`,
+              );
+              logger.debug(
+                `🔍 Expected filenames: ${validFiles.map((f) => f.fileData.name).join(', ')}`,
+              );
+            } else {
+              logger.debug(
+                `✅ All files already handled (uploaded: ${uploaded}, errors: ${errors}), skipping unprocessed marking`,
+              );
+            }
+          }
+        } else {
+          // Complete batch failure - mark all files as failed
+          const fileIds = validFiles.map((f) => f.dbRecord.id);
+          await supabase
+            .from('uploader')
+            .update({
+              status: 'upload-error',
+              message: uploadResult.error || 'Batch upload failed',
+            })
+            .in('id', fileIds);
+
+          errors += validFiles.length;
+          logger.error(
+            `❌ Complete batch failure: ${validFiles.length} files - ${uploadResult.error}`,
+          );
+        }
+      }
+    } catch (error) {
+      logger.error(`❌ Error processing batch: ${error.message}`);
+
+      // Mark all files as failed
+      const fileIds = files.map((f) => f.id);
+      await supabase
+        .from('uploader')
+        .update({
+          status: 'upload-error',
+          message: `Batch processing error: ${error.message}`,
+        })
+        .in('id', fileIds);
+
+      errors += files.length;
+    }
+
+    return { processed, uploaded, errors };
+  }
 }
 
 // Export singleton instance