@arela/uploader 0.2.6 → 0.2.8

package/.env.template

@@ -1,20 +1,74 @@
-# Test environment configuration for arela-uploader
-# Copy this to .env and update with your actual values
+# Arela Uploader Environment Configuration
+# Copy this to your .env file and adjust values for your setup
 
-# Supabase Configuration
-SUPABASE_URL=https://your-project.supabase.co
-SUPABASE_KEY=your-supabase-anon-key
-SUPABASE_BUCKET=your-bucket-name
+# =============================================================================
+# BASIC CONFIGURATION
+# =============================================================================
 
 # Arela API Configuration
 ARELA_API_URL=https://your-arela-api-url.com
-ARELA_API_TOKEN=your-api-token
+ARELA_API_TOKEN=your-api-token-here
+
+# Supabase Configuration (fallback)
+SUPABASE_URL=https://your-supabase-url.supabase.co
+SUPABASE_KEY=your-supabase-key-here
+SUPABASE_BUCKET=your-bucket-name
 
-# Upload Configuration
-UPLOAD_BASE_PATH=/Users/your-username/documents
+# Upload Sources (separate with |)
+UPLOAD_BASE_PATH=/path/to/your/upload/base
 UPLOAD_SOURCES=folder1|folder2|folder3
+UPLOAD_RFCS=rfc1|rfc2|rfc3
+
+# =============================================================================
+# PERFORMANCE OPTIMIZATION FOR MULTIPLE API REPLICAS
+# =============================================================================
+
+# API Connection Configuration
+# Set this to match your number of API replicas (e.g., if you have 10 API instances, set to 10)
+MAX_API_CONNECTIONS=10
+
+# API Connection Timeout (milliseconds)
+API_CONNECTION_TIMEOUT=60000
+
+# Batch Processing Configuration
+# Files processed concurrently per batch (should be >= MAX_API_CONNECTIONS for best performance)
+BATCH_SIZE=100
+
+# Delay between batches (0 for maximum speed)
+BATCH_DELAY=0
+
+# Source Processing Concurrency
+# Number of upload sources/folders to process simultaneously
+MAX_CONCURRENT_SOURCES=2
+
+# =============================================================================
+# EXAMPLE CONFIGURATIONS FOR DIFFERENT SCENARIOS
+# =============================================================================
+
+# For 10 API Replicas (High Performance Setup):
+# MAX_API_CONNECTIONS=10
+# BATCH_SIZE=100
+# MAX_CONCURRENT_SOURCES=3
+# BATCH_DELAY=0
+
+# For 5 API Replicas (Medium Performance Setup):
+# MAX_API_CONNECTIONS=5
+# BATCH_SIZE=50
+# MAX_CONCURRENT_SOURCES=2
+# BATCH_DELAY=0
+
+# For 1 API Instance (Single Instance Setup):
+# MAX_API_CONNECTIONS=5
+# BATCH_SIZE=20
+# MAX_CONCURRENT_SOURCES=1
+# BATCH_DELAY=100
+
+# =============================================================================
+# LOGGING AND MONITORING
+# =============================================================================
+
+# Progress bar update frequency
+PROGRESS_UPDATE_INTERVAL=10
 
-# RFC Upload Configuration
-# Pipe-separated list of RFCs to upload files for
-# Example: MMJ0810145N1|ABC1234567XY|DEF9876543ZZ
-UPLOAD_RFCS=RFC1|RFC2|RFC3
+# Enable verbose logging (true/false)
+VERBOSE_LOGGING=false

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arela/uploader",
-  "version": "0.2.6",
+  "version": "0.2.8",
   "description": "CLI to upload files/directories to Arela",
   "bin": {
     "arela": "./src/index.js"

package/src/commands/UploadCommand.js

@@ -50,7 +50,7 @@ export class UploadCommand {
         logger.info('Log file cleared');
       }
 
-      // Process each source
+      // Process each source with configurable concurrency
       let globalResults = {
         successCount: 0,
         detectedCount: 0,
@@ -59,28 +59,88 @@ export class UploadCommand {
         skippedCount: 0,
       };
 
-      for (const source of sources) {
-        const sourcePath = path.resolve(basePath, source).replace(/\\/g, '/');
-        logger.info(`Processing folder: ${sourcePath}`);
-
-        try {
-          const files = await this.#discoverFiles(sourcePath);
-          logger.info(`Found ${files.length} files to process`);
-
-          const result = await this.#processFilesInBatches(
-            files,
-            options,
-            uploadService,
-            basePath,
-            source,
-            sourcePath,
-          );
+      // Determine processing strategy based on configuration
+      const maxConcurrentSources =
+        appConfig.performance?.maxConcurrentSources || 1;
+
+      if (maxConcurrentSources > 1 && sources.length > 1) {
+        // Parallel source processing
+        logger.info(
+          `Processing ${sources.length} sources with concurrency: ${maxConcurrentSources}`,
+        );
 
-          this.#updateGlobalResults(globalResults, result);
-          this.#logSourceSummary(source, result, options);
-        } catch (error) {
-          this.errorHandler.handleError(error, { source, sourcePath });
-          globalResults.failureCount++;
+        // Process sources in batches to control concurrency
+        for (let i = 0; i < sources.length; i += maxConcurrentSources) {
+          const sourceBatch = sources.slice(i, i + maxConcurrentSources);
+
+          const sourcePromises = sourceBatch.map(async (source) => {
+            const sourcePath = path
+              .resolve(basePath, source)
+              .replace(/\\/g, '/');
+            logger.info(`Processing folder: ${sourcePath}`);
+
+            try {
+              const files = await this.#discoverFiles(sourcePath);
+              logger.info(`Found ${files.length} files in ${source}`);
+
+              const result = await this.#processFilesInBatches(
+                files,
+                options,
+                uploadService,
+                basePath,
+                source,
+                sourcePath,
+              );
+
+              this.#logSourceSummary(source, result, options);
+              return { success: true, source, result };
+            } catch (error) {
+              this.errorHandler.handleError(error, { source, sourcePath });
+              return { success: false, source, error: error.message };
+            }
+          });
+
+          // Wait for this batch of sources to complete
+          const results = await Promise.allSettled(sourcePromises);
+
+          results.forEach((result) => {
+            if (result.status === 'fulfilled') {
+              const sourceResult = result.value;
+              if (sourceResult.success) {
+                this.#updateGlobalResults(globalResults, sourceResult.result);
+              } else {
+                globalResults.failureCount++;
+              }
+            } else {
+              globalResults.failureCount++;
+            }
+          });
+        }
+      } else {
+        // Sequential source processing (original behavior)
+        for (const source of sources) {
+          const sourcePath = path.resolve(basePath, source).replace(/\\/g, '/');
+          logger.info(`Processing folder: ${sourcePath}`);
+
+          try {
+            const files = await this.#discoverFiles(sourcePath);
+            logger.info(`Found ${files.length} files to process`);
+
+            const result = await this.#processFilesInBatches(
+              files,
+              options,
+              uploadService,
+              basePath,
+              source,
+              sourcePath,
+            );
+
+            this.#updateGlobalResults(globalResults, result);
+            this.#logSourceSummary(source, result, options);
+          } catch (error) {
+            this.errorHandler.handleError(error, { source, sourcePath });
+            globalResults.failureCount++;
+          }
         }
       }
 
@@ -164,7 +224,8 @@ export class UploadCommand {
     source,
     sourcePath,
   ) {
-    const batchSize = parseInt(options.batchSize) || 10;
+    const batchSize =
+      parseInt(options.batchSize) || appConfig.performance.batchSize || 50;
     const results = {
       successCount: 0,
       detectedCount: 0,
@@ -184,6 +245,9 @@ export class UploadCommand {
       barCompleteChar: '█',
       barIncompleteChar: '░',
       hideCursor: true,
+      clearOnComplete: false,
+      stopOnComplete: true,
+      stream: process.stderr, // Use stderr to separate from stdout logging
     });
 
     progressBar.start(files.length, 0, { success: 0, errors: 0 });
@@ -265,22 +329,65 @@ export class UploadCommand {
         throw new Error(`Failed to insert stats: ${error.message}`);
       }
     } else {
-      // Upload mode: process files for upload
-      for (const filePath of batch) {
-        try {
-          await this.#processFile(
-            filePath,
-            options,
-            uploadService,
-            basePath,
-            processedPaths,
-            batchResults,
-          );
-        } catch (error) {
-          this.errorHandler.handleError(error, { filePath });
-          batchResults.failureCount++;
+      // Upload mode: process files with controlled concurrency to match API replicas
+      const maxConcurrentApiCalls =
+        appConfig.performance?.maxApiConnections || 10;
+
+      // Process batch in chunks to respect API replica limits
+      const allResults = [];
+      for (let i = 0; i < batch.length; i += maxConcurrentApiCalls) {
+        const chunk = batch.slice(i, i + maxConcurrentApiCalls);
+
+        // Process this chunk concurrently (up to API replica count)
+        const chunkPromises = chunk.map(async (filePath) => {
+          try {
+            const result = await this.#processFile(
+              filePath,
+              options,
+              uploadService,
+              basePath,
+              processedPaths,
+            );
+            return { success: true, filePath, result };
+          } catch (error) {
+            this.errorHandler.handleError(error, { filePath });
+            return { success: false, filePath, error: error.message };
+          }
+        });
+
+        // Wait for this chunk to complete before starting the next
+        const chunkResults = await Promise.allSettled(chunkPromises);
+        allResults.push(...chunkResults);
+
+        // Small delay between chunks to prevent overwhelming API
+        if (i + maxConcurrentApiCalls < batch.length) {
+          await new Promise((resolve) => setTimeout(resolve, 50));
         }
       }
+
+      // Process all results and update batch results
+      allResults.forEach((result) => {
+        if (result.status === 'fulfilled') {
+          const fileResult = result.value;
+          if (fileResult.success) {
+            if (fileResult.result && fileResult.result.skipped) {
+              batchResults.skippedCount++;
+            } else {
+              batchResults.successCount++;
+              if (fileResult.result && fileResult.result.detectedCount) {
+                batchResults.detectedCount += fileResult.result.detectedCount;
+              }
+              if (fileResult.result && fileResult.result.organizedCount) {
+                batchResults.organizedCount += fileResult.result.organizedCount;
+              }
+            }
+          } else {
+            batchResults.failureCount++;
+          }
+        } else {
+          batchResults.failureCount++;
+        }
+      });
     }
 
     return batchResults;
@@ -296,12 +403,10 @@ export class UploadCommand {
     uploadService,
     basePath,
     processedPaths,
-    batchResults,
   ) {
     // Skip if already processed
     if (processedPaths.has(filePath)) {
-      batchResults.skippedCount++;
-      return;
+      return { skipped: true };
     }
 
     // Prepare file for upload
@@ -325,24 +430,25 @@ export class UploadCommand {
     };
 
     // Upload based on service type
+    let result = { successCount: 1 };
+
     if (uploadService.getServiceName() === 'Arela API') {
-      const result = await uploadService.upload([fileObject], {
+      result = await uploadService.upload([fileObject], {
         ...options,
         uploadPath,
       });
-
-      batchResults.successCount++;
-      if (result.detectedCount)
-        batchResults.detectedCount += result.detectedCount;
-      if (result.organizedCount)
-        batchResults.organizedCount += result.organizedCount;
     } else {
       // Supabase direct upload
       await uploadService.upload([fileObject], { uploadPath });
-      batchResults.successCount++;
     }
 
     logger.info(`SUCCESS: ${path.basename(filePath)} -> ${uploadPath}`);
+
+    return {
+      skipped: false,
+      detectedCount: result.detectedCount || 0,
+      organizedCount: result.organizedCount || 0,
+    };
   }
 
   /**
@@ -428,7 +534,8 @@ export class UploadCommand {
       // Phase 2: PDF Detection
       console.log('\n🔍 === PHASE 2: PDF Detection ===');
       const detectionResult = await databaseService.detectPedimentosInDatabase({
-        batchSize: parseInt(options.batchSize) || 10,
+        batchSize:
+          parseInt(options.batchSize) || appConfig.performance.batchSize || 50,
       });
       console.log(
         `✅ Phase 2 Complete: ${detectionResult.detectedCount} detected, ${detectionResult.errorCount} errors`,

package/src/config/config.js

@@ -28,10 +28,10 @@ class Config {
       const __dirname = path.dirname(__filename);
       const packageJsonPath = path.resolve(__dirname, '../../package.json');
       const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
-      return packageJson.version || '0.2.6';
+      return packageJson.version || '0.2.8';
     } catch (error) {
       console.warn('⚠️ Could not read package.json version, using fallback');
-      return '0.2.6';
+      return '0.2.8';
     }
   }
 
@@ -85,7 +85,12 @@ class Config {
    */
   #loadPerformanceConfig() {
     return {
-      batchDelay: parseInt(process.env.BATCH_DELAY) || 100,
+      batchDelay: parseInt(process.env.BATCH_DELAY) || 0, // Removed default delay
+      batchSize: parseInt(process.env.BATCH_SIZE) || 50, // Increased from 10 to 50
+      maxConcurrentSources: parseInt(process.env.MAX_CONCURRENT_SOURCES) || 2,
+      maxApiConnections: parseInt(process.env.MAX_API_CONNECTIONS) || 10, // New: API replica support
+      apiConnectionTimeout:
+        parseInt(process.env.API_CONNECTION_TIMEOUT) || 60000, // New: API timeout
       progressUpdateInterval:
         parseInt(process.env.PROGRESS_UPDATE_INTERVAL) || 10,
       logBufferSize: 100,

package/src/services/DatabaseService.js

@@ -104,16 +104,28 @@ export class DatabaseService {
       }
 
       // Initialize record with basic file stats
+      const fileExtension = path
+        .extname(file.path)
+        .toLowerCase()
+        .replace('.', '');
+      const filename = file.originalName || path.basename(file.path);
+
       const record = {
         document_type: null,
         size: stats.size,
         num_pedimento: null,
-        filename: file.originalName || path.basename(file.path),
+        filename: filename,
         original_path: originalPath,
         arela_path: null,
         status: 'stats',
         rfc: null,
         message: null,
+        file_extension: fileExtension,
+        is_like_simplificado:
+          fileExtension === 'pdf' && filename.toLowerCase().includes('simp'),
+        year: null,
+        created_at: new Date().toISOString(),
+        modified_at: stats.mtime.toISOString(),
       };
 
       // Try to detect document type for supported files
@@ -130,6 +142,10 @@ export class DatabaseService {
               record.arela_path = detection.arelaPath;
             }
 
+            if (detection.detectedPedimentoYear) {
+              record.year = detection.detectedPedimentoYear;
+            }
+
             const rfcField = detection.fields.find(
               (f) => f.name === 'rfc' && f.found,
             );
@@ -210,6 +226,12 @@ export class DatabaseService {
           file_extension: fileExtension,
           created_at: new Date().toISOString(),
           modified_at: stats.mtime.toISOString(),
+          is_like_simplificado:
+            fileExtension === 'pdf' &&
+            (file.originalName || path.basename(file.path))
+              .toLowerCase()
+              .includes('simp'),
+          year: null,
         };
 
         allRecords.push(record);
@@ -258,9 +280,15 @@ export class DatabaseService {
           existingPaths.has(r.original_path),
         );
 
-        logger.info(
-          `Batch ${Math.floor(i / batchSize) + 1}: ${newRecords.length} new, ${updateRecords.length} updates`,
-        );
+        // Only log every 10th batch to reduce noise
+        if (
+          (Math.floor(i / batchSize) + 1) % 10 === 0 ||
+          Math.floor(i / batchSize) + 1 === 1
+        ) {
+          logger.info(
+            `Batch ${Math.floor(i / batchSize) + 1}: ${newRecords.length} new, ${updateRecords.length} updates`,
+          );
+        }
 
         // Insert new records
         if (newRecords.length > 0) {
@@ -272,7 +300,7 @@ export class DatabaseService {
             logger.error(`Error inserting new records: ${insertError.message}`);
           } else {
             totalInserted += newRecords.length;
-            logger.success(`Inserted ${newRecords.length} new records`);
+            // Only log the batch insertion, not the summary (which comes at the end)
           }
         }
 
@@ -287,6 +315,8 @@ export class DatabaseService {
                 modified_at: record.modified_at,
                 filename: record.filename,
                 file_extension: record.file_extension,
+                is_like_simplificado: record.is_like_simplificado,
+                year: record.year,
               })
               .eq('original_path', record.original_path);
 
@@ -295,7 +325,10 @@ export class DatabaseService {
             }
           }
           totalUpdated += batchUpdated;
-          logger.info(`Updated ${batchUpdated} existing records`);
+          // Reduce logging noise - only log when there are updates
+          if (batchUpdated > 0) {
+            logger.info(`Updated ${batchUpdated} existing records`);
+          }
         }
       } catch (error) {
         logger.error(
@@ -329,7 +362,7 @@ export class DatabaseService {
 
     const processingBatchSize = parseInt(options.batchSize) || 10;
     // Reduced query batch size to avoid timeouts
-    const queryBatchSize = 500; // Reduced from 1000 to 500
+    const queryBatchSize = 100; // Reduced from 500 to 100
 
     let totalDetected = 0;
     let totalProcessed = 0;
@@ -355,9 +388,9 @@ export class DatabaseService {
               .select('id, original_path, filename, file_extension, status')
               .eq('status', 'fs-stats')
               .eq('file_extension', 'pdf')
-              .ilike('filename', '%simp%')
+              .eq('is_like_simplificado', true)
               .range(offset, offset + queryBatchSize - 1)
-              .order('id'); // Add explicit ordering for consistent pagination
+              .order('created_at');
           }, `fetch PDF records chunk ${chunkNumber}`);
 
         if (queryError) {
@@ -416,6 +449,7 @@ export class DatabaseService {
                 num_pedimento: detection.detectedPedimento,
                 arela_path: detection.arelaPath,
                 message: detection.error || null,
+                year: detection.detectedPedimentoYear || null,
               };
 
               if (detection.fields) {
@@ -522,13 +556,15 @@ export class DatabaseService {
   async propagateArelaPath(options = {}) {
     const supabase = await this.#getSupabaseClient();
 
-    logger.info('Phase 3: Starting arela_path propagation process...');
-    console.log('🔍 Finding pedimento_simplificado records with arela_path...');
+    logger.info('Phase 3: Starting arela_path and year propagation process...');
+    console.log(
+      '🔍 Finding pedimento_simplificado records with arela_path and year...',
+    );
 
     // Get all pedimento_simplificado records that have arela_path
     const { data: pedimentoRecords, error: pedimentoError } = await supabase
       .from('uploader')
-      .select('id, original_path, arela_path, filename')
+      .select('id, original_path, arela_path, filename, year')
       .eq('document_type', 'pedimento_simplificado')
       .not('arela_path', 'is', null);
 
@@ -567,7 +603,7 @@ export class DatabaseService {
         const basePath = path.dirname(pedimento.original_path);
 
         logger.info(
-          `Processing pedimento: ${pedimento.filename} | Base path: ${basePath}`,
+          `Processing pedimento: ${pedimento.filename} | Base path: ${basePath} | Year: ${pedimento.year || 'N/A'}`,
         );
 
         // Extract folder part from existing arela_path
@@ -618,7 +654,10 @@ export class DatabaseService {
           try {
             const { error: updateError } = await supabase
               .from('uploader')
-              .update({ arela_path: folderArelaPath })
+              .update({
+                arela_path: folderArelaPath,
+                year: pedimento.year,
+              })
               .in('id', batchIds);
 
             if (updateError) {
@@ -629,7 +668,7 @@ export class DatabaseService {
             } else {
               totalUpdated += batchIds.length;
               logger.info(
-                `Successfully updated batch ${batchNumber}: ${batchIds.length} files`,
+                `Successfully updated batch ${batchNumber}: ${batchIds.length} files with arela_path and year`,
               );
             }
           } catch (batchError) {
@@ -654,7 +693,7 @@ export class DatabaseService {
     };
 
     logger.success(
-      `Phase 3 Summary: ${totalProcessed} pedimentos processed, ${totalUpdated} files updated, ${totalErrors} errors`,
+      `Phase 3 Summary: ${totalProcessed} pedimentos processed, ${totalUpdated} files updated with arela_path and year, ${totalErrors} errors`,
     );
 
     return result;
@@ -860,133 +899,54 @@ export class DatabaseService {
     console.log(`📋 Total files to upload: ${allRelatedFiles.length}`);
     logger.info(`Total files to upload: ${allRelatedFiles.length}`);
 
-    // Step 4: Upload all related files
+    // Step 4: Upload all related files using concurrent batch processing
     let totalProcessed = 0;
     let totalUploaded = 0;
     let totalErrors = 0;
     const batchSize = parseInt(options.batchSize) || 10;
 
-    for (let i = 0; i < allRelatedFiles.length; i += batchSize) {
-      const batch = allRelatedFiles.slice(i, i + batchSize);
-
-      for (const file of batch) {
-        try {
-          totalProcessed++;
-
-          // Check if file exists
-          if (!fs.existsSync(file.original_path)) {
-            logger.warn(
-              `File not found: ${file.filename} at ${file.original_path}`,
-            );
-            await supabase
-              .from('uploader')
-              .update({
-                status: 'file-not-found',
-                message: 'File no longer exists at original path',
-              })
-              .eq('id', file.id);
-            totalErrors++;
-            continue;
-          }
-
-          // Upload the file (handle both API and Supabase services)
-          let uploadResult;
-          if (uploadService.getServiceName() === 'Supabase') {
-            // Supabase requires single file upload with uploadPath
-            let uploadPath;
-            if (options.folderStructure && file.arela_path) {
-              // Combine folder structure with arela_path: palco/RFC/Year/Patente/Aduana/Pedimento/filename
-              uploadPath = `uploads/${options.folderStructure}/${file.arela_path}${file.filename}`;
-            } else if (file.arela_path) {
-              // Use existing arela_path: RFC/Year/Patente/Aduana/Pedimento/filename
-              uploadPath = `uploads/${file.arela_path}${file.filename}`;
-            } else {
-              // Fallback to RFC folder
-              uploadPath = `uploads/${file.rfc}/${file.filename}`;
-            }
+    // Import performance configuration
+    const { performance: perfConfig } = appConfig;
+    const maxConcurrency = perfConfig?.maxApiConnections || 3;
 
-            uploadResult = await uploadService.upload(
-              [
-                {
-                  path: file.original_path,
-                  name: file.filename,
-                  contentType: 'application/octet-stream',
-                },
-              ],
-              {
-                uploadPath: uploadPath,
-              },
-            );
-            uploadResult = { success: true, data: uploadResult };
-          } else {
-            // API service supports batch uploads and returns normalized response
-            let fullFolderStructure;
-            if (options.folderStructure && file.arela_path) {
-              // Combine folder structure with arela_path: palco/RFC/Year/Patente/Aduana/Pedimento/
-              fullFolderStructure = `${options.folderStructure}/${file.arela_path}`;
-            } else if (file.arela_path) {
-              // Use existing arela_path: RFC/Year/Patente/Aduana/Pedimento/
-              fullFolderStructure = file.arela_path;
-            } else {
-              // Fallback to RFC folder
-              fullFolderStructure = `${file.rfc}/`;
-            }
+    console.log(
+      `🚀 Starting batch upload: ${allRelatedFiles.length} files in batches of ${batchSize}`,
+    );
+    console.log(
+      `⚡ Concurrent processing: up to ${maxConcurrency} parallel operations`,
+    );
 
-            uploadResult = await uploadService.upload(
-              [
-                {
-                  path: file.original_path,
-                  name: file.filename,
-                  contentType: 'application/octet-stream',
-                },
-              ],
-              {
-                folderStructure: fullFolderStructure,
-              },
-            );
-          }
+    // Process files in batches with concurrent processing
+    for (let i = 0; i < allRelatedFiles.length; i += batchSize) {
+      const batch = allRelatedFiles.slice(i, i + batchSize);
+      const batchNum = Math.floor(i / batchSize) + 1;
+      const totalBatches = Math.ceil(allRelatedFiles.length / batchSize);
 
-          if (uploadResult.success) {
-            // Update database status
-            await supabase
-              .from('uploader')
-              .update({
-                status: 'file-uploaded',
-                message: 'Successfully uploaded to Arela API',
-              })
-              .eq('id', file.id);
+      console.log(
+        `📦 Processing batch ${batchNum}/${totalBatches} (${batch.length} files)`,
+      );
 
-            totalUploaded++;
-            logger.info(`Uploaded: ${file.filename}`);
-          } else {
-            await supabase
-              .from('uploader')
-              .update({
-                status: 'upload-error',
-                message: uploadResult.error || 'Upload failed',
-              })
-              .eq('id', file.id);
+      // Process batch using concurrent processing similar to UploadCommand
+      const batchResults = await this.#processRfcBatch(
+        batch,
+        uploadService,
+        supabase,
+        options,
+        maxConcurrency,
+      );
 
-            totalErrors++;
-            logger.error(
-              `Upload failed: ${file.filename} - ${uploadResult.error}`,
-            );
-          }
-        } catch (error) {
-          totalErrors++;
-          logger.error(
-            `Error processing file ${file.filename}: ${error.message}`,
-          );
+      totalProcessed += batchResults.processed;
+      totalUploaded += batchResults.uploaded;
+      totalErrors += batchResults.errors;
 
-          await supabase
-            .from('uploader')
-            .update({
-              status: 'upload-error',
-              message: `Processing error: ${error.message}`,
-            })
-            .eq('id', file.id);
-        }
-      }
+      // Progress update
+      const progress = (
+        ((i + batch.length) / allRelatedFiles.length) *
+        100
+      ).toFixed(1);
+      console.log(
+        `📊 Batch ${batchNum} complete - Progress: ${progress}% (${totalUploaded}/${allRelatedFiles.length} uploaded)`,
+      );
     }
 
     const result = {
@@ -1133,6 +1093,298 @@ export class DatabaseService {
 
     return readyFiles || [];
   }
+
+  /**
+   * Process a batch of files using concurrent processing for RFC uploads
+   * @param {Array} files - Files to process in this batch
+   * @param {Object} uploadService - Upload service instance
+   * @param {Object} supabase - Supabase client
+   * @param {Object} options - Upload options
+   * @param {number} maxConcurrency - Maximum concurrent operations
+   * @returns {Promise<Object>} Batch processing results
+   */
+  async #processRfcBatch(
+    files,
+    uploadService,
+    supabase,
+    options,
+    maxConcurrency,
+  ) {
+    const fs = (await import('fs')).default;
+
+    let processed = 0;
+    let uploaded = 0;
+    let errors = 0;
+
+    // For Supabase, process files individually (required by service)
+    if (uploadService.getServiceName() === 'Supabase') {
+      // Process files in concurrent chunks within the batch
+      const chunks = [];
+      for (let i = 0; i < files.length; i += maxConcurrency) {
+        chunks.push(files.slice(i, i + maxConcurrency));
+      }
+
+      // Process each chunk concurrently
+      for (const chunk of chunks) {
+        const chunkPromises = chunk.map(async (file) => {
+          return await this.#processRfcSingleFile(
+            file,
+            uploadService,
+            supabase,
+            options,
+            fs,
+          );
+        });
+
+        // Wait for all files in this chunk to complete
+        const chunkResults = await Promise.allSettled(chunkPromises);
+
+        // Count results
+        for (const result of chunkResults) {
+          processed++;
+          if (result.status === 'fulfilled' && result.value.success) {
+            uploaded++;
+          } else {
+            errors++;
+          }
+        }
+      }
+    } else {
+      // For API service, use true batch processing (multiple files per API call)
+      const apiChunks = [];
+      const apiChunkSize = Math.min(
+        5,
+        Math.ceil(files.length / maxConcurrency),
+      ); // 5 files per API call, or distribute evenly
+
+      for (let i = 0; i < files.length; i += apiChunkSize) {
+        apiChunks.push(files.slice(i, i + apiChunkSize));
+      }
+
+      console.log(
+        `   🚀 Processing ${apiChunks.length} API calls with ${apiChunkSize} files each (max ${maxConcurrency} concurrent)`,
+      );
+
+      // Process API chunks with controlled concurrency
+      const concurrentChunks = [];
+      for (let i = 0; i < apiChunks.length; i += maxConcurrency) {
+        concurrentChunks.push(apiChunks.slice(i, i + maxConcurrency));
+      }
+
+      for (const concurrentSet of concurrentChunks) {
+        const batchPromises = concurrentSet.map(async (chunk) => {
+          return await this.#processRfcApiBatch(
+            chunk,
+            uploadService,
+            supabase,
+            options,
+            fs,
+          );
+        });
+
+        // Wait for all concurrent batches to complete
+        const batchResults = await Promise.allSettled(batchPromises);
+
+        // Count results
+        for (const result of batchResults) {
+          if (result.status === 'fulfilled') {
+            processed += result.value.processed;
+            uploaded += result.value.uploaded;
+            errors += result.value.errors;
+          } else {
+            errors += result.value?.processed || 0;
+          }
+        }
+      }
+    }
+
+    return { processed, uploaded, errors };
+  }
+
+  /**
+   * Process a single file for RFC upload (Supabase mode)
+   */
+  async #processRfcSingleFile(file, uploadService, supabase, options, fs) {
+    try {
+      // Check if file exists
+      if (!fs.existsSync(file.original_path)) {
+        logger.warn(
+          `File not found: ${file.filename} at ${file.original_path}`,
+        );
+        await supabase
+          .from('uploader')
+          .update({
+            status: 'file-not-found',
+            message: 'File no longer exists at original path',
+          })
+          .eq('id', file.id);
+        return { success: false, error: 'File not found' };
+      }
+
+      // Supabase requires single file upload with uploadPath
+      let uploadPath;
+      if (options.folderStructure && file.arela_path) {
+        uploadPath = `uploads/${options.folderStructure}/${file.arela_path}${file.filename}`;
+      } else if (file.arela_path) {
+        uploadPath = `uploads/${file.arela_path}${file.filename}`;
+      } else {
+        uploadPath = `uploads/${file.rfc}/${file.filename}`;
+      }
+
+      const uploadResult = await uploadService.upload(
+        [
+          {
+            path: file.original_path,
+            name: file.filename,
+            contentType: 'application/octet-stream',
+          },
+        ],
+        { uploadPath: uploadPath },
+      );
+
+      // Update database status
+      await supabase
+        .from('uploader')
+        .update({
+          status: 'file-uploaded',
+          message: 'Successfully uploaded to Supabase',
+        })
+        .eq('id', file.id);
+
+      logger.info(`✅ Uploaded: ${file.filename}`);
+      return { success: true, filename: file.filename };
+    } catch (error) {
+      logger.error(
+        `❌ Error processing file ${file.filename}: ${error.message}`,
+      );
+
+      await supabase
+        .from('uploader')
+        .update({
+          status: 'upload-error',
+          message: `Processing error: ${error.message}`,
+        })
+        .eq('id', file.id);
+
+      return { success: false, error: error.message, filename: file.filename };
+    }
+  }
+
+  /**
+   * Process multiple files in a single API batch call (API service mode)
+   */
+  async #processRfcApiBatch(files, uploadService, supabase, options, fs) {
+    let processed = 0;
+    let uploaded = 0;
+    let errors = 0;
+
+    try {
+      // Prepare files for batch upload
+      const validFiles = [];
+      const invalidFiles = [];
+
+      for (const file of files) {
+        processed++;
+
+        if (!fs.existsSync(file.original_path)) {
+          logger.warn(
+            `File not found: ${file.filename} at ${file.original_path}`,
+          );
+          invalidFiles.push(file);
+          continue;
+        }
+
+        validFiles.push({
+          fileData: {
+            path: file.original_path,
+            name: file.filename,
+            contentType: 'application/octet-stream',
+          },
+          dbRecord: file,
+        });
+      }
+
+      // Update invalid files in database
+      for (const file of invalidFiles) {
+        await supabase
+          .from('uploader')
+          .update({
+            status: 'file-not-found',
+            message: 'File no longer exists at original path',
+          })
+          .eq('id', file.id);
+        errors++;
+      }
+
+      // Process valid files in batch if any exist
+      if (validFiles.length > 0) {
+        // Determine folder structure (all files in this batch should have same arela_path)
+        const sampleFile = validFiles[0].dbRecord;
+        let fullFolderStructure;
+        if (options.folderStructure && sampleFile.arela_path) {
+          fullFolderStructure = `${options.folderStructure}/${sampleFile.arela_path}`;
+        } else if (sampleFile.arela_path) {
+          fullFolderStructure = sampleFile.arela_path;
+        } else {
+          fullFolderStructure = `${sampleFile.rfc}/`;
+        }
+
+        // Make single API call with multiple files
+        const uploadResult = await uploadService.upload(
+          validFiles.map((f) => f.fileData),
+          { folderStructure: fullFolderStructure },
+        );
+
+        if (uploadResult.success) {
+          // Update all files as uploaded
+          const fileIds = validFiles.map((f) => f.dbRecord.id);
+          await supabase
+            .from('uploader')
+            .update({
+              status: 'file-uploaded',
+              message: 'Successfully uploaded to Arela API (batch)',
+            })
+            .in('id', fileIds);
+
+          uploaded += validFiles.length;
+          logger.info(
+            `✅ Batch uploaded: ${validFiles.length} files to ${fullFolderStructure}`,
+          );
+        } else {
+          // Update all files as failed
+          const fileIds = validFiles.map((f) => f.dbRecord.id);
+          await supabase
+            .from('uploader')
+            .update({
+              status: 'upload-error',
+              message: uploadResult.error || 'Batch upload failed',
+            })
+            .in('id', fileIds);
+
+          errors += validFiles.length;
+          logger.error(
+            `❌ Batch upload failed: ${validFiles.length} files - ${uploadResult.error}`,
+          );
+        }
+      }
+    } catch (error) {
+      logger.error(`❌ Error processing batch: ${error.message}`);
+
+      // Mark all files as failed
+      const fileIds = files.map((f) => f.id);
+      await supabase
+        .from('uploader')
+        .update({
+          status: 'upload-error',
+          message: `Batch processing error: ${error.message}`,
+        })
+        .in('id', fileIds);
+
+      errors += files.length;
+    }
+
+    return { processed, uploaded, errors };
+  }
 }
 
 // Export singleton instance

package/src/services/upload/ApiUploadService.js

@@ -1,6 +1,8 @@
 import { Blob } from 'buffer';
 import { FormData } from 'formdata-node';
 import fs from 'fs';
+import { Agent } from 'http';
+import { Agent as HttpsAgent } from 'https';
 import fetch from 'node-fetch';
 import path from 'path';
 
@@ -16,6 +18,36 @@ export class ApiUploadService extends BaseUploadService {
     super();
     this.baseUrl = appConfig.api.baseUrl;
     this.token = appConfig.api.token;
+
+    // Get API connection settings from config/environment
+    const maxApiConnections = parseInt(process.env.MAX_API_CONNECTIONS) || 10;
+    const connectionTimeout =
+      parseInt(process.env.API_CONNECTION_TIMEOUT) || 60000;
+
+    // Initialize HTTP agents optimized for multiple API replicas
+    this.httpAgent = new Agent({
+      keepAlive: true,
+      keepAliveMsecs: 30000,
+      maxSockets: maxApiConnections, // Match your API replica count
+      maxFreeSockets: Math.ceil(maxApiConnections / 2),
+      maxTotalSockets: maxApiConnections + 5, // Buffer for peak usage
+      timeout: connectionTimeout,
+      scheduling: 'fifo', // First-in-first-out scheduling
+    });
+
+    this.httpsAgent = new HttpsAgent({
+      keepAlive: true,
+      keepAliveMsecs: 30000,
+      maxSockets: maxApiConnections, // Match your API replica count
+      maxFreeSockets: Math.ceil(maxApiConnections / 2),
+      maxTotalSockets: maxApiConnections + 5, // Buffer for peak usage
+      timeout: connectionTimeout,
+      scheduling: 'fifo', // First-in-first-out scheduling
+    });
+
+    console.log(
+      `🔗 HTTP Agent configured for ${maxApiConnections} concurrent API connections`,
+    );
   }
 
   /**
@@ -27,12 +59,31 @@ export class ApiUploadService extends BaseUploadService {
   async upload(files, options) {
     const formData = new FormData();
 
-    // Add files to form data
-    files.forEach((file) => {
-      const fileBuffer = fs.readFileSync(file.path);
-      const blob = new Blob([fileBuffer], { type: file.contentType });
-      formData.append('files', blob, file.name);
-    });
+    // Add files to form data asynchronously
+    for (const file of files) {
+      try {
+        // Check file size for streaming vs buffer approach
+        const stats = await fs.promises.stat(file.path);
+        const fileSizeThreshold = 10 * 1024 * 1024; // 10MB threshold
+
+        if (stats.size > fileSizeThreshold) {
+          // Use streaming for large files
+          const fileStream = fs.createReadStream(file.path);
+          formData.append('files', fileStream, {
+            filename: file.name,
+            contentType: file.contentType,
+            knownLength: stats.size,
+          });
+        } else {
+          // Use buffer for smaller files
+          const fileBuffer = await fs.promises.readFile(file.path);
+          const blob = new Blob([fileBuffer], { type: file.contentType });
+          formData.append('files', blob, file.name);
+        }
+      } catch (error) {
+        throw new Error(`Failed to read file ${file.path}: ${error.message}`);
+      }
+    }
 
     // Add configuration parameters
     if (appConfig.supabase.bucket) {
@@ -61,6 +112,7 @@ export class ApiUploadService extends BaseUploadService {
     formData.append('clientVersion', appConfig.packageVersion);
 
     try {
+      const isHttps = this.baseUrl.startsWith('https');
       const response = await fetch(
         `${this.baseUrl}/api/storage/batch-upload-and-process`,
         {
@@ -69,6 +121,7 @@ export class ApiUploadService extends BaseUploadService {
             'x-api-key': this.token,
           },
           body: formData,
+          agent: isHttps ? this.httpsAgent : this.httpAgent,
         },
       );
 
@@ -99,10 +152,12 @@ export class ApiUploadService extends BaseUploadService {
     }
 
     try {
+      const isHttps = this.baseUrl.startsWith('https');
       const response = await fetch(`${this.baseUrl}/api/health`, {
         headers: {
           'x-api-key': this.token,
         },
+        agent: isHttps ? this.httpsAgent : this.httpAgent,
       });
 
       return response.ok;