@arela/uploader 0.2.7 → 0.2.9

package/.env.template

@@ -1,20 +1,74 @@
-# Test environment configuration for arela-uploader
-# Copy this to .env and update with your actual values
+# Arela Uploader Environment Configuration
+# Copy this to your .env file and adjust values for your setup
 
-# Supabase Configuration
-SUPABASE_URL=https://your-project.supabase.co
-SUPABASE_KEY=your-supabase-anon-key
-SUPABASE_BUCKET=your-bucket-name
+# =============================================================================
+# BASIC CONFIGURATION
+# =============================================================================
 
 # Arela API Configuration
 ARELA_API_URL=https://your-arela-api-url.com
-ARELA_API_TOKEN=your-api-token
+ARELA_API_TOKEN=your-api-token-here
+
+# Supabase Configuration (fallback)
+SUPABASE_URL=https://your-supabase-url.supabase.co
+SUPABASE_KEY=your-supabase-key-here
+SUPABASE_BUCKET=your-bucket-name
 
-# Upload Configuration
-UPLOAD_BASE_PATH=/Users/your-username/documents
+# Upload Sources (separate with |)
+UPLOAD_BASE_PATH=/path/to/your/upload/base
 UPLOAD_SOURCES=folder1|folder2|folder3
+UPLOAD_RFCS=rfc1|rfc2|rfc3
+
+# =============================================================================
+# PERFORMANCE OPTIMIZATION FOR MULTIPLE API REPLICAS
+# =============================================================================
+
+# API Connection Configuration
+# Set this to match your number of API replicas (e.g., if you have 10 API instances, set to 10)
+MAX_API_CONNECTIONS=10
+
+# API Connection Timeout (milliseconds)
+API_CONNECTION_TIMEOUT=60000
+
+# Batch Processing Configuration
+# Files processed concurrently per batch (should be >= MAX_API_CONNECTIONS for best performance)
+BATCH_SIZE=100
+
+# Delay between batches (0 for maximum speed)
+BATCH_DELAY=0
+
+# Source Processing Concurrency
+# Number of upload sources/folders to process simultaneously
+MAX_CONCURRENT_SOURCES=2
+
+# =============================================================================
+# EXAMPLE CONFIGURATIONS FOR DIFFERENT SCENARIOS
+# =============================================================================
+
+# For 10 API Replicas (High Performance Setup):
+# MAX_API_CONNECTIONS=10
+# BATCH_SIZE=100
+# MAX_CONCURRENT_SOURCES=3
+# BATCH_DELAY=0
+
+# For 5 API Replicas (Medium Performance Setup):
+# MAX_API_CONNECTIONS=5
+# BATCH_SIZE=50
+# MAX_CONCURRENT_SOURCES=2
+# BATCH_DELAY=0
+
+# For 1 API Instance (Single Instance Setup):
+# MAX_API_CONNECTIONS=5
+# BATCH_SIZE=20
+# MAX_CONCURRENT_SOURCES=1
+# BATCH_DELAY=100
+
+# =============================================================================
+# LOGGING AND MONITORING
+# =============================================================================
+
+# Progress bar update frequency
+PROGRESS_UPDATE_INTERVAL=10
 
-# RFC Upload Configuration
-# Pipe-separated list of RFCs to upload files for
-# Example: MMJ0810145N1|ABC1234567XY|DEF9876543ZZ
-UPLOAD_RFCS=RFC1|RFC2|RFC3
+# Enable verbose logging (true/false)
+VERBOSE_LOGGING=false

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arela/uploader",
-  "version": "0.2.7",
+  "version": "0.2.9",
   "description": "CLI to upload files/directories to Arela",
   "bin": {
     "arela": "./src/index.js"

package/src/commands/UploadCommand.js

@@ -50,7 +50,7 @@ export class UploadCommand {
         logger.info('Log file cleared');
       }
 
-      // Process each source
+      // Process each source with configurable concurrency
       let globalResults = {
         successCount: 0,
         detectedCount: 0,
@@ -59,28 +59,88 @@ export class UploadCommand {
         skippedCount: 0,
       };
 
-      for (const source of sources) {
-        const sourcePath = path.resolve(basePath, source).replace(/\\/g, '/');
-        logger.info(`Processing folder: ${sourcePath}`);
-
-        try {
-          const files = await this.#discoverFiles(sourcePath);
-          logger.info(`Found ${files.length} files to process`);
-
-          const result = await this.#processFilesInBatches(
-            files,
-            options,
-            uploadService,
-            basePath,
-            source,
-            sourcePath,
-          );
+      // Determine processing strategy based on configuration
+      const maxConcurrentSources =
+        appConfig.performance?.maxConcurrentSources || 1;
+
+      if (maxConcurrentSources > 1 && sources.length > 1) {
+        // Parallel source processing
+        logger.info(
+          `Processing ${sources.length} sources with concurrency: ${maxConcurrentSources}`,
+        );
 
-          this.#updateGlobalResults(globalResults, result);
-          this.#logSourceSummary(source, result, options);
-        } catch (error) {
-          this.errorHandler.handleError(error, { source, sourcePath });
-          globalResults.failureCount++;
+        // Process sources in batches to control concurrency
+        for (let i = 0; i < sources.length; i += maxConcurrentSources) {
+          const sourceBatch = sources.slice(i, i + maxConcurrentSources);
+
+          const sourcePromises = sourceBatch.map(async (source) => {
+            const sourcePath = path
+              .resolve(basePath, source)
+              .replace(/\\/g, '/');
+            logger.info(`Processing folder: ${sourcePath}`);
+
+            try {
+              const files = await this.#discoverFiles(sourcePath);
+              logger.info(`Found ${files.length} files in ${source}`);
+
+              const result = await this.#processFilesInBatches(
+                files,
+                options,
+                uploadService,
+                basePath,
+                source,
+                sourcePath,
+              );
+
+              this.#logSourceSummary(source, result, options);
+              return { success: true, source, result };
+            } catch (error) {
+              this.errorHandler.handleError(error, { source, sourcePath });
+              return { success: false, source, error: error.message };
+            }
+          });
+
+          // Wait for this batch of sources to complete
+          const results = await Promise.allSettled(sourcePromises);
+
+          results.forEach((result) => {
+            if (result.status === 'fulfilled') {
+              const sourceResult = result.value;
+              if (sourceResult.success) {
+                this.#updateGlobalResults(globalResults, sourceResult.result);
+              } else {
+                globalResults.failureCount++;
+              }
+            } else {
+              globalResults.failureCount++;
+            }
+          });
+        }
+      } else {
+        // Sequential source processing (original behavior)
+        for (const source of sources) {
+          const sourcePath = path.resolve(basePath, source).replace(/\\/g, '/');
+          logger.info(`Processing folder: ${sourcePath}`);
+
+          try {
+            const files = await this.#discoverFiles(sourcePath);
+            logger.info(`Found ${files.length} files to process`);
+
+            const result = await this.#processFilesInBatches(
+              files,
+              options,
+              uploadService,
+              basePath,
+              source,
+              sourcePath,
+            );
+
+            this.#updateGlobalResults(globalResults, result);
+            this.#logSourceSummary(source, result, options);
+          } catch (error) {
+            this.errorHandler.handleError(error, { source, sourcePath });
+            globalResults.failureCount++;
+          }
         }
       }
 
@@ -164,7 +224,8 @@ export class UploadCommand {
     source,
     sourcePath,
   ) {
-    const batchSize = parseInt(options.batchSize) || 10;
+    const batchSize =
+      parseInt(options.batchSize) || appConfig.performance.batchSize || 50;
     const results = {
       successCount: 0,
       detectedCount: 0,
@@ -184,6 +245,9 @@ export class UploadCommand {
       barCompleteChar: '█',
       barIncompleteChar: '░',
       hideCursor: true,
+      clearOnComplete: false,
+      stopOnComplete: true,
+      stream: process.stderr, // Use stderr to separate from stdout logging
     });
 
     progressBar.start(files.length, 0, { success: 0, errors: 0 });
@@ -265,22 +329,65 @@ export class UploadCommand {
         throw new Error(`Failed to insert stats: ${error.message}`);
       }
     } else {
-      // Upload mode: process files for upload
-      for (const filePath of batch) {
-        try {
-          await this.#processFile(
-            filePath,
-            options,
-            uploadService,
-            basePath,
-            processedPaths,
-            batchResults,
-          );
-        } catch (error) {
-          this.errorHandler.handleError(error, { filePath });
-          batchResults.failureCount++;
+      // Upload mode: process files with controlled concurrency to match API replicas
+      const maxConcurrentApiCalls =
+        appConfig.performance?.maxApiConnections || 10;
+
+      // Process batch in chunks to respect API replica limits
+      const allResults = [];
+      for (let i = 0; i < batch.length; i += maxConcurrentApiCalls) {
+        const chunk = batch.slice(i, i + maxConcurrentApiCalls);
+
+        // Process this chunk concurrently (up to API replica count)
+        const chunkPromises = chunk.map(async (filePath) => {
+          try {
+            const result = await this.#processFile(
+              filePath,
+              options,
+              uploadService,
+              basePath,
+              processedPaths,
+            );
+            return { success: true, filePath, result };
+          } catch (error) {
+            this.errorHandler.handleError(error, { filePath });
+            return { success: false, filePath, error: error.message };
+          }
+        });
+
+        // Wait for this chunk to complete before starting the next
+        const chunkResults = await Promise.allSettled(chunkPromises);
+        allResults.push(...chunkResults);
+
+        // Small delay between chunks to prevent overwhelming API
+        if (i + maxConcurrentApiCalls < batch.length) {
+          await new Promise((resolve) => setTimeout(resolve, 50));
         }
       }
+
+      // Process all results and update batch results
+      allResults.forEach((result) => {
+        if (result.status === 'fulfilled') {
+          const fileResult = result.value;
+          if (fileResult.success) {
+            if (fileResult.result && fileResult.result.skipped) {
+              batchResults.skippedCount++;
+            } else {
+              batchResults.successCount++;
+              if (fileResult.result && fileResult.result.detectedCount) {
+                batchResults.detectedCount += fileResult.result.detectedCount;
+              }
+              if (fileResult.result && fileResult.result.organizedCount) {
+                batchResults.organizedCount += fileResult.result.organizedCount;
+              }
+            }
+          } else {
+            batchResults.failureCount++;
+          }
+        } else {
+          batchResults.failureCount++;
+        }
+      });
     }
 
     return batchResults;
@@ -296,12 +403,10 @@ export class UploadCommand {
     uploadService,
     basePath,
     processedPaths,
-    batchResults,
   ) {
     // Skip if already processed
     if (processedPaths.has(filePath)) {
-      batchResults.skippedCount++;
-      return;
+      return { skipped: true };
     }
 
     // Prepare file for upload
@@ -325,24 +430,25 @@ export class UploadCommand {
     };
 
     // Upload based on service type
+    let result = { successCount: 1 };
+
     if (uploadService.getServiceName() === 'Arela API') {
-      const result = await uploadService.upload([fileObject], {
+      result = await uploadService.upload([fileObject], {
         ...options,
         uploadPath,
       });
-
-      batchResults.successCount++;
-      if (result.detectedCount)
-        batchResults.detectedCount += result.detectedCount;
-      if (result.organizedCount)
-        batchResults.organizedCount += result.organizedCount;
     } else {
       // Supabase direct upload
       await uploadService.upload([fileObject], { uploadPath });
-      batchResults.successCount++;
     }
 
     logger.info(`SUCCESS: ${path.basename(filePath)} -> ${uploadPath}`);
+
+    return {
+      skipped: false,
+      detectedCount: result.detectedCount || 0,
+      organizedCount: result.organizedCount || 0,
+    };
   }
 
   /**
@@ -428,7 +534,8 @@ export class UploadCommand {
       // Phase 2: PDF Detection
       console.log('\n🔍 === PHASE 2: PDF Detection ===');
       const detectionResult = await databaseService.detectPedimentosInDatabase({
-        batchSize: parseInt(options.batchSize) || 10,
+        batchSize:
+          parseInt(options.batchSize) || appConfig.performance.batchSize || 50,
       });
       console.log(
         `✅ Phase 2 Complete: ${detectionResult.detectedCount} detected, ${detectionResult.errorCount} errors`,

package/src/config/config.js

@@ -28,10 +28,10 @@ class Config {
       const __dirname = path.dirname(__filename);
       const packageJsonPath = path.resolve(__dirname, '../../package.json');
       const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
-      return packageJson.version || '0.2.7';
+      return packageJson.version || '0.2.9';
     } catch (error) {
       console.warn('⚠️ Could not read package.json version, using fallback');
-      return '0.2.7';
+      return '0.2.9';
     }
   }
 
@@ -85,7 +85,12 @@ class Config {
    */
   #loadPerformanceConfig() {
     return {
-      batchDelay: parseInt(process.env.BATCH_DELAY) || 100,
+      batchDelay: parseInt(process.env.BATCH_DELAY) || 0, // Removed default delay
+      batchSize: parseInt(process.env.BATCH_SIZE) || 50, // Increased from 10 to 50
+      maxConcurrentSources: parseInt(process.env.MAX_CONCURRENT_SOURCES) || 2,
+      maxApiConnections: parseInt(process.env.MAX_API_CONNECTIONS) || 10, // New: API replica support
+      apiConnectionTimeout:
+        parseInt(process.env.API_CONNECTION_TIMEOUT) || 60000, // New: API timeout
       progressUpdateInterval:
         parseInt(process.env.PROGRESS_UPDATE_INTERVAL) || 10,
       logBufferSize: 100,