@arela/uploader 0.2.7 → 0.2.9

package/src/services/upload/ApiUploadService.js

@@ -1,6 +1,8 @@
 import { Blob } from 'buffer';
 import { FormData } from 'formdata-node';
 import fs from 'fs';
+import { Agent } from 'http';
+import { Agent as HttpsAgent } from 'https';
 import fetch from 'node-fetch';
 import path from 'path';
 
@@ -16,6 +18,36 @@ export class ApiUploadService extends BaseUploadService {
     super();
     this.baseUrl = appConfig.api.baseUrl;
     this.token = appConfig.api.token;
+
+    // Get API connection settings from config/environment
+    const maxApiConnections = parseInt(process.env.MAX_API_CONNECTIONS) || 10;
+    const connectionTimeout =
+      parseInt(process.env.API_CONNECTION_TIMEOUT) || 60000;
+
+    // Initialize HTTP agents optimized for multiple API replicas
+    this.httpAgent = new Agent({
+      keepAlive: true,
+      keepAliveMsecs: 30000,
+      maxSockets: maxApiConnections, // Match your API replica count
+      maxFreeSockets: Math.ceil(maxApiConnections / 2),
+      maxTotalSockets: maxApiConnections + 5, // Buffer for peak usage
+      timeout: connectionTimeout,
+      scheduling: 'fifo', // First-in-first-out scheduling
+    });
+
+    this.httpsAgent = new HttpsAgent({
+      keepAlive: true,
+      keepAliveMsecs: 30000,
+      maxSockets: maxApiConnections, // Match your API replica count
+      maxFreeSockets: Math.ceil(maxApiConnections / 2),
+      maxTotalSockets: maxApiConnections + 5, // Buffer for peak usage
+      timeout: connectionTimeout,
+      scheduling: 'fifo', // First-in-first-out scheduling
+    });
+
+    console.log(
+      `🔗 HTTP Agent configured for ${maxApiConnections} concurrent API connections`,
+    );
   }
 
   /**
@@ -27,12 +59,31 @@ export class ApiUploadService extends BaseUploadService {
   async upload(files, options) {
     const formData = new FormData();
 
-    // Add files to form data
-    files.forEach((file) => {
-      const fileBuffer = fs.readFileSync(file.path);
-      const blob = new Blob([fileBuffer], { type: file.contentType });
-      formData.append('files', blob, file.name);
-    });
+    // Add files to form data asynchronously
+    for (const file of files) {
+      try {
+        // Check file size for streaming vs buffer approach
+        const stats = await fs.promises.stat(file.path);
+        const fileSizeThreshold = 10 * 1024 * 1024; // 10MB threshold
+
+        if (stats.size > fileSizeThreshold) {
+          // Use streaming for large files
+          const fileStream = fs.createReadStream(file.path);
+          formData.append('files', fileStream, {
+            filename: file.name,
+            contentType: file.contentType,
+            knownLength: stats.size,
+          });
+        } else {
+          // Use buffer for smaller files
+          const fileBuffer = await fs.promises.readFile(file.path);
+          const blob = new Blob([fileBuffer], { type: file.contentType });
+          formData.append('files', blob, file.name);
+        }
+      } catch (error) {
+        throw new Error(`Failed to read file ${file.path}: ${error.message}`);
+      }
+    }
 
     // Add configuration parameters
     if (appConfig.supabase.bucket) {
@@ -61,6 +112,7 @@ export class ApiUploadService extends BaseUploadService {
     formData.append('clientVersion', appConfig.packageVersion);
 
     try {
+      const isHttps = this.baseUrl.startsWith('https');
       const response = await fetch(
         `${this.baseUrl}/api/storage/batch-upload-and-process`,
         {
@@ -69,6 +121,7 @@ export class ApiUploadService extends BaseUploadService {
             'x-api-key': this.token,
           },
           body: formData,
+          agent: isHttps ? this.httpsAgent : this.httpAgent,
         },
       );
 
@@ -81,8 +134,16 @@ export class ApiUploadService extends BaseUploadService {
 
       const result = await response.json();
 
+      // Determine if the operation was successful
+      // Success means at least one file was uploaded successfully, even if some failed
+      const hasUploads = result.uploaded && result.uploaded.length > 0;
+      const hasErrors = result.errors && result.errors.length > 0;
+      // Consider it a success if there are uploads, or if there are no errors at all
+      const success =
+        hasUploads || (!hasErrors && result.stats?.totalFiles > 0);
+
       // Normalize response format to match DatabaseService expectations
-      return { success: true, data: result };
+      return { success, data: result };
     } catch (fetchError) {
       // Return normalized error format
       return { success: false, error: fetchError.message };
@@ -99,10 +160,12 @@ export class ApiUploadService extends BaseUploadService {
     }
 
     try {
+      const isHttps = this.baseUrl.startsWith('https');
       const response = await fetch(`${this.baseUrl}/api/health`, {
         headers: {
           'x-api-key': this.token,
         },
+        agent: isHttps ? this.httpsAgent : this.httpAgent,
       });
 
       return response.ok;

package/src/services/upload/SupabaseUploadService.js

@@ -88,10 +88,10 @@ export class SupabaseUploadService extends BaseUploadService {
       });
 
     if (error) {
-      throw new Error(error.message);
+      return { success: false, error: error.message };
     }
 
-    return data;
+    return { success: true, data };
   }
 
   /**