@arela/uploader 0.2.11 → 0.2.13

package/.vscode/settings.json

@@ -0,0 +1 @@
+{}

package/SUPABASE_UPLOAD_FIX.md

@@ -0,0 +1,157 @@
+# Supabase Upload Issue Fix
+
+## Problem Description
+
+When running the CLI with `upload --force-supabase`, the logs displayed that files were being uploaded successfully, but no files appeared in the Supabase bucket.
+
+## Root Cause
+
+The issue was in `/src/commands/UploadCommand.js` in the `#processFile` method. The code was **silently ignoring upload failures** from Supabase:
+
+```javascript
+// ❌ BEFORE - Upload result was ignored
+if (uploadService.getServiceName() === 'Arela API') {
+  result = await uploadService.upload([fileObject], { ...options, uploadPath });
+} else {
+  // Supabase direct upload
+  await uploadService.upload([fileObject], { uploadPath });
+  // Result ignored! No error checking!
+}
+
+logger.info(`SUCCESS: ${path.basename(filePath)} -> ${uploadPath}`);
+```
+
+Even if the Supabase upload failed and returned `{ success: false, error: "..." }`, the code would:
+1. Ignore the error
+2. Log "SUCCESS" 
+3. Continue processing
+
+This created the illusion that files were being uploaded when they weren't.
+
+## Changes Made
+
+### 1. Fixed Error Handling in `UploadCommand.js`
+
+**File:** `/src/commands/UploadCommand.js`
+
+Now properly captures and validates the Supabase upload result:
+
+```javascript
+// ✅ AFTER - Upload result is captured and validated
+if (uploadService.getServiceName() === 'Arela API') {
+  result = await uploadService.upload([fileObject], { ...options, uploadPath });
+} else {
+  // Supabase direct upload
+  const uploadResult = await uploadService.upload([fileObject], { uploadPath });
+  
+  // Check if upload was successful
+  if (!uploadResult.success) {
+    throw new Error(`Supabase upload failed: ${uploadResult.error}`);
+  }
+  
+  result = { successCount: 1 };
+}
+```
+
+### 2. Enhanced Logging in `SupabaseUploadService.js`
+
+**File:** `/src/services/upload/SupabaseUploadService.js`
+
+Added detailed logging to help diagnose upload issues:
+
+#### Connection Initialization
+```javascript
+console.log(`🔌 Connecting to Supabase: ${appConfig.supabase.url}`);
+console.log(`📦 Using bucket: ${this.bucket}`);
+console.log(`🧪 Testing bucket access...`);
+console.log(`✅ Successfully connected to Supabase bucket: ${this.bucket}`);
+```
+
+#### Upload Operations
+```javascript
+if (error) {
+  console.error(`❌ Supabase upload failed for ${normalizedPath}:`, error.message);
+  return { success: false, error: error.message };
+}
+
+console.log(`✅ Supabase upload successful: ${normalizedPath}`);
+```
+
+#### Better Error Messages
+```javascript
+if (!appConfig.supabase.url || !appConfig.supabase.key || !this.bucket) {
+  const missing = [];
+  if (!appConfig.supabase.url) missing.push('SUPABASE_URL');
+  if (!appConfig.supabase.key) missing.push('SUPABASE_KEY');
+  if (!this.bucket) missing.push('SUPABASE_BUCKET');
+  throw new Error(`Missing Supabase configuration: ${missing.join(', ')}`);
+}
+```
+
+## Expected Behavior After Fix
+
+### On Successful Upload
+```
+🔌 Connecting to Supabase: https://your-project.supabase.co
+📦 Using bucket: your-bucket-name
+🧪 Testing bucket access...
+✅ Successfully connected to Supabase bucket: your-bucket-name
+✅ Supabase upload successful: 2023/2000601/file.xml
+```
+
+### On Failed Upload
+```
+❌ Supabase upload failed for 2023/2000601/file.xml: [error details]
+Error: Supabase upload failed: [error details]
+```
+
+### On Configuration Error
+```
+Error: Missing Supabase configuration: SUPABASE_URL, SUPABASE_KEY
+```
+
+## Common Issues to Check
+
+If uploads still fail after this fix, check:
+
+1. **Environment Variables**
+   ```bash
+   echo $SUPABASE_URL
+   echo $SUPABASE_KEY  # Should be anon or service_role key
+   echo $SUPABASE_BUCKET
+   ```
+
+2. **Bucket Permissions**
+   - Check if the bucket exists in Supabase Storage
+   - Verify the API key has write permissions to the bucket
+   - Check bucket policies (public vs private)
+
+3. **Network/Firewall**
+   - Ensure the server can reach Supabase (check firewall rules)
+   - Test connection: `curl https://your-project.supabase.co`
+
+4. **File Size Limits**
+   - Check Supabase storage limits for your plan
+   - Verify file sizes are within allowed limits
+
+5. **Path Format**
+   - Ensure paths don't contain invalid characters
+   - Paths are normalized (forward slashes only)
+
+## Testing the Fix
+
+Run the upload command with verbose logging:
+
+```bash
+arela upload --force-supabase -v
+```
+
+You should now see:
+- Clear connection status messages
+- Per-file upload success/failure messages
+- Proper error messages if uploads fail
+- Accurate success/error counts in the summary
+
+## Migration Notes
+
+No database migrations required. This is a code-only fix that improves error handling and logging.

package/commands.md

@@ -3,4 +3,12 @@ node src/index.js --detect-pdfs
 node src/index.js --propagate-arela-path
 node src/index.js --upload-by-rfc --folder-structure palco
 
-UPLOAD_RFCS="RFC1|RFC2" node src/index.js --upload-by-rfc --folder-structure target-folder
+UPLOAD_RFCS="RFC1|RFC2" node src/index.js --upload-by-rfc --folder-structure target-folder
+
+
+
+
+node src/index.js stats --stats-only
+node src/index.js detect --detect-pdfs
+node src/index.js detect --propagate-arela-path
+node src/index.js upload --upload-by-rfc --folder-structure palco

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arela/uploader",
-  "version": "0.2.11",
+  "version": "0.2.13",
   "description": "CLI to upload files/directories to Arela",
   "bin": {
     "arela": "./src/index.js"

package/src/commands/UploadCommand.js

@@ -439,7 +439,14 @@ export class UploadCommand {
       });
     } else {
       // Supabase direct upload
-      await uploadService.upload([fileObject], { uploadPath });
+      const uploadResult = await uploadService.upload([fileObject], { uploadPath });
+      
+      // Check if upload was successful
+      if (!uploadResult.success) {
+        throw new Error(`Supabase upload failed: ${uploadResult.error}`);
+      }
+      
+      result = { successCount: 1 };
     }
 
     logger.info(`SUCCESS: ${path.basename(filePath)} -> ${uploadPath}`);

package/src/config/config.js

@@ -28,10 +28,10 @@ class Config {
       const __dirname = path.dirname(__filename);
       const packageJsonPath = path.resolve(__dirname, '../../package.json');
       const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
-      return packageJson.version || '0.2.11';
+      return packageJson.version || '0.2.12';
     } catch (error) {
       console.warn('⚠️ Could not read package.json version, using fallback');
-      return '0.2.11';
+      return '0.2.12';
     }
   }
 

package/src/services/DatabaseService.js

@@ -51,6 +51,7 @@ export class DatabaseService {
           error.message?.includes('timeout') ||
           error.message?.includes('canceling statement') ||
           error.message?.includes('connection') ||
+          error.message?.includes('fetch failed') ||
           error.code === 'PGRST301'; // PostgREST timeout
 
         if (!isRetriableError || attempt === maxRetries) {
@@ -650,16 +651,18 @@ export class DatabaseService {
 
           while (hasMoreRelatedFiles) {
             const { data: relatedFilesPage, error: relatedError } =
-              await supabase
-                .from('uploader')
-                .select('id, filename, original_path')
-                .like('original_path', `${basePath}%`)
-                .is('arela_path', null)
-                .neq('id', pedimento.id) // Exclude the pedimento itself
-                .range(
-                  relatedFilesFrom,
-                  relatedFilesFrom + relatedFilesPageSize - 1,
-                );
+              await this.#queryWithRetry(async () => {
+                return await supabase
+                  .from('uploader')
+                  .select('id, filename, original_path')
+                  .like('original_path', `${basePath}%`)
+                  .is('arela_path', null)
+                  .neq('id', pedimento.id) // Exclude the pedimento itself
+                  .range(
+                    relatedFilesFrom,
+                    relatedFilesFrom + relatedFilesPageSize - 1,
+                  );
+              }, `query related files for ${pedimento.filename} (page ${relatedFilesPageNumber})`);
 
             if (relatedError) {
               logger.error(
@@ -730,7 +733,7 @@ export class DatabaseService {
 
             // Check if we need to fetch the next page of related files
             if (relatedFilesPage.length < relatedFilesPageSize) {
-              console.log('no more related files for ', basePath);
+              logger.info('No more related files for', basePath);
               hasMoreRelatedFiles = false;
               logger.info(
                 `Completed processing related files for ${pedimento.filename}. Total processed: ${totalRelatedFilesProcessed}`,
@@ -930,193 +933,87 @@ export class DatabaseService {
       `Found ${allPedimentoRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths across ${pageNumber - 1} pages`,
     );
 
-    // Step 2: Get all files with these arela_paths that haven't been uploaded yet
-    let rfcRecords = [];
-    const chunkSize = 50;
-
-    for (let i = 0; i < uniqueArelaPaths.length; i += chunkSize) {
-      const pathChunk = uniqueArelaPaths.slice(i, i + chunkSize);
-
-      const { data: chunkFiles, error: chunkError } = await supabase
-        .from('uploader')
-        .select('arela_path')
-        .in('arela_path', pathChunk)
-        .neq('status', 'file-uploaded')
-        .not('arela_path', 'is', null);
-
-      if (chunkError) {
-        const errorMsg = `Error fetching files for arela_paths chunk: ${chunkError.message}`;
-        logger.error(errorMsg);
-        throw new Error(errorMsg);
-      }
-
-      if (chunkFiles && chunkFiles.length > 0) {
-        rfcRecords = rfcRecords.concat(chunkFiles);
-      }
-    }
-
-    if (!rfcRecords || rfcRecords.length === 0) {
-      if (totalRfcFiles && totalRfcFiles > 0) {
-        console.log(
-          `ℹ️  All ${totalRfcFiles} files for the specified RFCs are already uploaded (status: file-uploaded)`,
-        );
-        console.log('   No new files to upload.');
-        logger.info(
-          `All ${totalRfcFiles} files for specified RFCs already uploaded`,
-        );
-      } else {
-        console.log(
-          'ℹ️  No files found for the specified RFCs with arela_path',
-        );
-        console.log(
-          `   Make sure files for RFCs [${appConfig.upload.rfcs.join(', ')}] have been processed and have arela_path values`,
-        );
-        logger.info('No files found for specified RFCs with arela_path');
-      }
-      return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
-    }
-
-    // Show filtering effect
-    const uploadableArelaPaths = [
-      ...new Set(rfcRecords.map((r) => r.arela_path)),
-    ];
-    const skipped = (totalRfcFiles || 0) - rfcRecords.length;
-    if (skipped > 0) {
-      console.log(
-        `📊 Found ${rfcRecords.length} files ready for upload (${skipped} already uploaded, skipped)`,
-      );
-    } else {
-      console.log(`📊 Found ${rfcRecords.length} files ready for upload`);
-    }
-    logger.info(
-      `Found ${rfcRecords.length} files ready for upload, ${skipped} skipped`,
-    );
-
-    // Step 3: Process files with streaming pagination to avoid memory overload
+    // Step 2: Process files with optimized single query per chunk
     let totalProcessed = 0;
     let totalUploaded = 0;
     let totalErrors = 0;
     let globalFileCount = 0;
     const arelaPathChunkSize = 50;
-    const queryBatchSize = 500; // Reduced batch size for better memory management
     const batchSize = parseInt(options.batchSize) || 10;
 
     // Import performance configuration
     const { performance: perfConfig } = appConfig;
     const maxConcurrency = perfConfig?.maxApiConnections || 3;
 
-    console.log(
-      '📥 Processing files with streaming pagination (processing arela_paths in chunks to avoid URI limits and memory overload)...',
-    );
+    console.log('📥 Processing files in chunks to avoid URI limits...');
 
     // Process arela_paths in chunks and upload files as we fetch them
-    for (let i = 0; i < uploadableArelaPaths.length; i += arelaPathChunkSize) {
-      const arelaPathChunk = uploadableArelaPaths.slice(
-        i,
-        i + arelaPathChunkSize,
-      );
+    for (let i = 0; i < uniqueArelaPaths.length; i += arelaPathChunkSize) {
+      const arelaPathChunk = uniqueArelaPaths.slice(i, i + arelaPathChunkSize);
       const chunkNumber = Math.floor(i / arelaPathChunkSize) + 1;
       const totalChunks = Math.ceil(
-        uploadableArelaPaths.length / arelaPathChunkSize,
+        uniqueArelaPaths.length / arelaPathChunkSize,
       );
 
       console.log(
         `   Processing arela_path chunk ${chunkNumber}/${totalChunks} (${arelaPathChunk.length} paths)`,
       );
 
-      // For each chunk of arela_paths, use pagination to get related files and process them immediately
-      let hasMore = true;
-      let offset = 0;
-      let chunkFileCount = 0;
-
-      while (hasMore) {
-        const { data: batch, error: queryError } = await supabase
-          .from('uploader')
-          .select('id, original_path, arela_path, filename, rfc, document_type')
-          .in('arela_path', arelaPathChunk)
-          .not('original_path', 'is', null)
-          .neq('status', 'file-uploaded')
-          .range(offset, offset + queryBatchSize - 1)
-          .order('created_at');
-
-        if (queryError) {
-          const errorMsg = `Error fetching related files for chunk ${chunkNumber}: ${queryError.message}`;
-          logger.error(errorMsg);
-          throw new Error(errorMsg);
-        }
-
-        if (!batch || batch.length === 0) {
-          hasMore = false;
-          break;
-        }
+      // Fetch all files for this chunk in a single query
+      const { data: batch, error: queryError } = await supabase
+        .from('uploader')
+        .select('id, original_path, arela_path, filename, rfc, document_type')
+        .in('arela_path', arelaPathChunk)
+        .neq('status', 'file-uploaded')
+        .order('created_at');
 
-        chunkFileCount += batch.length;
-        globalFileCount += batch.length;
+      if (queryError) {
+        const errorMsg = `Error fetching files for chunk ${chunkNumber}: ${queryError.message}`;
+        logger.error(errorMsg);
+        throw new Error(errorMsg);
+      }
 
+      if (!batch || batch.length === 0) {
         console.log(
-          `   📦 Processing batch within chunk ${chunkNumber}: ${batch.length} files (total processed so far: ${globalFileCount})`,
+          `   ℹ️  Chunk ${chunkNumber}/${totalChunks}: No files to upload`,
         );
+        continue;
+      }
 
-        // Track if any uploads occurred in this batch
-        let uploadsOccurred = false;
-
-        // Process this batch of files immediately using concurrent processing
-        // Split batch into upload batches
-        for (let j = 0; j < batch.length; j += batchSize) {
-          const uploadBatch = batch.slice(j, j + batchSize);
-          const batchNum = Math.floor(globalFileCount / batchSize) + 1;
-
-          console.log(
-            `📦 Processing upload batch ${batchNum} (${uploadBatch.length} files)`,
-          );
+      const chunkFileCount = batch.length;
+      globalFileCount += chunkFileCount;
 
-          // Process batch using concurrent processing similar to UploadCommand
-          const batchResults = await this.#processRfcBatch(
-            uploadBatch,
-            uploadService,
-            supabase,
-            options,
-            maxConcurrency,
-          );
+      console.log(
+        `   📦 Chunk ${chunkNumber}/${totalChunks}: Processing ${chunkFileCount} files`,
+      );
 
-          totalProcessed += batchResults.processed;
-          totalUploaded += batchResults.uploaded;
-          totalErrors += batchResults.errors;
+      // Process this batch of files immediately using concurrent processing
+      // Split batch into upload batches
+      for (let j = 0; j < batch.length; j += batchSize) {
+        const uploadBatch = batch.slice(j, j + batchSize);
+        const batchNum = Math.floor(j / batchSize) + 1;
+        const totalBatches = Math.ceil(batch.length / batchSize);
 
-          // Track if uploads occurred (status changes from non-uploaded to uploaded)
-          if (batchResults.uploaded > 0) {
-            uploadsOccurred = true;
-          }
+        console.log(
+          `   📦 Processing upload batch ${batchNum}/${totalBatches} within chunk ${chunkNumber} (${uploadBatch.length} files)`,
+        );
 
-          console.log(
-            `📊 Upload batch complete - Total progress: ${totalUploaded} uploaded, ${totalErrors} errors`,
-          );
-        }
+        // Process batch using concurrent processing similar to UploadCommand
+        const batchResults = await this.#processRfcBatch(
+          uploadBatch,
+          uploadService,
+          supabase,
+          options,
+          maxConcurrency,
+        );
 
-        // Check if we need more data from this chunk
-        if (batch.length < queryBatchSize) {
-          hasMore = false;
-        } else {
-          // If uploads occurred, reset pagination to start from beginning
-          // since records that matched the query may no longer match after upload
-          if (uploadsOccurred) {
-            offset = 0;
-            console.log(
-              `   📄 Batch complete with uploads: ${batch.length} files processed, restarting pagination from beginning due to query condition changes...`,
-            );
-          } else {
-            offset += queryBatchSize;
-            console.log(
-              `   📄 Batch complete: ${batch.length} files processed, continuing to next page (offset: ${offset})...`,
-            );
-          }
-        }
+        totalProcessed += batchResults.processed;
+        totalUploaded += batchResults.uploaded;
+        totalErrors += batchResults.errors;
 
-        if (hasMore) {
-          console.log(
-            `   📄 Fetching more files for chunk ${chunkNumber}... (offset: ${offset})`,
-          );
-        }
+        console.log(
+          `   📊 Batch complete - Progress: ${totalUploaded} uploaded, ${totalErrors} errors`,
+        );
       }
 
       console.log(

package/src/services/upload/ApiUploadService.js

@@ -63,16 +63,20 @@ export class ApiUploadService extends BaseUploadService {
     for (const file of files) {
       try {
         // Check file size for streaming vs buffer approach
-        const stats = await fs.promises.stat(file.path);
+        let size = file.size;
+        if (size === undefined || size === null) {
+          const stats = await fs.promises.stat(file.path);
+          size = stats.size;
+        }
         const fileSizeThreshold = 10 * 1024 * 1024; // 10MB threshold
 
-        if (stats.size > fileSizeThreshold) {
+        if (size > fileSizeThreshold) {
           // Use streaming for large files
           const fileStream = fs.createReadStream(file.path);
           formData.append('files', fileStream, {
             filename: file.name,
             contentType: file.contentType,
-            knownLength: stats.size,
+            knownLength: size,
           });
         } else {
           // Use buffer for smaller files

package/src/services/upload/SupabaseUploadService.js

@@ -26,9 +26,16 @@ export class SupabaseUploadService extends BaseUploadService {
     }
 
     if (!appConfig.supabase.url || !appConfig.supabase.key || !this.bucket) {
-      throw new Error('Missing Supabase configuration');
+      const missing = [];
+      if (!appConfig.supabase.url) missing.push('SUPABASE_URL');
+      if (!appConfig.supabase.key) missing.push('SUPABASE_KEY');
+      if (!this.bucket) missing.push('SUPABASE_BUCKET');
+      throw new Error(`Missing Supabase configuration: ${missing.join(', ')}`);
     }
 
+    console.log(`🔌 Connecting to Supabase: ${appConfig.supabase.url}`);
+    console.log(`📦 Using bucket: ${this.bucket}`);
+
     this.client = createClient(appConfig.supabase.url, appConfig.supabase.key, {
       db: {
         schema: 'public',
@@ -48,11 +55,15 @@ export class SupabaseUploadService extends BaseUploadService {
       },
     });
 
-    // Test connection
-    const { error } = await this.client.storage.from(this.bucket).list('');
+    // Test connection and bucket access
+    console.log(`🧪 Testing bucket access...`);
+    const { error } = await this.client.storage.from(this.bucket).list('', { limit: 1 });
     if (error) {
-      throw new Error(`Error connecting to Supabase: ${error.message}`);
+      console.error(`❌ Bucket access test failed:`, error.message);
+      throw new Error(`Error connecting to Supabase bucket '${this.bucket}': ${error.message}`);
     }
+    
+    console.log(`✅ Successfully connected to Supabase bucket: ${this.bucket}`);
   }
 
   /**
@@ -80,18 +91,22 @@ export class SupabaseUploadService extends BaseUploadService {
     const content = fs.readFileSync(file.path);
     const contentType = mime.lookup(file.path) || 'application/octet-stream';
 
+    const normalizedPath = uploadPath.replace(/\\/g, '/');
+    
     const { data, error } = await this.client.storage
       .from(this.bucket)
-      .upload(uploadPath.replace(/\\/g, '/'), content, {
+      .upload(normalizedPath, content, {
         upsert: true,
         contentType,
       });
 
     if (error) {
+      console.error(`❌ Supabase upload failed for ${normalizedPath}:`, error.message);
       return { success: false, error: error.message };
     }
 
-    return { success: true, data };
+    console.log(`✅ Supabase upload successful: ${normalizedPath}`);
+    return { success: true, data, uploadPath: normalizedPath };
   }
 
   /**