@arela/uploader 0.2.2 → 0.2.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arela/uploader",
-  "version": "0.2.2",
+  "version": "0.2.3",
   "description": "CLI to upload files/directories to Arela",
   "bin": {
     "arela": "./src/index.js"

package/src/index.js

@@ -158,8 +158,8 @@ const checkCredentials = async (forceSupabase = false) => {
     if (!supabaseUrl || !supabaseKey || !bucket) {
       console.error(
         '⚠️ Missing credentials. Please set either:\n' +
-          '   - ARELA_API_URL and ARELA_API_TOKEN for API mode, or\n' +
-          '   - SUPABASE_URL, SUPABASE_KEY, and SUPABASE_BUCKET for direct mode',
+        '   - ARELA_API_URL and ARELA_API_TOKEN for API mode, or\n' +
+        '   - SUPABASE_URL, SUPABASE_KEY, and SUPABASE_BUCKET for direct mode',
       );
       process.exit(1);
     }
@@ -1360,38 +1360,52 @@ const uploadFilesByRfc = async (options = {}) => {
   );
 
   // Step 3: Get ALL files that have these arela_paths (including supporting documents)
-  // Use pagination to ensure we get all files, regardless of count
+  // Process arela_paths in smaller chunks to avoid URI length limits
   let allRelatedFiles = [];
-  let hasMore = true;
-  let offset = 0;
+  const arelaPathChunkSize = 50; // Process 50 arela_paths at a time to avoid URI limits
   const queryBatchSize = 1000;
 
-  console.log('📥 Fetching all related files (with pagination)...');
+  console.log('📥 Fetching all related files (processing arela_paths in chunks to avoid URI limits)...');
 
-  while (hasMore) {
-    const { data: batch, error: queryError } = await supabase
-      .from('uploader')
-      .select('id, original_path, arela_path, filename, rfc, document_type')
-      .in('arela_path', uniqueArelaPaths)
-      .not('original_path', 'is', null)
-      .range(offset, offset + queryBatchSize - 1);
+  // Process arela_paths in chunks
+  for (let i = 0; i < uniqueArelaPaths.length; i += arelaPathChunkSize) {
+    const arelaPathChunk = uniqueArelaPaths.slice(i, i + arelaPathChunkSize);
+    console.log(`   Processing arela_path chunk ${Math.floor(i / arelaPathChunkSize) + 1}/${Math.ceil(uniqueArelaPaths.length / arelaPathChunkSize)} (${arelaPathChunk.length} paths)`);
 
-    if (queryError) {
-      console.error('❌ Error fetching related files:', queryError.message);
-      return { processedCount: 0, uploadedCount: 0, errorCount: 1 };
-    }
+    // For each chunk of arela_paths, use pagination to get all related files
+    let hasMore = true;
+    let offset = 0;
 
-    if (!batch || batch.length === 0) {
-      hasMore = false;
-    } else {
-      allRelatedFiles = allRelatedFiles.concat(batch);
-      offset += queryBatchSize;
+    while (hasMore) {
+      const { data: batch, error: queryError } = await supabase
+        .from('uploader')
+        .select('id, original_path, arela_path, filename, rfc, document_type')
+        .in('arela_path', arelaPathChunk)
+        .not('original_path', 'is', null)
+        .range(offset, offset + queryBatchSize - 1);
+
+      if (queryError) {
+        console.error(`❌ Error fetching related files for chunk ${Math.floor(i / arelaPathChunkSize) + 1}:`, queryError.message);
+        return { processedCount: 0, uploadedCount: 0, errorCount: 1 };
+      }
 
-      // If we got less than queryBatchSize, we've reached the end
-      if (batch.length < queryBatchSize) {
+      if (!batch || batch.length === 0) {
         hasMore = false;
+      } else {
+        allRelatedFiles = allRelatedFiles.concat(batch);
+        offset += queryBatchSize;
+
+        // If we got less than queryBatchSize, we've reached the end for this chunk
+        if (batch.length < queryBatchSize) {
+          hasMore = false;
+        }
       }
     }
+
+    // Small delay between chunks to avoid overwhelming the database
+    if (i + arelaPathChunkSize < uniqueArelaPaths.length) {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    }
   }
 
   if (!allRelatedFiles || allRelatedFiles.length === 0) {
@@ -1560,8 +1574,8 @@ const uploadFilesByRfc = async (options = {}) => {
           // Set folder structure for this group - concatenate custom prefix with arela_path
           const folderStructure = options.folderStructure
             ? `${options.folderStructure}/${arelaPath}`
-                .replace(/\/+/g, '/')
-                .replace(/\/$/, '')
+              .replace(/\/+/g, '/')
+              .replace(/\/$/, '')
             : arelaPath;
           pathFormData.append('folderStructure', folderStructure);
           pathFormData.append('autoDetect', 'true');