@arela/uploader 0.2.9 → 0.2.11

package/.envbackup

@@ -0,0 +1,37 @@
+# SUPABASE_URL=https://supabase.prod.app.trader-docs.com
+# SUPABASE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoic2VydmljZV9yb2xlIiwiaXNzIjoic3VwYWJhc2UiLCJpYXQiOjE3NDU3MzM2MDAsImV4cCI6MTkwMzUwMDAwMH0.eAoXvaBZzZa31txItwEkoLnplWG10ee-U5GMc265xwk
+# SUPABASE_BUCKET=sample
+
+# PATH_VARS=o:/ExpedienteElectronicko/expediente/archivos/:rfcPatente/:patente-:aduana/:year/:pedimento/ALSJDKHF ASD,KFHJA SDFK,ASLDKJFH
+
+SUPABASE_URL=http://127.0.0.1:54321
+SUPABASE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU
+SUPABASE_BUCKET=arela
+
+# Arela API Configuration
+ARELA_API_URL=http://localhost:3010
+ARELA_API_TOKEN=f0608f83e5faa4c6be32c19975635c4e9012b31f7249a9f0a9270f54d74ec599
+
+# Upload Configuration
+UPLOAD_BASE_PATH=./sample
+UPLOAD_SOURCES=2023|2024
+
+# RFC Upload Configuration
+# Pipe-separated list of RFCs to upload files for
+# Example: MMJ0810145N1|ABC1234567XY|DEF9876543ZZ
+UPLOAD_RFCS=AKS151005E46|IMS030409FZ0|RDG1107154L7|SHP031226BV2|CSM9301219B4|LIN960124HT8|LME971009SW4|AKM9707151B6|FEL000822AG2|FDM060802J54|MTM9807279B4|AUM9207011CA|MMJ0810145N1|ACC010328EQ6|PED781129JT6|CAD890407NK7|SME140411IK7|JME1903121C2|EIJ110429NF9|PTJ080414TM6|TME050503BM4
+
+# AKS151005E46|IMS030409FZ0|RDG1107154L7|SHP031226BV2|CSM9301219B4|LIN960124HT8|LME971009SW4|AKM9707151B6|FEL000822AG2|PTJ080414TM6|TME050503BM4
+# FDM060802J54|MTM9807279B4|AUM9207011CA|MMJ0810145N1|ACC010328EQ6|PED781129JT6|CAD890407NK7|SME140411IK7|JME1903121C2|EIJ110429NF9
+
+# Logging and Verbosity
+VERBOSE_LOGGING=true          # Disable verbose path logging for better performance
+BATCH_DELAY=50                 # Reduce delay between batches (default: 100ms)
+PROGRESS_UPDATE_INTERVAL=20    # Update progress every 20 items (default: 10)
+
+# Log Buffering
+LOG_BUFFER_SIZE=200           # Increase buffer size for fewer I/O ops (default: 100)
+LOG_FLUSH_INTERVAL=3000       # Flush logs every 3 seconds (default: 5000ms)
+
+# Example for maximum performance
+# VERBOSE_LOGGING=false BATCH_DELAY=25 LOG_BUFFER_SIZE=500 arela --stats-only /path/to/files

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arela/uploader",
-  "version": "0.2.9",
+  "version": "0.2.11",
   "description": "CLI to upload files/directories to Arela",
   "bin": {
     "arela": "./src/index.js"

package/src/config/config.js

@@ -28,10 +28,10 @@ class Config {
       const __dirname = path.dirname(__filename);
       const packageJsonPath = path.resolve(__dirname, '../../package.json');
       const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
-      return packageJson.version || '0.2.9';
+      return packageJson.version || '0.2.11';
     } catch (error) {
       console.warn('⚠️ Could not read package.json version, using fallback');
-      return '0.2.9';
+      return '0.2.11';
     }
   }
 
@@ -72,10 +72,15 @@ class Config {
       .map((s) => s.trim())
       .filter(Boolean);
 
+    const uploadYears = process.env.UPLOAD_YEARS?.split('|')
+      .map((s) => s.trim())
+      .filter(Boolean);
+
     return {
       basePath,
       sources,
       rfcs: uploadRfcs,
+      years: uploadYears,
     };
   }
 

package/src/services/DatabaseService.js

@@ -316,7 +316,6 @@ export class DatabaseService {
                 filename: record.filename,
                 file_extension: record.file_extension,
                 is_like_simplificado: record.is_like_simplificado,
-                year: record.year,
               })
               .eq('original_path', record.original_path);
 
@@ -557,169 +556,240 @@ export class DatabaseService {
     const supabase = await this.#getSupabaseClient();
 
     logger.info('Phase 3: Starting arela_path and year propagation process...');
-    console.log(
-      '🔍 Finding pedimento_simplificado records with arela_path and year...',
-    );
-
-    // Get all pedimento_simplificado records that have arela_path
-    const { data: pedimentoRecords, error: pedimentoError } = await supabase
-      .from('uploader')
-      .select('id, original_path, arela_path, filename, year')
-      .eq('document_type', 'pedimento_simplificado')
-      .not('arela_path', 'is', null);
-
-    if (pedimentoError) {
-      const errorMsg = `Error fetching pedimento records: ${pedimentoError.message}`;
-      logger.error(errorMsg);
-      throw new Error(errorMsg);
-    }
+    console.log('🔍 Processing pedimento_simplificado records page by page...');
 
-    if (!pedimentoRecords || pedimentoRecords.length === 0) {
-      logger.info('No pedimento_simplificado records with arela_path found');
+    // Log year filtering configuration
+    if (appConfig.upload.years && appConfig.upload.years.length > 0) {
+      logger.info(
+        `🗓️  Year filter enabled: ${appConfig.upload.years.join(', ')}`,
+      );
       console.log(
-        'ℹ️  No pedimento_simplificado records with arela_path found',
+        `🗓️  Year filter enabled: ${appConfig.upload.years.join(', ')}`,
       );
-      return { processedCount: 0, updatedCount: 0, errorCount: 0 };
+    } else {
+      logger.info('🗓️  No year filter configured - processing all years');
+      console.log('🗓️  No year filter configured - processing all years');
     }
 
-    console.log(
-      `📋 Found ${pedimentoRecords.length} pedimento records with arela_path`,
-    );
-    logger.info(
-      `Found ${pedimentoRecords.length} pedimento records with arela_path to process`,
-    );
-
     let totalProcessed = 0;
     let totalUpdated = 0;
     let totalErrors = 0;
+    let offset = 0;
+    const pageSize = 50;
+    let hasMoreData = true;
+    let pageNumber = 1;
     const BATCH_SIZE = 50; // Process files in batches
 
-    // Process each pedimento record
-    for (const pedimento of pedimentoRecords) {
-      try {
-        totalProcessed++;
-
-        // Extract base path from original_path (remove filename)
-        const basePath = path.dirname(pedimento.original_path);
+    // Process pedimento records page by page for memory efficiency
+    while (hasMoreData) {
+      logger.info(
+        `Fetching and processing pedimento records page ${pageNumber} (records ${offset + 1} to ${offset + pageSize})...`,
+      );
 
-        logger.info(
-          `Processing pedimento: ${pedimento.filename} | Base path: ${basePath} | Year: ${pedimento.year || 'N/A'}`,
-        );
+      let query = supabase
+        .from('uploader')
+        .select('id, original_path, arela_path, filename, year')
+        .eq('document_type', 'pedimento_simplificado')
+        .not('arela_path', 'is', null);
 
-        // Extract folder part from existing arela_path
-        const existingPath = pedimento.arela_path;
-        const folderArelaPath = existingPath.includes('/')
-          ? existingPath.substring(0, existingPath.lastIndexOf('/')) + '/'
-          : existingPath.endsWith('/')
-            ? existingPath
-            : existingPath + '/';
-
-        // Find all files with the same base path that don't have arela_path yet
-        // Use pagination to handle cases with more than 1000 related files
-        let allRelatedFiles = [];
-        let from = 0;
-        const pageSize = 1000;
-        let hasMoreData = true;
-        let paginationError = null;
-
-        logger.info(`Searching for related files in base path: ${basePath}`);
-
-        while (hasMoreData) {
-          const { data: relatedFilesPage, error: relatedError } = await supabase
-            .from('uploader')
-            .select('id, filename, original_path')
-            .like('original_path', `${basePath}%`)
-            .is('arela_path', null)
-            .neq('id', pedimento.id) // Exclude the pedimento itself
-            .range(from, from + pageSize - 1);
+      // Add year filter if UPLOAD_YEARS is configured
+      if (appConfig.upload.years && appConfig.upload.years.length > 0) {
+        query = query.in('year', appConfig.upload.years);
+      }
 
-          if (relatedError) {
-            logger.error(
-              `Error finding related files for ${pedimento.filename}: ${relatedError.message}`,
-            );
-            paginationError = relatedError;
-            totalErrors++;
-            break;
-          }
+      const { data: pedimentoPage, error: pedimentoError } = await query
+        .range(offset, offset + pageSize - 1)
+        .order('created_at');
 
-          if (relatedFilesPage && relatedFilesPage.length > 0) {
-            allRelatedFiles = allRelatedFiles.concat(relatedFilesPage);
+      if (pedimentoError) {
+        const errorMsg = `Error fetching pedimento records page ${pageNumber}: ${pedimentoError.message}`;
+        logger.error(errorMsg);
+        throw new Error(errorMsg);
+      }
 
-            // Check if we got a full page, indicating there might be more data
-            if (relatedFilesPage.length < pageSize) {
-              hasMoreData = false;
-            } else {
-              from += pageSize;
-              logger.info(
-                `Fetched ${relatedFilesPage.length} files, continuing pagination (total so far: ${allRelatedFiles.length})`,
-              );
-            }
-          } else {
-            hasMoreData = false;
-          }
-        }
+      if (!pedimentoPage || pedimentoPage.length === 0) {
+        hasMoreData = false;
+        logger.info('No more pedimento records found');
+        break;
+      }
 
-        // Continue with error handling if pagination failed
-        if (paginationError) {
-          continue;
-        }
+      logger.info(
+        `Processing page ${pageNumber}: ${pedimentoPage.length} pedimento records`,
+      );
 
-        if (!allRelatedFiles || allRelatedFiles.length === 0) {
-          logger.info(`No related files found for ${pedimento.filename}`);
-          continue;
-        }
+      // Process each pedimento record in the current page
+      for (const pedimento of pedimentoPage) {
+        try {
+          totalProcessed++;
 
-        logger.info(
-          `Found ${allRelatedFiles.length} related files to update for ${pedimento.filename}`,
-        );
+          // Extract base path from original_path (remove filename)
+          const basePath = path.dirname(pedimento.original_path);
 
-        // Process files in batches
-        const fileIds = allRelatedFiles.map((f) => f.id);
+          logger.info(
+            `Processing pedimento: ${pedimento.filename} | Base path: ${basePath} | Year: ${pedimento.year || 'N/A'}`,
+          );
 
-        for (let i = 0; i < fileIds.length; i += BATCH_SIZE) {
-          const batchIds = fileIds.slice(i, i + BATCH_SIZE);
-          const batchNumber = Math.floor(i / BATCH_SIZE) + 1;
-          const totalBatches = Math.ceil(fileIds.length / BATCH_SIZE);
+          // Extract folder part from existing arela_path
+          const existingPath = pedimento.arela_path;
+          const folderArelaPath = existingPath.includes('/')
+            ? existingPath.substring(0, existingPath.lastIndexOf('/')) + '/'
+            : existingPath.endsWith('/')
+              ? existingPath
+              : existingPath + '/';
+
+          // Process related files page by page for memory efficiency
+          let relatedFilesFrom = 0;
+          const relatedFilesPageSize = 50;
+          let hasMoreRelatedFiles = true;
+          let relatedFilesPageNumber = 1;
+          let totalRelatedFilesProcessed = 0;
 
           logger.info(
-            `Batch ${batchNumber}/${totalBatches}: Updating ${batchIds.length} files...`,
+            `Searching and processing related files in base path: ${basePath}`,
           );
 
-          try {
-            const { error: updateError } = await supabase
-              .from('uploader')
-              .update({
-                arela_path: folderArelaPath,
-                year: pedimento.year,
-              })
-              .in('id', batchIds);
+          while (hasMoreRelatedFiles) {
+            const { data: relatedFilesPage, error: relatedError } =
+              await supabase
+                .from('uploader')
+                .select('id, filename, original_path')
+                .like('original_path', `${basePath}%`)
+                .is('arela_path', null)
+                .neq('id', pedimento.id) // Exclude the pedimento itself
+                .range(
+                  relatedFilesFrom,
+                  relatedFilesFrom + relatedFilesPageSize - 1,
+                );
 
-            if (updateError) {
+            if (relatedError) {
               logger.error(
-                `Error in batch ${batchNumber}: ${updateError.message}`,
+                `Error finding related files for ${pedimento.filename}: ${relatedError.message}`,
               );
               totalErrors++;
-            } else {
-              totalUpdated += batchIds.length;
+              break;
+            }
+            // console.log(`query by basePath: ${basePath} count: [${relatedFilesPage.length}]`);
+
+            if (!relatedFilesPage || relatedFilesPage.length === 0) {
+              hasMoreRelatedFiles = false;
+              if (totalRelatedFilesProcessed === 0) {
+                logger.info(`No related files found for ${pedimento.filename}`);
+              }
+              break;
+            }
+
+            logger.info(
+              `Processing related files page ${relatedFilesPageNumber}: ${relatedFilesPage.length} files for ${pedimento.filename}`,
+            );
+
+            // Track if any updates occurred in this page
+            let updatesOccurred = false;
+            // Process this page of related files in batches
+            const pageFileIds = relatedFilesPage.map((f) => f.id);
+
+            for (let i = 0; i < pageFileIds.length; i += BATCH_SIZE) {
+              const batchIds = pageFileIds.slice(i, i + BATCH_SIZE);
+              const batchNumber =
+                Math.floor(relatedFilesFrom / BATCH_SIZE) +
+                Math.floor(i / BATCH_SIZE) +
+                1;
+
               logger.info(
-                `Successfully updated batch ${batchNumber}: ${batchIds.length} files with arela_path and year`,
+                `Updating batch ${batchNumber}: ${batchIds.length} files with arela_path and year...`,
               );
+
+              try {
+                const { error: updateError } = await supabase
+                  .from('uploader')
+                  .update({
+                    arela_path: folderArelaPath,
+                    year: pedimento.year,
+                  })
+                  .in('id', batchIds);
+
+                if (updateError) {
+                  logger.error(
+                    `Error in batch ${batchNumber}: ${updateError.message}`,
+                  );
+                  totalErrors++;
+                } else {
+                  totalUpdated += batchIds.length;
+                  totalRelatedFilesProcessed += batchIds.length;
+                  updatesOccurred = true; // Mark that updates occurred
+                  logger.info(
+                    `Successfully updated batch ${batchNumber}: ${batchIds.length} files with arela_path and year`,
+                  );
+                }
+              } catch (batchError) {
+                logger.error(
+                  `Exception in batch ${batchNumber}: ${batchError.message}`,
+                );
+                totalErrors++;
+              }
+            }
+
+            // Check if we need to fetch the next page of related files
+            if (relatedFilesPage.length < relatedFilesPageSize) {
+              console.log('no more related files for ', basePath);
+              hasMoreRelatedFiles = false;
+              logger.info(
+                `Completed processing related files for ${pedimento.filename}. Total processed: ${totalRelatedFilesProcessed}`,
+              );
+            } else {
+              // If updates occurred, reset pagination to start from beginning
+              // since records that matched the query may no longer match after update
+              if (updatesOccurred) {
+                relatedFilesFrom = 0;
+                logger.info(
+                  `Page ${relatedFilesPageNumber} complete with updates: ${relatedFilesPage.length} files processed, restarting pagination from beginning due to query condition changes...`,
+                );
+              } else {
+                relatedFilesFrom += relatedFilesPageSize - 1;
+                logger.info(
+                  `Page ${relatedFilesPageNumber} complete: ${relatedFilesPage.length} files processed, continuing to next page...`,
+                );
+              }
+              relatedFilesPageNumber++;
             }
-          } catch (batchError) {
-            logger.error(
-              `Exception in batch ${batchNumber}: ${batchError.message}`,
-            );
-            totalErrors++;
           }
+        } catch (error) {
+          logger.error(
+            `Error processing pedimento ${pedimento.filename}: ${error.message}`,
+          );
+          totalErrors++;
         }
-      } catch (error) {
-        logger.error(
-          `Error processing pedimento ${pedimento.filename}: ${error.message}`,
+      }
+
+      // Check if we need to fetch the next page
+      if (pedimentoPage.length < pageSize) {
+        hasMoreData = false;
+        logger.info(
+          `Completed processing. Last page ${pageNumber} had ${pedimentoPage.length} records`,
+        );
+      } else {
+        offset += pageSize;
+        pageNumber++;
+        logger.info(
+          `Page ${pageNumber - 1} complete: ${pedimentoPage.length} records processed, moving to next page...`,
         );
-        totalErrors++;
       }
     }
 
+    // Final summary
+    if (totalProcessed === 0) {
+      logger.info('No pedimento_simplificado records with arela_path found');
+      console.log(
+        'ℹ️  No pedimento_simplificado records with arela_path found',
+      );
+    } else {
+      console.log(
+        `📋 Processed ${totalProcessed} pedimento records across ${pageNumber - 1} pages`,
+      );
+      logger.info(
+        `Processed ${totalProcessed} pedimento records across ${pageNumber - 1} pages`,
+      );
+    }
+
     const result = {
       processedCount: totalProcessed,
       updatedCount: totalUpdated,
@@ -773,25 +843,75 @@ export class DatabaseService {
       logger.info(`Total files for specified RFCs: ${totalRfcFiles || 0}`);
     }
 
-    // Step 1: Get all pedimento_simplificado records that match the specified RFCs and have arela_path
+    // Step 1: Get all pedimento_simplificado records that match the specified RFCs and have arela_path using pagination
     console.log(
       '🎯 Finding pedimento_simplificado records for specified RFCs...',
     );
-    const { data: pedimentoRfcRecords, error: pedimentoRfcError } =
-      await supabase
+
+    let allPedimentoRecords = [];
+    let offset = 0;
+    const pageSize = 500; // Process pedimento records in pages
+    let hasMorePedimentoData = true;
+    let pageNumber = 1;
+
+    // Process pedimento records page by page for memory efficiency
+    while (hasMorePedimentoData) {
+      logger.info(
+        `Fetching pedimento records page ${pageNumber} (records ${offset + 1} to ${offset + pageSize})...`,
+      );
+
+      let pedimentoQuery = supabase
         .from('uploader')
         .select('arela_path')
         .eq('document_type', 'pedimento_simplificado')
         .in('rfc', appConfig.upload.rfcs)
         .not('arela_path', 'is', null);
 
-    if (pedimentoRfcError) {
-      const errorMsg = `Error fetching pedimento RFC records: ${pedimentoRfcError.message}`;
-      logger.error(errorMsg);
-      throw new Error(errorMsg);
+      // Add year filter if UPLOAD_YEARS is configured
+      if (appConfig.upload.years && appConfig.upload.years.length > 0) {
+        pedimentoQuery = pedimentoQuery.in('year', appConfig.upload.years);
+      }
+
+      const { data: pedimentoPage, error: pedimentoError } =
+        await pedimentoQuery
+          .range(offset, offset + pageSize - 1)
+          .order('created_at');
+
+      if (pedimentoError) {
+        const errorMsg = `Error fetching pedimento RFC records page ${pageNumber}: ${pedimentoError.message}`;
+        logger.error(errorMsg);
+        throw new Error(errorMsg);
+      }
+
+      if (!pedimentoPage || pedimentoPage.length === 0) {
+        hasMorePedimentoData = false;
+        logger.info('No more pedimento records found');
+        break;
+      }
+
+      logger.info(
+        `Processing pedimento page ${pageNumber}: ${pedimentoPage.length} pedimento records`,
+      );
+
+      // Add this page to our collection
+      allPedimentoRecords = allPedimentoRecords.concat(pedimentoPage);
+
+      // Check if we need to fetch the next page
+      if (pedimentoPage.length < pageSize) {
+        hasMorePedimentoData = false;
+        logger.info(
+          `Completed fetching pedimento records. Last page ${pageNumber} had ${pedimentoPage.length} records`,
+        );
+      } else {
+        offset += pageSize;
+        pageNumber++;
+        logger.info(
+          `Page ${pageNumber - 1} complete: ${pedimentoPage.length} records fetched, moving to next page...`,
+        );
+      }
     }
 
-    if (!pedimentoRfcRecords || pedimentoRfcRecords.length === 0) {
+    if (!allPedimentoRecords || allPedimentoRecords.length === 0) {
       console.log(
         'ℹ️  No pedimento_simplificado records found for the specified RFCs with arela_path',
       );
@@ -799,15 +919,15 @@ export class DatabaseService {
       return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
     }
 
-    // Get unique arela_paths from pedimento records
+    // Get unique arela_paths from all pedimento records
     const uniqueArelaPaths = [
-      ...new Set(pedimentoRfcRecords.map((r) => r.arela_path)),
+      ...new Set(allPedimentoRecords.map((r) => r.arela_path)),
     ];
     console.log(
-      `📋 Found ${pedimentoRfcRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths for specified RFCs`,
+      `📋 Found ${allPedimentoRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths for specified RFCs across ${pageNumber - 1} pages`,
     );
     logger.info(
-      `Found ${pedimentoRfcRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths`,
+      `Found ${allPedimentoRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths across ${pageNumber - 1} pages`,
     );
 
     // Step 2: Get all files with these arela_paths that haven't been uploaded yet
@@ -872,16 +992,24 @@ export class DatabaseService {
       `Found ${rfcRecords.length} files ready for upload, ${skipped} skipped`,
     );
 
-    // Step 3: Get ALL files that have these arela_paths (including supporting documents)
-    let allRelatedFiles = [];
+    // Step 3: Process files with streaming pagination to avoid memory overload
+    let totalProcessed = 0;
+    let totalUploaded = 0;
+    let totalErrors = 0;
+    let globalFileCount = 0;
     const arelaPathChunkSize = 50;
-    const queryBatchSize = 1000;
+    const queryBatchSize = 500; // Reduced batch size for better memory management
+    const batchSize = parseInt(options.batchSize) || 10;
+
+    // Import performance configuration
+    const { performance: perfConfig } = appConfig;
+    const maxConcurrency = perfConfig?.maxApiConnections || 3;
 
     console.log(
-      '📥 Fetching all related files (processing arela_paths in chunks to avoid URI limits)...',
+      '📥 Processing files with streaming pagination (processing arela_paths in chunks to avoid URI limits and memory overload)...',
     );
 
-    // Process arela_paths in chunks
+    // Process arela_paths in chunks and upload files as we fetch them
     for (let i = 0; i < uploadableArelaPaths.length; i += arelaPathChunkSize) {
       const arelaPathChunk = uploadableArelaPaths.slice(
         i,
@@ -896,9 +1024,10 @@ export class DatabaseService {
         `   Processing arela_path chunk ${chunkNumber}/${totalChunks} (${arelaPathChunk.length} paths)`,
       );
 
-      // For each chunk of arela_paths, use pagination to get all related files
+      // For each chunk of arela_paths, use pagination to get related files and process them immediately
       let hasMore = true;
       let offset = 0;
+      let chunkFileCount = 0;
 
       while (hasMore) {
         const { data: batch, error: queryError } = await supabase
@@ -907,7 +1036,8 @@ export class DatabaseService {
           .in('arela_path', arelaPathChunk)
           .not('original_path', 'is', null)
           .neq('status', 'file-uploaded')
-          .range(offset, offset + queryBatchSize - 1);
+          .range(offset, offset + queryBatchSize - 1)
+          .order('created_at');
 
         if (queryError) {
           const errorMsg = `Error fetching related files for chunk ${chunkNumber}: ${queryError.message}`;
@@ -915,74 +1045,94 @@ export class DatabaseService {
           throw new Error(errorMsg);
         }
 
-        if (batch && batch.length > 0) {
-          allRelatedFiles = allRelatedFiles.concat(batch);
+        if (!batch || batch.length === 0) {
+          hasMore = false;
+          break;
         }
 
-        hasMore = batch && batch.length === queryBatchSize;
-        offset += queryBatchSize;
-      }
-    }
+        chunkFileCount += batch.length;
+        globalFileCount += batch.length;
 
-    if (!allRelatedFiles || allRelatedFiles.length === 0) {
-      console.log('ℹ️  No related files found to upload');
-      logger.info('No related files found to upload');
-      return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
-    }
+        console.log(
+          `   📦 Processing batch within chunk ${chunkNumber}: ${batch.length} files (total processed so far: ${globalFileCount})`,
+        );
 
-    console.log(`📋 Total files to upload: ${allRelatedFiles.length}`);
-    logger.info(`Total files to upload: ${allRelatedFiles.length}`);
+        // Track if any uploads occurred in this batch
+        let uploadsOccurred = false;
 
-    // Step 4: Upload all related files using concurrent batch processing
-    let totalProcessed = 0;
-    let totalUploaded = 0;
-    let totalErrors = 0;
-    const batchSize = parseInt(options.batchSize) || 10;
+        // Process this batch of files immediately using concurrent processing
+        // Split batch into upload batches
+        for (let j = 0; j < batch.length; j += batchSize) {
+          const uploadBatch = batch.slice(j, j + batchSize);
+          const batchNum = Math.floor(globalFileCount / batchSize) + 1;
 
-    // Import performance configuration
-    const { performance: perfConfig } = appConfig;
-    const maxConcurrency = perfConfig?.maxApiConnections || 3;
+          console.log(
+            `📦 Processing upload batch ${batchNum} (${uploadBatch.length} files)`,
+          );
 
-    console.log(
-      `🚀 Starting batch upload: ${allRelatedFiles.length} files in batches of ${batchSize}`,
-    );
-    console.log(
-      `⚡ Concurrent processing: up to ${maxConcurrency} parallel operations`,
-    );
+          // Process batch using concurrent processing similar to UploadCommand
+          const batchResults = await this.#processRfcBatch(
+            uploadBatch,
+            uploadService,
+            supabase,
+            options,
+            maxConcurrency,
+          );
 
-    // Process files in batches with concurrent processing
-    for (let i = 0; i < allRelatedFiles.length; i += batchSize) {
-      const batch = allRelatedFiles.slice(i, i + batchSize);
-      const batchNum = Math.floor(i / batchSize) + 1;
-      const totalBatches = Math.ceil(allRelatedFiles.length / batchSize);
+          totalProcessed += batchResults.processed;
+          totalUploaded += batchResults.uploaded;
+          totalErrors += batchResults.errors;
 
-      console.log(
-        `📦 Processing batch ${batchNum}/${totalBatches} (${batch.length} files)`,
-      );
+          // Track if uploads occurred (status changes from non-uploaded to uploaded)
+          if (batchResults.uploaded > 0) {
+            uploadsOccurred = true;
+          }
 
-      // Process batch using concurrent processing similar to UploadCommand
-      const batchResults = await this.#processRfcBatch(
-        batch,
-        uploadService,
-        supabase,
-        options,
-        maxConcurrency,
-      );
+          console.log(
+            `📊 Upload batch complete - Total progress: ${totalUploaded} uploaded, ${totalErrors} errors`,
+          );
+        }
 
-      totalProcessed += batchResults.processed;
-      totalUploaded += batchResults.uploaded;
-      totalErrors += batchResults.errors;
+        // Check if we need more data from this chunk
+        if (batch.length < queryBatchSize) {
+          hasMore = false;
+        } else {
+          // If uploads occurred, reset pagination to start from beginning
+          // since records that matched the query may no longer match after upload
+          if (uploadsOccurred) {
+            offset = 0;
+            console.log(
+              `   📄 Batch complete with uploads: ${batch.length} files processed, restarting pagination from beginning due to query condition changes...`,
+            );
+          } else {
+            offset += queryBatchSize;
+            console.log(
+              `   📄 Batch complete: ${batch.length} files processed, continuing to next page (offset: ${offset})...`,
+            );
+          }
+        }
+
+        if (hasMore) {
+          console.log(
+            `   📄 Fetching more files for chunk ${chunkNumber}... (offset: ${offset})`,
+          );
+        }
+      }
 
-      // Progress update
-      const progress = (
-        ((i + batch.length) / allRelatedFiles.length) *
-        100
-      ).toFixed(1);
       console.log(
-        `📊 Batch ${batchNum} complete - Progress: ${progress}% (${totalUploaded}/${allRelatedFiles.length} uploaded)`,
+        `   ✅ Chunk ${chunkNumber}/${totalChunks} complete: ${chunkFileCount} files processed`,
       );
     }
 
+    if (globalFileCount === 0) {
+      console.log('ℹ️  No related files found to upload');
+      logger.info('No related files found to upload');
+      return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
+    }
+
+    console.log(`📋 Total files processed: ${globalFileCount}`);
+    logger.info(`Total files processed: ${globalFileCount}`);
+
     const result = {
       processedCount: totalProcessed,
       uploadedCount: totalUploaded,
@@ -1037,13 +1187,25 @@ export class DatabaseService {
     console.log(
       '🎯 Finding pedimento_simplificado documents for specified RFCs with arela_path...',
     );
-    const { data: pedimentoRecords, error: pedimentoError } = await supabase
+
+    let pedimentoReadyQuery = supabase
       .from('uploader')
       .select('arela_path')
       .eq('document_type', 'pedimento_simplificado')
       .in('rfc', uploadRfcs)
       .not('arela_path', 'is', null);
 
+    // Add year filter if UPLOAD_YEARS is configured
+    if (appConfig.upload.years && appConfig.upload.years.length > 0) {
+      pedimentoReadyQuery = pedimentoReadyQuery.in(
+        'year',
+        appConfig.upload.years,
+      );
+    }
+
+    const { data: pedimentoRecords, error: pedimentoError } =
+      await pedimentoReadyQuery;
+
     if (pedimentoError) {
       throw new Error(
         `Error querying pedimento_simplificado records: ${pedimentoError.message}`,