bulltrackers-module 1.0.500 → 1.0.501

package/functions/task-engine/utils/task_engine_utils.js

@@ -297,32 +297,59 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
 
     // 6. For batch processing (global cron route), check if we need to run root data indexer
     // This handles the edge case where no on-demand requests were made
+    // NOTE: We add a small delay to ensure all Firestore counter updates have propagated
     if (targetDate && (batchCounterRef || socialCounterRef)) {
-        // Check both counters to see if all tasks are complete
-        let allComplete = true;
+        // Wait a moment for Firestore writes to propagate
+        await new Promise(resolve => setTimeout(resolve, 2000));
+        
+        // Check both counters to see if all tasks are complete (with retry logic)
+        let allComplete = false;
         let allTasks = [...tasksToRun, ...otherTasks, ...socialTasks];
+        const maxRetries = 3;
+        let retries = 0;
         
-        if (batchCounterRef) {
-            const batchCounterDoc = await batchCounterRef.get();
-            if (batchCounterDoc.exists) {
-                const batchData = batchCounterDoc.data();
-                if (batchData.remainingTasks > 0) {
+        while (retries < maxRetries && !allComplete) {
+            allComplete = true;
+            
+            if (batchCounterRef) {
+                const batchCounterDoc = await batchCounterRef.get();
+                if (batchCounterDoc.exists) {
+                    const batchData = batchCounterDoc.data();
+                    logger.log('INFO', `[TaskEngine/${taskId}] Batch counter check (attempt ${retries + 1}): remainingTasks=${batchData.remainingTasks}, totalTasks=${batchData.totalTasks}, failedTasks=${batchData.failedTasks || 0}`);
+                    if (batchData.remainingTasks > 0) {
+                        allComplete = false;
+                    }
+                } else {
+                    logger.log('WARN', `[TaskEngine/${taskId}] Batch counter document does not exist`);
                     allComplete = false;
                 }
             }
-        }
-        
-        if (socialCounterRef) {
-            const socialCounterDoc = await socialCounterRef.get();
-            if (socialCounterDoc.exists) {
-                const socialData = socialCounterDoc.data();
-                if (socialData.remainingTasks > 0) {
+            
+            if (socialCounterRef) {
+                const socialCounterDoc = await socialCounterRef.get();
+                if (socialCounterDoc.exists) {
+                    const socialData = socialCounterDoc.data();
+                    logger.log('INFO', `[TaskEngine/${taskId}] Social counter check (attempt ${retries + 1}): remainingTasks=${socialData.remainingTasks}, totalTasks=${socialData.totalTasks}, failedTasks=${socialData.failedTasks || 0}`);
+                    if (socialData.remainingTasks > 0) {
+                        allComplete = false;
+                    }
+                } else {
+                    logger.log('WARN', `[TaskEngine/${taskId}] Social counter document does not exist`);
                     allComplete = false;
                 }
             }
+            
+            if (!allComplete && retries < maxRetries - 1) {
+                // Wait before retrying (exponential backoff)
+                await new Promise(resolve => setTimeout(resolve, 1000 * (retries + 1)));
+                retries++;
+            } else {
+                break;
+            }
         }
         
         if (allComplete) {
+            logger.log('INFO', `[TaskEngine/${taskId}] All tasks complete. Triggering root data indexer for ${targetDate}...`);
             // All tasks complete - conditionally trigger root data indexer
             const dataTypesRun = determineDataTypesRun(allTasks);
             
@@ -338,6 +365,8 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
                 counterRef: primaryCounterRef,
                 dataTypesRun
             });
+        } else {
+            logger.log('WARN', `[TaskEngine/${taskId}] Not all tasks complete after ${maxRetries} checks. Remaining tasks may still be processing. Root data indexer will be triggered by the next task to complete or by the global cron.`);
         }
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.500",
+  "version": "1.0.501",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [