bulltrackers-module 1.0.499 → 1.0.501

package/functions/root-data-indexer/index.js

@@ -95,12 +95,23 @@ exports.runRootDataIndexer = async (config, dependencies) => {
     logger.log('INFO', `[RootDataIndexer] Starting Root Data Availability Scan... Mode: ${scanMode}`, { targetDate });
     
     // 1. Price Availability - Read from date tracking documents
-    // Find the latest pricedatastoreddates document and extract available dates
+    // Find the latest price tracking document and extract available dates
     const priceAvailabilitySet = new Set();
     
+    // Get price tracking collection name from registry if available
+    let priceTrackingCollectionName = 'pricedatastoreddates';
+    if (dependencies.collectionRegistry && dependencies.collectionRegistry.getCollectionPath) {
+        try {
+            const trackingPath = dependencies.collectionRegistry.getCollectionPath('rootData', 'priceTracking', { fetchDate: '2025-01-01' });
+            priceTrackingCollectionName = trackingPath.split('/')[0];
+        } catch (e) {
+            logger.log('WARN', `[RootDataIndexer] Failed to get price tracking collection from registry, using default: ${e.message}`);
+        }
+    }
+    
     try {
         // Get the latest price date tracking document
-        const dateTrackingRef = db.collection('pricedatastoreddates')
+        const dateTrackingRef = db.collection(priceTrackingCollectionName)
             .orderBy('fetchDate', 'desc')
             .limit(1);
         
@@ -109,6 +120,7 @@ exports.runRootDataIndexer = async (config, dependencies) => {
         if (!dateTrackingSnapshot.empty) {
             const latestTrackingDoc = dateTrackingSnapshot.docs[0].data();
             const datesAvailable = latestTrackingDoc.datesAvailable || [];
+            const fetchDate = latestTrackingDoc.fetchDate;
             
             // Add all dates from the tracking document
             datesAvailable.forEach(dateKey => {
@@ -117,7 +129,22 @@ exports.runRootDataIndexer = async (config, dependencies) => {
                 }
             });
             
-            logger.log('INFO', `[RootDataIndexer] Loaded ${priceAvailabilitySet.size} price dates from tracking document (fetchDate: ${latestTrackingDoc.fetchDate})`);
+            // IMPORTANT: If the tracking document was written for today (fetchDate matches targetDate),
+            // we should consider prices available for that date even if the API didn't return that exact date.
+            // This is because the price fetcher ran for that date and stored prices (even if they're historical).
+            if (targetDate && fetchDate === targetDate) {
+                priceAvailabilitySet.add(targetDate);
+                logger.log('INFO', `[RootDataIndexer] Added fetchDate (${fetchDate}) to price availability set for target date check`);
+            }
+            
+            logger.log('INFO', `[RootDataIndexer] Loaded ${priceAvailabilitySet.size} price dates from tracking document (fetchDate: ${fetchDate})`);
+            
+            // Debug: Log a sample of dates and check if target date is present
+            if (targetDate) {
+                const sampleDates = Array.from(priceAvailabilitySet).slice(0, 5);
+                const hasTargetDate = priceAvailabilitySet.has(targetDate);
+                logger.log('INFO', `[RootDataIndexer] Price availability check for ${targetDate}: ${hasTargetDate ? 'FOUND' : 'NOT FOUND'}. Sample dates: ${sampleDates.join(', ')}`);
+            }
         } else {
             logger.log('WARN', '[RootDataIndexer] No price date tracking documents found. Falling back to empty set.');
         }
@@ -395,17 +422,13 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             availability.hasSocial    = foundPISocial || foundSignedInSocial || genericSocialExists; 
             
             // Price Check
-            if (targetDate) {
-                // In single-date mode, we do a quick check on the shard if we don't have the full set loaded
-                // For performance, we default hasPrices to false unless we implement a specific single-date shard check.
-                // However, since this is usually called right after fetching, we can leave it as a secondary concern 
-                // or assume the Global Sync will catch it later. 
-                // OPTIONAL: Implementation of specific shard check for single date:
-                // For now, we reuse the set if available (it won't be) or default false.
-                // NOTE: The Global Scan at 01:00 UTC will strictly verify prices.
-                availability.hasPrices = priceAvailabilitySet.has(dateStr); 
-            } else {
-                availability.hasPrices = priceAvailabilitySet.has(dateStr);
+            // Check if the target date exists in the price availability set
+            // The set is populated from the price tracking document's datesAvailable array
+            const hasPriceForDate = priceAvailabilitySet.has(dateStr);
+            availability.hasPrices = hasPriceForDate;
+            
+            if (targetDate && !hasPriceForDate) {
+                logger.log('WARN', `[RootDataIndexer/${dateStr}] Price data not found in tracking document. Set size: ${priceAvailabilitySet.size}, Date checked: ${dateStr}`);
             }
 
             await db.collection(availabilityCollection).doc(dateStr).set(availability);

package/functions/task-engine/utils/task_engine_utils.js

@@ -297,32 +297,59 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
 
     // 6. For batch processing (global cron route), check if we need to run root data indexer
     // This handles the edge case where no on-demand requests were made
+    // NOTE: We add a small delay to ensure all Firestore counter updates have propagated
     if (targetDate && (batchCounterRef || socialCounterRef)) {
-        // Check both counters to see if all tasks are complete
-        let allComplete = true;
+        // Wait a moment for Firestore writes to propagate
+        await new Promise(resolve => setTimeout(resolve, 2000));
+        
+        // Check both counters to see if all tasks are complete (with retry logic)
+        let allComplete = false;
         let allTasks = [...tasksToRun, ...otherTasks, ...socialTasks];
+        const maxRetries = 3;
+        let retries = 0;
         
-        if (batchCounterRef) {
-            const batchCounterDoc = await batchCounterRef.get();
-            if (batchCounterDoc.exists) {
-                const batchData = batchCounterDoc.data();
-                if (batchData.remainingTasks > 0) {
+        while (retries < maxRetries && !allComplete) {
+            allComplete = true;
+            
+            if (batchCounterRef) {
+                const batchCounterDoc = await batchCounterRef.get();
+                if (batchCounterDoc.exists) {
+                    const batchData = batchCounterDoc.data();
+                    logger.log('INFO', `[TaskEngine/${taskId}] Batch counter check (attempt ${retries + 1}): remainingTasks=${batchData.remainingTasks}, totalTasks=${batchData.totalTasks}, failedTasks=${batchData.failedTasks || 0}`);
+                    if (batchData.remainingTasks > 0) {
+                        allComplete = false;
+                    }
+                } else {
+                    logger.log('WARN', `[TaskEngine/${taskId}] Batch counter document does not exist`);
                     allComplete = false;
                 }
             }
-        }
-        
-        if (socialCounterRef) {
-            const socialCounterDoc = await socialCounterRef.get();
-            if (socialCounterDoc.exists) {
-                const socialData = socialCounterDoc.data();
-                if (socialData.remainingTasks > 0) {
+            
+            if (socialCounterRef) {
+                const socialCounterDoc = await socialCounterRef.get();
+                if (socialCounterDoc.exists) {
+                    const socialData = socialCounterDoc.data();
+                    logger.log('INFO', `[TaskEngine/${taskId}] Social counter check (attempt ${retries + 1}): remainingTasks=${socialData.remainingTasks}, totalTasks=${socialData.totalTasks}, failedTasks=${socialData.failedTasks || 0}`);
+                    if (socialData.remainingTasks > 0) {
+                        allComplete = false;
+                    }
+                } else {
+                    logger.log('WARN', `[TaskEngine/${taskId}] Social counter document does not exist`);
                     allComplete = false;
                 }
             }
+            
+            if (!allComplete && retries < maxRetries - 1) {
+                // Wait before retrying (exponential backoff)
+                await new Promise(resolve => setTimeout(resolve, 1000 * (retries + 1)));
+                retries++;
+            } else {
+                break;
+            }
         }
         
         if (allComplete) {
+            logger.log('INFO', `[TaskEngine/${taskId}] All tasks complete. Triggering root data indexer for ${targetDate}...`);
             // All tasks complete - conditionally trigger root data indexer
             const dataTypesRun = determineDataTypesRun(allTasks);
             
@@ -338,6 +365,8 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
                 counterRef: primaryCounterRef,
                 dataTypesRun
             });
+        } else {
+            logger.log('WARN', `[TaskEngine/${taskId}] Not all tasks complete after ${maxRetries} checks. Remaining tasks may still be processing. Root data indexer will be triggered by the next task to complete or by the global cron.`);
         }
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.499",
+  "version": "1.0.501",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [