bulltrackers-module 1.0.468 → 1.0.470

package/functions/core/utils/firestore_utils.js

@@ -397,10 +397,42 @@ async function getSignedInUsersToUpdate(dependencies, config) {
             targets.push({ cid: String(cid), username });
         });
         
-        // Now filter out users who were updated in the last 18 hours
-        // Check user_sync_requests for last completed sync
+        // Now filter out users who were updated today
+        // Check actual portfolio/history snapshot dates, not just sync request timestamps
         const filteredTargets = [];
+        const today = new Date().toISOString().split('T')[0];
         const checkPromises = targets.map(async (target) => {
+            // Check if user has portfolio or history data for today
+            const blockId = `${Math.floor(parseInt(target.cid) / 1000000)}M`;
+            
+            // Check portfolio snapshot for today
+            const portfolioSnapshotRef = db.collection(collectionName)
+                .doc(blockId)
+                .collection('snapshots')
+                .doc(today)
+                .collection('parts')
+                .limit(1);
+            
+            const portfolioSnapshot = await portfolioSnapshotRef.get();
+            
+            // Check history snapshot for today
+            const historyCollection = config.signedInUserHistoryCollection || process.env.FIRESTORE_COLLECTION_SIGNED_IN_USER_HISTORY || 'signed_in_user_history';
+            const historySnapshotRef = db.collection(historyCollection)
+                .doc(blockId)
+                .collection('snapshots')
+                .doc(today)
+                .collection('parts')
+                .limit(1);
+            
+            const historySnapshot = await historySnapshotRef.get();
+            
+            // If user has data for today, skip them (they're already up-to-date)
+            if (!portfolioSnapshot.empty || !historySnapshot.empty) {
+                skippedCount++;
+                return null; // Skip this user - already updated today
+            }
+            
+            // Also check sync requests as a fallback (in case data exists but snapshot check failed)
             const syncRef = db.collection('user_sync_requests')
                 .doc(target.cid)
                 .collection('global')
@@ -418,7 +450,7 @@ async function getSignedInUsersToUpdate(dependencies, config) {
                 }
             }
             
-            return target; // Include this user
+            return target; // Include this user - needs update
         });
         
         const results = await Promise.all(checkPromises);

package/functions/etoro-price-fetcher/helpers/handler_helpers.js

@@ -49,6 +49,51 @@ exports.fetchAndStorePrices = async (config, dependencies) => {
         const batchPromises = [];
         for (const shardId in shardUpdates) { const docRef = db.collection(priceCollectionName).doc(shardId); const payload = shardUpdates[shardId]; batchPromises.push(docRef.update(payload)); }
         await Promise.all(batchPromises);
+        
+        // Extract all dates from the price data and create a date tracking document
+        const priceDatesSet = new Set();
+        const AUGUST_2025 = '2025-08-01';
+        
+        for (const instrumentData of results) {
+            const dailyData = instrumentData?.ClosingPrices?.Daily;
+            const weeklyData = instrumentData?.ClosingPrices?.Weekly;
+            const monthlyData = instrumentData?.ClosingPrices?.Monthly;
+            
+            if (dailyData?.Date) {
+                const dateKey = dailyData.Date.substring(0, 10);
+                if (dateKey >= AUGUST_2025) {
+                    priceDatesSet.add(dateKey);
+                }
+            }
+            if (weeklyData?.Date) {
+                const dateKey = weeklyData.Date.substring(0, 10);
+                if (dateKey >= AUGUST_2025) {
+                    priceDatesSet.add(dateKey);
+                }
+            }
+            if (monthlyData?.Date) {
+                const dateKey = monthlyData.Date.substring(0, 10);
+                if (dateKey >= AUGUST_2025) {
+                    priceDatesSet.add(dateKey);
+                }
+            }
+        }
+        
+        // Write date tracking document
+        const today = new Date().toISOString().split('T')[0];
+        const dateTrackingRef = db.collection('pricedatastoreddates').doc(today);
+        const priceDatesArray = Array.from(priceDatesSet).sort();
+        
+        await dateTrackingRef.set({
+            fetchDate: today,
+            datesAvailable: priceDatesArray,
+            dateCount: priceDatesArray.length,
+            instrumentsProcessed: results.length,
+            storedAt: FieldValue.serverTimestamp()
+        });
+        
+        logger.log('INFO', `[PriceFetcherHelpers] Wrote price date tracking document for ${today} with ${priceDatesArray.length} dates (from August 2025 onwards)`);
+        
         const successMessage = `Successfully processed and saved daily prices for ${results.length} instruments to ${batchPromises.length} shards.`;
         logger.log('SUCCESS', `[PriceFetcherHelpers] ${successMessage}`);
         return { success: true, message: successMessage, instrumentsProcessed: results.length };

package/functions/root-data-indexer/index.js

@@ -75,35 +75,61 @@ exports.runRootDataIndexer = async (config, dependencies) => {
     const scanMode = targetDate ? 'SINGLE_DATE' : 'FULL_SCAN';
     logger.log('INFO', `[RootDataIndexer] Starting Root Data Availability Scan... Mode: ${scanMode}`, { targetDate });
     
-    // 1. Price Availability (Global Scan Mode Only)
-    // If running for a single targetDate, we skip the global shard sampling to save time.
+    // 1. Price Availability - Read from date tracking documents
+    // Find the latest pricedatastoreddates document and extract available dates
     const priceAvailabilitySet = new Set();
     
-    if (!targetDate) {
-        try {
-            // Path: asset_prices/shard_*
-            const priceCollectionRef = db.collection(PRICE_COLLECTION_NAME);
-            const priceShardsSnapshot = await priceCollectionRef.limit(10).get();
+    try {
+        // Get the latest price date tracking document
+        const dateTrackingRef = db.collection('pricedatastoreddates')
+            .orderBy('fetchDate', 'desc')
+            .limit(1);
+        
+        const dateTrackingSnapshot = await dateTrackingRef.get();
+        
+        if (!dateTrackingSnapshot.empty) {
+            const latestTrackingDoc = dateTrackingSnapshot.docs[0].data();
+            const datesAvailable = latestTrackingDoc.datesAvailable || [];
+            
+            // Add all dates from the tracking document
+            datesAvailable.forEach(dateKey => {
+                if (/^\d{4}-\d{2}-\d{2}$/.test(dateKey)) {
+                    priceAvailabilitySet.add(dateKey);
+                }
+            });
             
-            if (!priceShardsSnapshot.empty) {
-                // Sample up to 10 shards and extract date keys from them
-                for (const shardDoc of priceShardsSnapshot.docs) {
-                    if (shardDoc.id.startsWith('shard_')) {
-                        const data = shardDoc.data();
-                        Object.values(data).forEach(instrument => { 
-                            if (instrument && instrument.prices) { 
-                                Object.keys(instrument.prices).forEach(dateKey => { 
-                                    if (/^\d{4}-\d{2}-\d{2}$/.test(dateKey)) { 
-                                        priceAvailabilitySet.add(dateKey); 
-                                    } 
-                                }); 
-                            } 
-                        });
+            logger.log('INFO', `[RootDataIndexer] Loaded ${priceAvailabilitySet.size} price dates from tracking document (fetchDate: ${latestTrackingDoc.fetchDate})`);
+        } else {
+            logger.log('WARN', '[RootDataIndexer] No price date tracking documents found. Falling back to empty set.');
+        }
+    } catch (e) {
+        logger.log('ERROR', '[RootDataIndexer] Failed to load price date tracking document.', { error: e.message });
+        // Fallback: try to sample shards if tracking document fails
+        if (!targetDate) {
+            try {
+                const priceCollectionRef = db.collection(PRICE_COLLECTION_NAME);
+                const priceShardsSnapshot = await priceCollectionRef.limit(10).get();
+                
+                if (!priceShardsSnapshot.empty) {
+                    for (const shardDoc of priceShardsSnapshot.docs) {
+                        if (shardDoc.id.startsWith('shard_')) {
+                            const data = shardDoc.data();
+                            Object.values(data).forEach(instrument => { 
+                                if (instrument && instrument.prices) { 
+                                    Object.keys(instrument.prices).forEach(dateKey => { 
+                                        if (/^\d{4}-\d{2}-\d{2}$/.test(dateKey)) { 
+                                            priceAvailabilitySet.add(dateKey); 
+                                        } 
+                                    }); 
+                                } 
+                            });
+                        }
                     }
+                    logger.log('INFO', `[RootDataIndexer] Fallback: Loaded ${priceAvailabilitySet.size} price dates from shard sampling.`);
                 }
+            } catch (fallbackError) {
+                logger.log('ERROR', '[RootDataIndexer] Fallback shard sampling also failed.', { error: fallbackError.message });
             }
-        } catch (e) {
-            logger.log('ERROR', '[RootDataIndexer] Failed to sample price shards.', { error: e.message });
         }
     }
 
@@ -130,9 +156,9 @@ exports.runRootDataIndexer = async (config, dependencies) => {
     const promises = datesToScan.map(dateStr => limit(async () => {
         try {
             // Define Time Range for Social Query (Full Day UTC)
-            const dayStart = new Date(dateStr);
-            const dayEnd = new Date(dateStr);
-            dayEnd.setHours(23, 59, 59, 999);
+            // Use UTC methods to ensure correct timezone handling
+            const dayStart = new Date(dateStr + 'T00:00:00.000Z');
+            const dayEnd = new Date(dateStr + 'T23:59:59.999Z');
 
             const availability = {
                 date: dateStr,
@@ -211,10 +237,14 @@ exports.runRootDataIndexer = async (config, dependencies) => {
 
             // 4. Universal Social Check via Collection Group
             // Path: Collection Group 'posts' where 'fetchedAt' is within the day
+            // This queries ALL 'posts' subcollections across:
+            // - signed_in_users_social/{userId}/posts/{postId}
+            // - pi_social_posts/{userId}/posts/{postId}
+            // - daily_social_insights/{date}/posts/{postId}
             const universalSocialQuery = db.collectionGroup('posts')
                 .where('fetchedAt', '>=', dayStart)
                 .where('fetchedAt', '<=', dayEnd)
-                .limit(50); 
+                .limit(100); // Increased limit to ensure we catch all posts 
 
             // --- Execute Checks ---
             const [
@@ -249,18 +279,64 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             let foundPISocial = false;
             let foundSignedInSocial = false;
 
+            logger.log('INFO', `[RootDataIndexer/${dateStr}] Checking social data availability (dayStart: ${dayStart.toISOString()}, dayEnd: ${dayEnd.toISOString()})`);
+            logger.log('INFO', `[RootDataIndexer/${dateStr}] Looking for PI social in: "${PI_SOCIAL_COLL_NAME}", Signed-in social in: "${SIGNED_IN_SOCIAL_COLL_NAME}"`);
+
             if (!universalSocialSnap.empty) {
+                logger.log('INFO', `[RootDataIndexer/${dateStr}] Collection group query returned ${universalSocialSnap.docs.length} posts`);
+                
+                // Track all paths found for debugging
+                const allPaths = [];
+                const piPaths = [];
+                const signedInPaths = [];
+                const otherPaths = [];
+                
                 universalSocialSnap.docs.forEach(doc => {
                     const path = doc.ref.path;
-                    // Use includes() to match collection name anywhere in path (more robust)
+                    const data = doc.data();
+                    const fetchedAt = data.fetchedAt;
+                    allPaths.push(path);
+                    
+                    // Convert fetchedAt to string for logging
+                    let fetchedAtStr = 'missing';
+                    if (fetchedAt) {
+                        if (fetchedAt.toDate) {
+                            fetchedAtStr = fetchedAt.toDate().toISOString();
+                        } else if (fetchedAt.toMillis) {
+                            fetchedAtStr = new Date(fetchedAt.toMillis()).toISOString();
+                        } else {
+                            fetchedAtStr = String(fetchedAt);
+                        }
+                    }
+                    
+                    // Use includes() to match collection name anywhere in path
                     // Path format: {collectionName}/{userId}/posts/{postId}
-                    if (path.includes(`/${PI_SOCIAL_COLL_NAME}/`) || path.startsWith(`${PI_SOCIAL_COLL_NAME}/`)) {
+                    const piMatchPattern = `${PI_SOCIAL_COLL_NAME}/`;
+                    const signedInMatchPattern = `${SIGNED_IN_SOCIAL_COLL_NAME}/`;
+                    
+                    if (path.includes(piMatchPattern)) {
                         foundPISocial = true;
-                    }
-                    if (path.includes(`/${SIGNED_IN_SOCIAL_COLL_NAME}/`) || path.startsWith(`${SIGNED_IN_SOCIAL_COLL_NAME}/`)) {
+                        piPaths.push({ path, fetchedAt: fetchedAtStr });
+                        logger.log('INFO', `[RootDataIndexer/${dateStr}] ✓ PI social: ${path} (fetchedAt: ${fetchedAtStr})`);
+                    } else if (path.includes(signedInMatchPattern)) {
                         foundSignedInSocial = true;
+                        signedInPaths.push({ path, fetchedAt: fetchedAtStr });
+                        logger.log('INFO', `[RootDataIndexer/${dateStr}] ✓ Signed-in social: ${path} (fetchedAt: ${fetchedAtStr})`);
+                    } else {
+                        otherPaths.push({ path, fetchedAt: fetchedAtStr });
+                        logger.log('INFO', `[RootDataIndexer/${dateStr}] ? Other social: ${path} (fetchedAt: ${fetchedAtStr})`);
                     }
                 });
+                
+                logger.log('INFO', `[RootDataIndexer/${dateStr}] Summary - Total: ${allPaths.length}, PI: ${piPaths.length}, Signed-in: ${signedInPaths.length}, Other: ${otherPaths.length}`);
+                
+                if (signedInPaths.length === 0 && !foundSignedInSocial) {
+                    logger.log('WARN', `[RootDataIndexer/${dateStr}] ⚠️ No signed-in social posts found! Expected pattern: "${signedInMatchPattern}"`);
+                    logger.log('WARN', `[RootDataIndexer/${dateStr}] All paths found: ${allPaths.slice(0, 10).join(', ')}${allPaths.length > 10 ? '...' : ''}`);
+                }
+            } else {
+                logger.log('WARN', `[RootDataIndexer/${dateStr}] ⚠️ Collection group query returned NO posts! (dayStart: ${dayStart.toISOString()}, dayEnd: ${dayEnd.toISOString()})`);
+                logger.log('WARN', `[RootDataIndexer/${dateStr}] This might indicate: 1) No posts with fetchedAt in this range, 2) Missing Firestore index, or 3) Query error`);
             }
 
             // --- Assign to Availability ---

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.468",
+  "version": "1.0.470",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [