bulltrackers-module 1.0.514 → 1.0.515

package/functions/core/utils/firestore_utils.js

@@ -256,52 +256,85 @@ async function findLatestRankingsDate(db, rankingsCollection, maxDaysBack = 30)
 
 /**
  * [NEW] Fetches Popular Investors from the daily rankings document.
- * UPDATED: Uses latest available rankings date if today's doesn't exist (fallback for edge cases).
- * This ensures the dispatcher can still queue tasks even if today's rankings haven't been fetched yet.
+ * UPDATED: Checks last 7 days of rankings data and builds a de-duplicated map.
+ * This solves the problem of a single day where a given popular investor is not in the rankings.
+ * UPDATED: Uses collection registry for path resolution.
  */
 async function getPopularInvestorsToUpdate(dependencies, config) {
-    const { db, logger } = dependencies;
-    const collectionName = config.popularInvestorRankingsCollection || process.env.FIRESTORE_COLLECTION_PI_RANKINGS || 'popular_investor_rankings';
+    const { db, logger, collectionRegistry } = dependencies;
+    
+    // Get collection name from registry if available
+    let collectionName = config.popularInvestorRankingsCollection || process.env.FIRESTORE_COLLECTION_PI_RANKINGS || 'popular_investor_rankings';
+    
+    if (collectionRegistry && collectionRegistry.getCollectionPath) {
+        try {
+            // Extract collection name from registry path: popular_investor_rankings/{date}
+            const samplePath = collectionRegistry.getCollectionPath('rootData', 'popularInvestorRankings', { date: '2025-01-01' });
+            collectionName = samplePath.split('/')[0];
+        } catch (err) {
+            logger.log('WARN', `[Core Utils] Failed to get collection from registry, using config: ${err.message}`);
+        }
+    }
 
-    logger.log('INFO', `[Core Utils] Getting Popular Investors to update from ${collectionName}...`);
+    logger.log('INFO', `[Core Utils] Getting Popular Investors to update from ${collectionName} (checking last 7 days)...`);
     
-    // Try today's date first
-    const today = new Date().toISOString().split('T')[0];
-    let rankingsDate = today;
+    // Check last 7 days of rankings data
+    const today = new Date();
+    const piMap = new Map(); // De-duplicated map: cid -> { cid, username, latestDate }
     
     try {
-        let docRef = db.collection(collectionName).doc(today);
-        let docSnap = await docRef.get();
-
-        // If today's rankings don't exist, fallback to latest available date
-        if (!docSnap.exists) {
-            logger.log('WARN', `[Core Utils] No Popular Investor rankings found for date: ${today}. Looking for latest available date...`);
-            rankingsDate = await findLatestRankingsDate(db, collectionName, 30);
-            
-            if (!rankingsDate) {
-                logger.log('WARN', `[Core Utils] No Popular Investor rankings found in last 30 days. Returning empty array.`);
-                return [];
-            }
-            
-            logger.log('INFO', `[Core Utils] Using fallback rankings date: ${rankingsDate} (today: ${today})`);
-            docRef = db.collection(collectionName).doc(rankingsDate);
-            docSnap = await docRef.get();
+        // Check each of the last 7 days
+        for (let i = 0; i < 7; i++) {
+            const checkDate = new Date(today);
+            checkDate.setDate(checkDate.getDate() - i);
+            const dateStr = checkDate.toISOString().split('T')[0];
             
-            if (!docSnap.exists) {
-                logger.log('WARN', `[Core Utils] Rankings doc does not exist for fallback date ${rankingsDate}`);
-                return [];
+            try {
+                const docRef = db.collection(collectionName).doc(dateStr);
+                const docSnap = await docRef.get();
+                
+                if (docSnap.exists) {
+                    const data = docSnap.data();
+                    const items = data.Items || [];
+                    
+                    // Add all PIs from this date to the map (de-duplicated by CID)
+                    for (const item of items) {
+                        const cid = String(item.CustomerId);
+                        const username = item.UserName;
+                        
+                        // Only add if not already in map, or if this date is more recent
+                        if (!piMap.has(cid)) {
+                            piMap.set(cid, {
+                                cid: cid,
+                                username: username,
+                                latestDate: dateStr
+                            });
+                        } else {
+                            // Update if this date is more recent
+                            const existing = piMap.get(cid);
+                            if (dateStr > existing.latestDate) {
+                                existing.latestDate = dateStr;
+                                if (username) existing.username = username;
+                            }
+                        }
+                    }
+                }
+            } catch (dateErr) {
+                logger.log('WARN', `[Core Utils] Error checking rankings for date ${dateStr}: ${dateErr.message}`);
+                continue; // Continue to next date
             }
         }
+        
+        if (piMap.size === 0) {
+            logger.log('WARN', `[Core Utils] No Popular Investor rankings found in last 7 days. Returning empty array.`);
+            return [];
+        }
+        
+        logger.log('INFO', `[Core Utils] Found ${piMap.size} unique Popular Investors across last 7 days of rankings.`);
 
-        const data = docSnap.data();
-        const items = data.Items || [];
-
-        // Map to { cid, username }
-        const allTargets = items.map(item => ({
-            cid: String(item.CustomerId),
-            username: item.UserName
-        }));
-
+        // Convert map to array and filter out PIs updated in the last 18 hours
+        const allTargets = Array.from(piMap.values());
+        
         // Filter out PIs updated in the last 18 hours
         const HOURS_THRESHOLD = 18;
         const thresholdMs = HOURS_THRESHOLD * 60 * 60 * 1000;
@@ -310,12 +343,29 @@ async function getPopularInvestorsToUpdate(dependencies, config) {
         const filteredTargets = [];
         let skippedCount = 0;
         
-        // Check each PI's last sync time
+        // Check each PI's last sync time using user-centric path
         for (const target of allTargets) {
-            const syncRef = db.collection('user_sync_requests')
-                .doc(target.cid)
-                .collection('global')
-                .doc('latest');
+            // Use collection registry for sync status path if available
+            let syncRef;
+            if (collectionRegistry && collectionRegistry.getCollectionPath) {
+                try {
+                    // syncStatus path is: SignedInUsers/{cid}/syncStatus/latest (document path)
+                    const syncStatusPath = collectionRegistry.getCollectionPath('signedInUsers', 'syncStatus', { cid: target.cid });
+                    syncRef = db.doc(syncStatusPath);
+                } catch (err) {
+                    // Fallback to legacy path
+                    syncRef = db.collection('user_sync_requests')
+                        .doc(target.cid)
+                        .collection('global')
+                        .doc('latest');
+                }
+            } else {
+                // Fallback to legacy path
+                syncRef = db.collection('user_sync_requests')
+                    .doc(target.cid)
+                    .collection('global')
+                    .doc('latest');
+            }
             
             const syncDoc = await syncRef.get();
             if (syncDoc.exists) {
@@ -336,7 +386,7 @@ async function getPopularInvestorsToUpdate(dependencies, config) {
             logger.log('INFO', `[Core Utils Debug] First PI Target Mapped: ${JSON.stringify(filteredTargets[0])}`);
         }
 
-        logger.log('INFO', `[Core Utils] Found ${allTargets.length} Popular Investors from ${rankingsDate} ranking${rankingsDate !== today ? ` (fallback from ${today})` : ''}. Skipped ${skippedCount} recently updated. ${filteredTargets.length} will be updated.`);
+        logger.log('INFO', `[Core Utils] Found ${allTargets.length} Popular Investors from last 7 days of rankings. Skipped ${skippedCount} recently updated. ${filteredTargets.length} will be updated.`);
         return filteredTargets;
 
     } catch (error) {
@@ -347,21 +397,37 @@ async function getPopularInvestorsToUpdate(dependencies, config) {
 
 /**
  * [NEW] Fetches Signed-In Users for daily update.
+ * UPDATED: Uses user-centric collection model and collection registry.
  * Skips users updated in the last 18 hours to avoid duplicate refreshes.
  */
 async function getSignedInUsersToUpdate(dependencies, config) {
-    const { db, logger } = dependencies;
-    const collectionName = config.signedInUsersCollection || process.env.FIRESTORE_COLLECTION_SIGNED_IN_USER_PORTFOLIOS || 'signed_in_users';
+    const { db, logger, collectionRegistry } = dependencies;
+    
+    // Get collection name from registry if available
+    let signedInUsersCollection = config.signedInUsersCollection || 'signedInUsers';
+    
+    if (collectionRegistry && collectionRegistry.getCollectionPath) {
+        try {
+            // Extract collection name from registry path: signedInUsers/{firebaseUid}
+            const samplePath = collectionRegistry.getCollectionPath('signedInUsers', 'main', {});
+            signedInUsersCollection = samplePath.split('/')[0];
+        } catch (err) {
+            logger.log('WARN', `[Core Utils] Failed to get collection from registry, using config: ${err.message}`);
+        }
+    }
     
     // 18 hours threshold - skip users updated more recently than this
     const HOURS_THRESHOLD = 18;
     const thresholdMs = HOURS_THRESHOLD * 60 * 60 * 1000;
     const now = Date.now();
+    const today = new Date().toISOString().split('T')[0];
 
-    logger.log('INFO', `[Core Utils] Getting Signed-In Users to update from ${collectionName} (skipping users updated in last ${HOURS_THRESHOLD} hours)...`);
+    logger.log('INFO', `[Core Utils] Getting Signed-In Users to update from ${signedInUsersCollection} (skipping users updated in last ${HOURS_THRESHOLD} hours)...`);
 
     try {
-        const snapshot = await db.collection(collectionName).get();
+        // Get all signed-in users from the main collection
+        // This collection maps Firebase UID to eToro CID
+        const snapshot = await db.collection(signedInUsersCollection).get();
         
         if (snapshot.empty) {
             logger.log('INFO', '[Core Utils] No Signed-In Users found.');
@@ -369,74 +435,67 @@ async function getSignedInUsersToUpdate(dependencies, config) {
         }
 
         const targets = [];
-        let skippedCount = 0;
         
+        // Extract CIDs from user documents
         snapshot.forEach(doc => {
             const data = doc.data();
-            const cid = data.cid || doc.id; 
-            const username = data.username || 'Unknown';
+            const cid = data.etoroCID || data.cid || null;
+            const username = data.etoroUsername || data.username || 'Unknown';
             
             if (!cid) return;
             
-            // Check if user was recently updated (check sync requests and portfolio timestamps)
-            // First check sync requests for last update time
-            let shouldSkip = false;
-            
-            // Check user_sync_requests for last completed sync
-            // We'll check this asynchronously, but for now we'll check portfolio data timestamps
-            // The portfolio data is stored in snapshots with dates, so we can check the latest snapshot date
-            const portfolioCollection = collectionName; // Same collection
-            const blockId = '19M';
-            const today = new Date().toISOString().split('T')[0];
-            
-            // For efficiency, we'll check the sync requests collection for the last update time
-            // This is a synchronous check we can do here
-            // Actually, we need to make this async, so we'll do a batch check after
-            
-            // For now, include all users - we'll filter in the dispatcher
             targets.push({ cid: String(cid), username });
         });
         
-        // Now filter out users who were updated today
-        // Check actual portfolio/history snapshot dates, not just sync request timestamps
+        logger.log('INFO', `[Core Utils] Found ${targets.length} Signed-In Users. Checking update status...`);
+        
+        // Filter out users who were updated recently
         const filteredTargets = [];
-        const today = new Date().toISOString().split('T')[0];
+        let skippedCount = 0;
+        
         const checkPromises = targets.map(async (target) => {
-            // Check if user has portfolio or history data for today
-            const blockId = `${Math.floor(parseInt(target.cid) / 1000000)}M`;
-            
-            // Check portfolio snapshot for today
-            const portfolioSnapshotRef = db.collection(collectionName)
-                .doc(blockId)
-                .collection('snapshots')
-                .doc(today)
-                .collection('parts')
-                .limit(1);
-            
-            const portfolioSnapshot = await portfolioSnapshotRef.get();
+            // Check if user has portfolio data for today (using root data format)
+            let hasTodayData = false;
             
-            // Check history snapshot for today
-            const historyCollection = config.signedInUserHistoryCollection || process.env.FIRESTORE_COLLECTION_SIGNED_IN_USER_HISTORY || 'signed_in_user_history';
-            const historySnapshotRef = db.collection(historyCollection)
-                .doc(blockId)
-                .collection('snapshots')
-                .doc(today)
-                .collection('parts')
-                .limit(1);
-            
-            const historySnapshot = await historySnapshotRef.get();
-            
-            // If user has data for today, skip them (they're already up-to-date)
-            if (!portfolioSnapshot.empty || !historySnapshot.empty) {
-                skippedCount++;
-                return null; // Skip this user - already updated today
+            if (collectionRegistry && collectionRegistry.getCollectionPath) {
+                try {
+                    // Check root data portfolio for today
+                    const portfolioPath = collectionRegistry.getCollectionPath('rootData', 'signedInUserPortfolio', { 
+                        date: today, 
+                        cid: target.cid 
+                    });
+                    const portfolioRef = db.doc(portfolioPath);
+                    const portfolioDoc = await portfolioRef.get();
+                    
+                    if (portfolioDoc.exists) {
+                        hasTodayData = true;
+                    }
+                } catch (err) {
+                    logger.log('WARN', `[Core Utils] Error checking portfolio for ${target.cid}: ${err.message}`);
+                }
             }
             
-            // Also check sync requests as a fallback (in case data exists but snapshot check failed)
-            const syncRef = db.collection('user_sync_requests')
-                .doc(target.cid)
-                .collection('global')
-                .doc('latest');
+            // Also check sync status using user-centric path
+            let syncRef;
+            if (collectionRegistry && collectionRegistry.getCollectionPath) {
+                try {
+                    // syncStatus path is: SignedInUsers/{cid}/syncStatus/latest (document path)
+                    const syncStatusPath = collectionRegistry.getCollectionPath('signedInUsers', 'syncStatus', { cid: target.cid });
+                    syncRef = db.doc(syncStatusPath);
+                } catch (err) {
+                    // Fallback to legacy path
+                    syncRef = db.collection('user_sync_requests')
+                        .doc(target.cid)
+                        .collection('global')
+                        .doc('latest');
+                }
+            } else {
+                // Fallback to legacy path
+                syncRef = db.collection('user_sync_requests')
+                    .doc(target.cid)
+                    .collection('global')
+                    .doc('latest');
+            }
             
             const syncDoc = await syncRef.get();
             if (syncDoc.exists) {
@@ -446,10 +505,16 @@ async function getSignedInUsersToUpdate(dependencies, config) {
                 
                 if (lastRequestedAt && (now - lastRequestedAt) < thresholdMs) {
                     skippedCount++;
-                    return null; // Skip this user
+                    return null; // Skip this user - recently updated
                 }
             }
             
+            // If user has data for today, skip them (they're already up-to-date)
+            if (hasTodayData) {
+                skippedCount++;
+                return null; // Skip this user - already updated today
+            }
+            
             return target; // Include this user - needs update
         });
         

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.514",
+  "version": "1.0.515",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [