bulltrackers-module 1.0.514 → 1.0.516

package/functions/core/utils/firestore_utils.js

@@ -256,52 +256,85 @@ async function findLatestRankingsDate(db, rankingsCollection, maxDaysBack = 30)
 
 /**
  * [NEW] Fetches Popular Investors from the daily rankings document.
- * UPDATED: Uses latest available rankings date if today's doesn't exist (fallback for edge cases).
- * This ensures the dispatcher can still queue tasks even if today's rankings haven't been fetched yet.
+ * UPDATED: Checks last 7 days of rankings data and builds a de-duplicated map.
+ * This solves the problem of a single day where a given popular investor is not in the rankings.
+ * UPDATED: Uses collection registry for path resolution.
  */
 async function getPopularInvestorsToUpdate(dependencies, config) {
-    const { db, logger } = dependencies;
-    const collectionName = config.popularInvestorRankingsCollection || process.env.FIRESTORE_COLLECTION_PI_RANKINGS || 'popular_investor_rankings';
+    const { db, logger, collectionRegistry } = dependencies;
+    
+    // Get collection name from registry if available
+    let collectionName = config.popularInvestorRankingsCollection || process.env.FIRESTORE_COLLECTION_PI_RANKINGS || 'popular_investor_rankings';
+    
+    if (collectionRegistry && collectionRegistry.getCollectionPath) {
+        try {
+            // Extract collection name from registry path: popular_investor_rankings/{date}
+            const samplePath = collectionRegistry.getCollectionPath('rootData', 'popularInvestorRankings', { date: '2025-01-01' });
+            collectionName = samplePath.split('/')[0];
+        } catch (err) {
+            logger.log('WARN', `[Core Utils] Failed to get collection from registry, using config: ${err.message}`);
+        }
+    }
 
-    logger.log('INFO', `[Core Utils] Getting Popular Investors to update from ${collectionName}...`);
+    logger.log('INFO', `[Core Utils] Getting Popular Investors to update from ${collectionName} (checking last 7 days)...`);
     
-    // Try today's date first
-    const today = new Date().toISOString().split('T')[0];
-    let rankingsDate = today;
+    // Check last 7 days of rankings data
+    const today = new Date();
+    const piMap = new Map(); // De-duplicated map: cid -> { cid, username, latestDate }
     
     try {
-        let docRef = db.collection(collectionName).doc(today);
-        let docSnap = await docRef.get();
-
-        // If today's rankings don't exist, fallback to latest available date
-        if (!docSnap.exists) {
-            logger.log('WARN', `[Core Utils] No Popular Investor rankings found for date: ${today}. Looking for latest available date...`);
-            rankingsDate = await findLatestRankingsDate(db, collectionName, 30);
-            
-            if (!rankingsDate) {
-                logger.log('WARN', `[Core Utils] No Popular Investor rankings found in last 30 days. Returning empty array.`);
-                return [];
-            }
-            
-            logger.log('INFO', `[Core Utils] Using fallback rankings date: ${rankingsDate} (today: ${today})`);
-            docRef = db.collection(collectionName).doc(rankingsDate);
-            docSnap = await docRef.get();
+        // Check each of the last 7 days
+        for (let i = 0; i < 7; i++) {
+            const checkDate = new Date(today);
+            checkDate.setDate(checkDate.getDate() - i);
+            const dateStr = checkDate.toISOString().split('T')[0];
             
-            if (!docSnap.exists) {
-                logger.log('WARN', `[Core Utils] Rankings doc does not exist for fallback date ${rankingsDate}`);
-                return [];
+            try {
+                const docRef = db.collection(collectionName).doc(dateStr);
+                const docSnap = await docRef.get();
+                
+                if (docSnap.exists) {
+                    const data = docSnap.data();
+                    const items = data.Items || [];
+                    
+                    // Add all PIs from this date to the map (de-duplicated by CID)
+                    for (const item of items) {
+                        const cid = String(item.CustomerId);
+                        const username = item.UserName;
+                        
+                        // Only add if not already in map, or if this date is more recent
+                        if (!piMap.has(cid)) {
+                            piMap.set(cid, {
+                                cid: cid,
+                                username: username,
+                                latestDate: dateStr
+                            });
+                        } else {
+                            // Update if this date is more recent
+                            const existing = piMap.get(cid);
+                            if (dateStr > existing.latestDate) {
+                                existing.latestDate = dateStr;
+                                if (username) existing.username = username;
+                            }
+                        }
+                    }
+                }
+            } catch (dateErr) {
+                logger.log('WARN', `[Core Utils] Error checking rankings for date ${dateStr}: ${dateErr.message}`);
+                continue; // Continue to next date
             }
         }
+        
+        if (piMap.size === 0) {
+            logger.log('WARN', `[Core Utils] No Popular Investor rankings found in last 7 days. Returning empty array.`);
+            return [];
+        }
+        
+        logger.log('INFO', `[Core Utils] Found ${piMap.size} unique Popular Investors across last 7 days of rankings.`);
 
-        const data = docSnap.data();
-        const items = data.Items || [];
-
-        // Map to { cid, username }
-        const allTargets = items.map(item => ({
-            cid: String(item.CustomerId),
-            username: item.UserName
-        }));
-
+        // Convert map to array and filter out PIs updated in the last 18 hours
+        const allTargets = Array.from(piMap.values());
+        
         // Filter out PIs updated in the last 18 hours
         const HOURS_THRESHOLD = 18;
         const thresholdMs = HOURS_THRESHOLD * 60 * 60 * 1000;
@@ -310,12 +343,29 @@ async function getPopularInvestorsToUpdate(dependencies, config) {
         const filteredTargets = [];
         let skippedCount = 0;
         
-        // Check each PI's last sync time
+        // Check each PI's last sync time using user-centric path
         for (const target of allTargets) {
-            const syncRef = db.collection('user_sync_requests')
-                .doc(target.cid)
-                .collection('global')
-                .doc('latest');
+            // Use collection registry for sync status path if available
+            let syncRef;
+            if (collectionRegistry && collectionRegistry.getCollectionPath) {
+                try {
+                    // syncStatus path is: SignedInUsers/{cid}/syncStatus/latest (document path)
+                    const syncStatusPath = collectionRegistry.getCollectionPath('signedInUsers', 'syncStatus', { cid: target.cid });
+                    syncRef = db.doc(syncStatusPath);
+                } catch (err) {
+                    // Fallback to legacy path
+                    syncRef = db.collection('user_sync_requests')
+                        .doc(target.cid)
+                        .collection('global')
+                        .doc('latest');
+                }
+            } else {
+                // Fallback to legacy path
+                syncRef = db.collection('user_sync_requests')
+                    .doc(target.cid)
+                    .collection('global')
+                    .doc('latest');
+            }
             
             const syncDoc = await syncRef.get();
             if (syncDoc.exists) {
@@ -336,7 +386,7 @@ async function getPopularInvestorsToUpdate(dependencies, config) {
             logger.log('INFO', `[Core Utils Debug] First PI Target Mapped: ${JSON.stringify(filteredTargets[0])}`);
         }
 
-        logger.log('INFO', `[Core Utils] Found ${allTargets.length} Popular Investors from ${rankingsDate} ranking${rankingsDate !== today ? ` (fallback from ${today})` : ''}. Skipped ${skippedCount} recently updated. ${filteredTargets.length} will be updated.`);
+        logger.log('INFO', `[Core Utils] Found ${allTargets.length} Popular Investors from last 7 days of rankings. Skipped ${skippedCount} recently updated. ${filteredTargets.length} will be updated.`);
         return filteredTargets;
 
     } catch (error) {
@@ -347,21 +397,37 @@ async function getPopularInvestorsToUpdate(dependencies, config) {
 
 /**
  * [NEW] Fetches Signed-In Users for daily update.
+ * UPDATED: Uses user-centric collection model and collection registry.
  * Skips users updated in the last 18 hours to avoid duplicate refreshes.
  */
 async function getSignedInUsersToUpdate(dependencies, config) {
-    const { db, logger } = dependencies;
-    const collectionName = config.signedInUsersCollection || process.env.FIRESTORE_COLLECTION_SIGNED_IN_USER_PORTFOLIOS || 'signed_in_users';
+    const { db, logger, collectionRegistry } = dependencies;
+    
+    // Get collection name from registry if available
+    let signedInUsersCollection = config.signedInUsersCollection || 'signedInUsers';
+    
+    if (collectionRegistry && collectionRegistry.getCollectionPath) {
+        try {
+            // Extract collection name from registry path: signedInUsers/{firebaseUid}
+            const samplePath = collectionRegistry.getCollectionPath('signedInUsers', 'main', {});
+            signedInUsersCollection = samplePath.split('/')[0];
+        } catch (err) {
+            logger.log('WARN', `[Core Utils] Failed to get collection from registry, using config: ${err.message}`);
+        }
+    }
     
     // 18 hours threshold - skip users updated more recently than this
     const HOURS_THRESHOLD = 18;
     const thresholdMs = HOURS_THRESHOLD * 60 * 60 * 1000;
     const now = Date.now();
+    const today = new Date().toISOString().split('T')[0];
 
-    logger.log('INFO', `[Core Utils] Getting Signed-In Users to update from ${collectionName} (skipping users updated in last ${HOURS_THRESHOLD} hours)...`);
+    logger.log('INFO', `[Core Utils] Getting Signed-In Users to update from ${signedInUsersCollection} (skipping users updated in last ${HOURS_THRESHOLD} hours)...`);
 
     try {
-        const snapshot = await db.collection(collectionName).get();
+        // Get all signed-in users from the main collection
+        // This collection maps Firebase UID to eToro CID
+        const snapshot = await db.collection(signedInUsersCollection).get();
         
         if (snapshot.empty) {
             logger.log('INFO', '[Core Utils] No Signed-In Users found.');
@@ -369,74 +435,67 @@ async function getSignedInUsersToUpdate(dependencies, config) {
         }
 
         const targets = [];
-        let skippedCount = 0;
         
+        // Extract CIDs from user documents
         snapshot.forEach(doc => {
             const data = doc.data();
-            const cid = data.cid || doc.id; 
-            const username = data.username || 'Unknown';
+            const cid = data.etoroCID || data.cid || null;
+            const username = data.etoroUsername || data.username || 'Unknown';
             
             if (!cid) return;
             
-            // Check if user was recently updated (check sync requests and portfolio timestamps)
-            // First check sync requests for last update time
-            let shouldSkip = false;
-            
-            // Check user_sync_requests for last completed sync
-            // We'll check this asynchronously, but for now we'll check portfolio data timestamps
-            // The portfolio data is stored in snapshots with dates, so we can check the latest snapshot date
-            const portfolioCollection = collectionName; // Same collection
-            const blockId = '19M';
-            const today = new Date().toISOString().split('T')[0];
-            
-            // For efficiency, we'll check the sync requests collection for the last update time
-            // This is a synchronous check we can do here
-            // Actually, we need to make this async, so we'll do a batch check after
-            
-            // For now, include all users - we'll filter in the dispatcher
             targets.push({ cid: String(cid), username });
         });
         
-        // Now filter out users who were updated today
-        // Check actual portfolio/history snapshot dates, not just sync request timestamps
+        logger.log('INFO', `[Core Utils] Found ${targets.length} Signed-In Users. Checking update status...`);
+        
+        // Filter out users who were updated recently
         const filteredTargets = [];
-        const today = new Date().toISOString().split('T')[0];
+        let skippedCount = 0;
+        
         const checkPromises = targets.map(async (target) => {
-            // Check if user has portfolio or history data for today
-            const blockId = `${Math.floor(parseInt(target.cid) / 1000000)}M`;
-            
-            // Check portfolio snapshot for today
-            const portfolioSnapshotRef = db.collection(collectionName)
-                .doc(blockId)
-                .collection('snapshots')
-                .doc(today)
-                .collection('parts')
-                .limit(1);
-            
-            const portfolioSnapshot = await portfolioSnapshotRef.get();
+            // Check if user has portfolio data for today (using root data format)
+            let hasTodayData = false;
             
-            // Check history snapshot for today
-            const historyCollection = config.signedInUserHistoryCollection || process.env.FIRESTORE_COLLECTION_SIGNED_IN_USER_HISTORY || 'signed_in_user_history';
-            const historySnapshotRef = db.collection(historyCollection)
-                .doc(blockId)
-                .collection('snapshots')
-                .doc(today)
-                .collection('parts')
-                .limit(1);
-            
-            const historySnapshot = await historySnapshotRef.get();
-            
-            // If user has data for today, skip them (they're already up-to-date)
-            if (!portfolioSnapshot.empty || !historySnapshot.empty) {
-                skippedCount++;
-                return null; // Skip this user - already updated today
+            if (collectionRegistry && collectionRegistry.getCollectionPath) {
+                try {
+                    // Check root data portfolio for today
+                    const portfolioPath = collectionRegistry.getCollectionPath('rootData', 'signedInUserPortfolio', { 
+                        date: today, 
+                        cid: target.cid 
+                    });
+                    const portfolioRef = db.doc(portfolioPath);
+                    const portfolioDoc = await portfolioRef.get();
+                    
+                    if (portfolioDoc.exists) {
+                        hasTodayData = true;
+                    }
+                } catch (err) {
+                    logger.log('WARN', `[Core Utils] Error checking portfolio for ${target.cid}: ${err.message}`);
+                }
             }
             
-            // Also check sync requests as a fallback (in case data exists but snapshot check failed)
-            const syncRef = db.collection('user_sync_requests')
-                .doc(target.cid)
-                .collection('global')
-                .doc('latest');
+            // Also check sync status using user-centric path
+            let syncRef;
+            if (collectionRegistry && collectionRegistry.getCollectionPath) {
+                try {
+                    // syncStatus path is: SignedInUsers/{cid}/syncStatus/latest (document path)
+                    const syncStatusPath = collectionRegistry.getCollectionPath('signedInUsers', 'syncStatus', { cid: target.cid });
+                    syncRef = db.doc(syncStatusPath);
+                } catch (err) {
+                    // Fallback to legacy path
+                    syncRef = db.collection('user_sync_requests')
+                        .doc(target.cid)
+                        .collection('global')
+                        .doc('latest');
+                }
+            } else {
+                // Fallback to legacy path
+                syncRef = db.collection('user_sync_requests')
+                    .doc(target.cid)
+                    .collection('global')
+                    .doc('latest');
+            }
             
             const syncDoc = await syncRef.get();
             if (syncDoc.exists) {
@@ -446,10 +505,16 @@ async function getSignedInUsersToUpdate(dependencies, config) {
                 
                 if (lastRequestedAt && (now - lastRequestedAt) < thresholdMs) {
                     skippedCount++;
-                    return null; // Skip this user
+                    return null; // Skip this user - recently updated
                 }
             }
             
+            // If user has data for today, skip them (they're already up-to-date)
+            if (hasTodayData) {
+                skippedCount++;
+                return null; // Skip this user - already updated today
+            }
+            
             return target; // Include this user - needs update
         });
         

package/functions/generic-api/user-api/helpers/sync/user_sync_helpers.js

@@ -30,6 +30,15 @@ async function requestUserSync(req, res, dependencies, config) {
         });
     }
     
+    // Check the referrer or source header to determine if this is from PI page or profile page
+    const referrer = req.headers.referer || req.headers.referrer || '';
+    const sourcePage = req.query?.sourcePage || req.body?.sourcePage || '';
+    
+    // Determine user type based on where the request came from
+    // If from /popular-investors/{cid}, it's a PI
+    // If from /profile, it's a signed-in user
+    const isPI = referrer.includes('/popular-investors/') || sourcePage === 'popular-investor';
+    
     // Check for dev override impersonation
     const { getEffectiveCid, getDevOverride } = require('../dev/dev_helpers');
     const effectiveCid = await getEffectiveCid(db, requestingUserCid, config, logger);
@@ -71,103 +80,17 @@ async function requestUserSync(req, res, dependencies, config) {
             logger.log('INFO', `[requestUserSync] Developer account ${requestingUserCid} bypassing rate limits${isImpersonating ? ` (impersonating ${effectiveCid})` : ''}`);
         }
         
-        // Determine user type (PI or signed-in user)
-        // Check if user is in rankings (PI) or has signed-in user data
-        const { checkIfUserIsPI } = require('../data_helpers');
-        let isPI = await checkIfUserIsPI(db, targetCidNum, config, logger);
-        
-        // Fallback: If not found in rankings, check if they have PI data (portfolio, history, or social)
-        // This handles cases where a PI might not be in the latest rankings but has historical data
-        if (!isPI) {
-            try {
-                const today = new Date().toISOString().split('T')[0];
-                let hasPiData = false;
-                
-                // Check if they have PI portfolio data (any date in last 7 days)
-                for (let i = 0; i <= 7; i++) {
-                    const checkDate = new Date();
-                    checkDate.setDate(checkDate.getDate() - i);
-                    const dateStr = checkDate.toISOString().split('T')[0];
-                    
-                    // Check portfolio data
-                    const portfolioRef = db.collection('PopularInvestorPortfolioData')
-                        .doc(dateStr)
-                        .collection(String(targetCidNum))
-                        .doc(String(targetCidNum));
-                    const portfolioDoc = await portfolioRef.get();
-                    
-                    if (portfolioDoc.exists) {
-                        logger.log('INFO', `[requestUserSync] User ${targetCidNum} has PI portfolio data from ${dateStr}, treating as PI`);
-                        hasPiData = true;
-                        break;
-                    }
-                    
-                    // Check trade history data
-                    const historyRef = db.collection('PopularInvestorTradeHistoryData')
-                        .doc(dateStr)
-                        .collection(String(targetCidNum))
-                        .doc(String(targetCidNum));
-                    const historyDoc = await historyRef.get();
-                    
-                    if (historyDoc.exists) {
-                        logger.log('INFO', `[requestUserSync] User ${targetCidNum} has PI trade history data from ${dateStr}, treating as PI`);
-                        hasPiData = true;
-                        break;
-                    }
-                    
-                    // Check social posts data
-                    const socialRef = db.collection('PopularInvestorSocialPostData')
-                        .doc(dateStr)
-                        .collection(String(targetCidNum))
-                        .doc(String(targetCidNum));
-                    const socialDoc = await socialRef.get();
-                    
-                    if (socialDoc.exists) {
-                        logger.log('INFO', `[requestUserSync] User ${targetCidNum} has PI social posts data from ${dateStr}, treating as PI`);
-                        hasPiData = true;
-                        break;
-                    }
-                }
-                
-                if (hasPiData) {
-                    isPI = { CustomerId: targetCidNum, UserName: null }; // Fake ranking entry
-                } else {
-                    logger.log('INFO', `[requestUserSync] User ${targetCidNum} not found in rankings and no PI data found, treating as signed-in user`);
-                }
-            } catch (fallbackErr) {
-                logger.log('WARN', `[requestUserSync] Error checking PI data fallback for ${targetCidNum}:`, fallbackErr.message);
-            }
-        } else {
-            logger.log('INFO', `[requestUserSync] User ${targetCidNum} found in rankings as PI`);
-        }
-        
-        // Get username - for PIs, get from rankings; for signed-in users, we'll need to fetch from their profile
+        // Get username based on user type
         let username = null;
         if (isPI) {
-            // Get PI username from rankings (if available)
-            if (isPI.UserName || isPI.username) {
-                username = isPI.UserName || isPI.username;
-            } else {
-                // Try to get from rankings if we have a fake entry
-                const { findLatestRankingsDate } = require('../data_helpers');
-                const rankingsCollection = config.popularInvestorRankingsCollection || 'popular_investor_rankings';
-                const rankingsDate = await findLatestRankingsDate(db, rankingsCollection, 30);
-                
-                if (rankingsDate) {
-                    const rankingsRef = db.collection(rankingsCollection).doc(rankingsDate);
-                    const rankingsDoc = await rankingsRef.get();
-                    if (rankingsDoc.exists) {
-                        const rankingsData = rankingsDoc.data();
-                        const rankingsItems = rankingsData.Items || [];
-                        const rankingEntry = rankingsItems.find(item => Number(item.CustomerId) === targetCidNum);
-                        if (rankingEntry) {
-                            username = rankingEntry.UserName || rankingEntry.username || null;
-                        }
-                    }
-                }
+            // For PIs, get username from rankings
+            const { checkIfUserIsPI } = require('../core/user_status_helpers');
+            const rankEntry = await checkIfUserIsPI(db, targetCidNum, config, logger);
+            if (rankEntry && (rankEntry.UserName || rankEntry.username)) {
+                username = rankEntry.UserName || rankEntry.username;
             }
         } else {
-            // For signed-in users, try to get username from verification data
+            // For signed-in users, try verification data
             const signedInUsersCollection = config.signedInUsersCollection || 'signed_in_users';
             const userDoc = await db.collection(signedInUsersCollection).doc(String(targetCidNum)).get();
             if (userDoc.exists) {
@@ -177,8 +100,7 @@ async function requestUserSync(req, res, dependencies, config) {
         }
         
         if (!username) {
-            logger.log('WARN', `[requestUserSync] Could not find username for user ${targetCidNum}, will use CID as fallback`);
-            username = String(targetCidNum);
+            username = String(targetCidNum); // Fallback to CID
         }
         
         // Create request document

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -832,9 +832,9 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
         logger.log('INFO', `[On-Demand Update] Skipping history fetch (portfolioOnly=false) for ${username}`);
     }
 
-        // Fetch social data if requested (for user signup or explicit request)
+        // Fetch social data - always for on-demand syncs
         let socialFetched = false;
-        if (includeSocial && username) {
+        if (username) {
             try {
                 logger.log('INFO', `[On-Demand Update] Fetching social data for ${username} (${cid})...`);
                 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.514",
+  "version": "1.0.516",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [