bulltrackers-module 1.0.516 → 1.0.518

package/functions/core/utils/firestore_utils.js

@@ -255,14 +255,129 @@ async function findLatestRankingsDate(db, rankingsCollection, maxDaysBack = 30)
 }
 
 /**
- * [NEW] Fetches Popular Investors from the daily rankings document.
- * UPDATED: Checks last 7 days of rankings data and builds a de-duplicated map.
- * This solves the problem of a single day where a given popular investor is not in the rankings.
+ * [NEW] Fetches Popular Investors from the master list and filters by last updated times.
+ * UPDATED: Uses the master list as single source of truth, then checks last updated timestamps
+ * for each data type (portfolio, tradeHistory, socialPosts) to determine who needs updating.
  * UPDATED: Uses collection registry for path resolution.
  */
 async function getPopularInvestorsToUpdate(dependencies, config) {
     const { db, logger, collectionRegistry } = dependencies;
     
+    // 24 hours threshold - users not updated in the last 24 hours need updating
+    const HOURS_THRESHOLD = 24;
+    const thresholdMs = HOURS_THRESHOLD * 60 * 60 * 1000;
+    const now = Date.now();
+    
+    logger.log('INFO', `[Core Utils] Getting Popular Investors to update (checking master list and last updated times, threshold: ${HOURS_THRESHOLD} hours)...`);
+    
+    try {
+        // Get the master list of Popular Investors
+        let masterListPath = 'system_state/popular_investor_master_list';
+        
+        if (collectionRegistry && collectionRegistry.getCollectionPath) {
+            try {
+                masterListPath = collectionRegistry.getCollectionPath('system', 'popularInvestorMasterList', {});
+            } catch (err) {
+                logger.log('WARN', `[Core Utils] Failed to get master list path from registry, using default: ${err.message}`);
+            }
+        }
+        
+        const masterListRef = db.doc(masterListPath);
+        const masterListDoc = await masterListRef.get();
+        
+        if (!masterListDoc.exists) {
+            logger.log('WARN', `[Core Utils] Master list not found. Falling back to rankings-based approach.`);
+            // Fallback to old method if master list doesn't exist yet
+            return await getPopularInvestorsToUpdateLegacy(dependencies, config);
+        }
+        
+        const masterListData = masterListDoc.data();
+        const investors = masterListData.investors || {};
+        
+        if (Object.keys(investors).length === 0) {
+            logger.log('WARN', `[Core Utils] Master list is empty. Returning empty array.`);
+            return [];
+        }
+        
+        logger.log('INFO', `[Core Utils] Found ${Object.keys(investors).length} Popular Investors in master list. Checking last updated times...`);
+        
+        const targets = [];
+        let skippedCount = 0;
+        
+        // Check each PI's last updated times for each data type
+        for (const [cid, piData] of Object.entries(investors)) {
+            const username = piData.username || String(cid);
+            
+            // Get last updated document for this PI
+            let lastUpdatedPath;
+            if (collectionRegistry && collectionRegistry.getCollectionPath) {
+                try {
+                    lastUpdatedPath = collectionRegistry.getCollectionPath('popularInvestors', 'lastUpdated', { piCid: cid });
+                } catch (err) {
+                    logger.log('WARN', `[Core Utils] Failed to get lastUpdated path for ${cid}: ${err.message}`);
+                    // Include this PI if we can't check (better to update than skip)
+                    targets.push({ cid, username });
+                    continue;
+                }
+            } else {
+                // Include this PI if registry not available
+                targets.push({ cid, username });
+                continue;
+            }
+            
+            const lastUpdatedRef = db.doc(lastUpdatedPath);
+            const lastUpdatedDoc = await lastUpdatedRef.get();
+            
+            let needsUpdate = false;
+            
+            if (!lastUpdatedDoc.exists) {
+                // No last updated data - needs update
+                needsUpdate = true;
+            } else {
+                const lastUpdatedData = lastUpdatedDoc.data();
+                
+                // Check each data type - if any is missing or older than threshold, needs update
+                const dataTypes = ['portfolio', 'tradeHistory', 'socialPosts'];
+                for (const dataType of dataTypes) {
+                    const lastUpdated = lastUpdatedData[dataType];
+                    if (!lastUpdated) {
+                        needsUpdate = true;
+                        break;
+                    }
+                    
+                    const lastUpdatedMs = lastUpdated.toDate?.()?.getTime() || lastUpdated.toMillis?.() || null;
+                    if (lastUpdatedMs && (now - lastUpdatedMs) >= thresholdMs) {
+                        needsUpdate = true;
+                        break;
+                    }
+                }
+            }
+            
+            if (needsUpdate) {
+                targets.push({ cid, username });
+            } else {
+                skippedCount++;
+            }
+        }
+        
+        logger.log('INFO', `[Core Utils] Found ${Object.keys(investors).length} Popular Investors in master list. Skipped ${skippedCount} recently updated. ${targets.length} will be updated.`);
+        return targets;
+        
+    } catch (error) {
+        logger.log('ERROR', '[Core Utils] Error getting Popular Investors', { errorMessage: error.message });
+        // Fallback to legacy method on error
+        logger.log('INFO', '[Core Utils] Falling back to legacy rankings-based approach due to error.');
+        return await getPopularInvestorsToUpdateLegacy(dependencies, config);
+    }
+}
+
+/**
+ * Legacy method: Fetches Popular Investors from the daily rankings document.
+ * Used as fallback when master list is not available.
+ */
+async function getPopularInvestorsToUpdateLegacy(dependencies, config) {
+    const { db, logger, collectionRegistry } = dependencies;
+    
     // Get collection name from registry if available
     let collectionName = config.popularInvestorRankingsCollection || process.env.FIRESTORE_COLLECTION_PI_RANKINGS || 'popular_investor_rankings';
     
@@ -276,7 +391,7 @@ async function getPopularInvestorsToUpdate(dependencies, config) {
         }
     }
 
-    logger.log('INFO', `[Core Utils] Getting Popular Investors to update from ${collectionName} (checking last 7 days)...`);
+    logger.log('INFO', `[Core Utils] Using legacy method: Getting Popular Investors from ${collectionName} (checking last 7 days)...`);
     
     // Check last 7 days of rankings data
     const today = new Date();
@@ -331,66 +446,10 @@ async function getPopularInvestorsToUpdate(dependencies, config) {
         }
         
         logger.log('INFO', `[Core Utils] Found ${piMap.size} unique Popular Investors across last 7 days of rankings.`);
-
-        // Convert map to array and filter out PIs updated in the last 18 hours
-        const allTargets = Array.from(piMap.values());
-        
-        // Filter out PIs updated in the last 18 hours
-        const HOURS_THRESHOLD = 18;
-        const thresholdMs = HOURS_THRESHOLD * 60 * 60 * 1000;
-        const now = Date.now();
-        
-        const filteredTargets = [];
-        let skippedCount = 0;
-        
-        // Check each PI's last sync time using user-centric path
-        for (const target of allTargets) {
-            // Use collection registry for sync status path if available
-            let syncRef;
-            if (collectionRegistry && collectionRegistry.getCollectionPath) {
-                try {
-                    // syncStatus path is: SignedInUsers/{cid}/syncStatus/latest (document path)
-                    const syncStatusPath = collectionRegistry.getCollectionPath('signedInUsers', 'syncStatus', { cid: target.cid });
-                    syncRef = db.doc(syncStatusPath);
-                } catch (err) {
-                    // Fallback to legacy path
-                    syncRef = db.collection('user_sync_requests')
-                        .doc(target.cid)
-                        .collection('global')
-                        .doc('latest');
-                }
-            } else {
-                // Fallback to legacy path
-                syncRef = db.collection('user_sync_requests')
-                    .doc(target.cid)
-                    .collection('global')
-                    .doc('latest');
-            }
-            
-            const syncDoc = await syncRef.get();
-            if (syncDoc.exists) {
-                const syncData = syncDoc.data();
-                const lastRequestedAt = syncData.lastRequestedAt?.toDate?.()?.getTime() || 
-                                      syncData.lastRequestedAt?.toMillis?.() || null;
-                
-                if (lastRequestedAt && (now - lastRequestedAt) < thresholdMs) {
-                    skippedCount++;
-                    continue; // Skip this PI
-                }
-            }
-            
-            filteredTargets.push(target); // Include this PI
-        }
-
-        if (filteredTargets.length > 0) {
-            logger.log('INFO', `[Core Utils Debug] First PI Target Mapped: ${JSON.stringify(filteredTargets[0])}`);
-        }
-
-        logger.log('INFO', `[Core Utils] Found ${allTargets.length} Popular Investors from last 7 days of rankings. Skipped ${skippedCount} recently updated. ${filteredTargets.length} will be updated.`);
-        return filteredTargets;
+        return Array.from(piMap.values());
 
     } catch (error) {
-        logger.log('ERROR', '[Core Utils] Error getting Popular Investors', { errorMessage: error.message });
+        logger.log('ERROR', '[Core Utils] Error getting Popular Investors (legacy)', { errorMessage: error.message });
         throw error;
     }
 }
@@ -398,7 +457,8 @@ async function getPopularInvestorsToUpdate(dependencies, config) {
 /**
  * [NEW] Fetches Signed-In Users for daily update.
  * UPDATED: Uses user-centric collection model and collection registry.
- * Skips users updated in the last 18 hours to avoid duplicate refreshes.
+ * UPDATED: Checks last updated timestamps for each data type (portfolio, tradeHistory, socialPosts)
+ * to determine who needs updating. Users not updated in the last 24 hours need updating.
  */
 async function getSignedInUsersToUpdate(dependencies, config) {
     const { db, logger, collectionRegistry } = dependencies;
@@ -416,13 +476,12 @@ async function getSignedInUsersToUpdate(dependencies, config) {
         }
     }
     
-    // 18 hours threshold - skip users updated more recently than this
-    const HOURS_THRESHOLD = 18;
+    // 24 hours threshold - users not updated in the last 24 hours need updating
+    const HOURS_THRESHOLD = 24;
     const thresholdMs = HOURS_THRESHOLD * 60 * 60 * 1000;
     const now = Date.now();
-    const today = new Date().toISOString().split('T')[0];
 
-    logger.log('INFO', `[Core Utils] Getting Signed-In Users to update from ${signedInUsersCollection} (skipping users updated in last ${HOURS_THRESHOLD} hours)...`);
+    logger.log('INFO', `[Core Utils] Getting Signed-In Users to update from ${signedInUsersCollection} (threshold: ${HOURS_THRESHOLD} hours)...`);
 
     try {
         // Get all signed-in users from the main collection
@@ -434,7 +493,7 @@ async function getSignedInUsersToUpdate(dependencies, config) {
             return [];
         }
 
-        const targets = [];
+        const allUsers = [];
         
         // Extract CIDs from user documents
         snapshot.forEach(doc => {
@@ -444,84 +503,71 @@ async function getSignedInUsersToUpdate(dependencies, config) {
             
             if (!cid) return;
             
-            targets.push({ cid: String(cid), username });
+            allUsers.push({ cid: String(cid), username });
         });
         
-        logger.log('INFO', `[Core Utils] Found ${targets.length} Signed-In Users. Checking update status...`);
+        logger.log('INFO', `[Core Utils] Found ${allUsers.length} Signed-In Users. Checking last updated times...`);
         
         // Filter out users who were updated recently
-        const filteredTargets = [];
+        const targets = [];
         let skippedCount = 0;
         
-        const checkPromises = targets.map(async (target) => {
-            // Check if user has portfolio data for today (using root data format)
-            let hasTodayData = false;
-            
+        const checkPromises = allUsers.map(async (target) => {
+            // Get last updated document for this user
+            let lastUpdatedPath;
             if (collectionRegistry && collectionRegistry.getCollectionPath) {
                 try {
-                    // Check root data portfolio for today
-                    const portfolioPath = collectionRegistry.getCollectionPath('rootData', 'signedInUserPortfolio', { 
-                        date: today, 
-                        cid: target.cid 
-                    });
-                    const portfolioRef = db.doc(portfolioPath);
-                    const portfolioDoc = await portfolioRef.get();
-                    
-                    if (portfolioDoc.exists) {
-                        hasTodayData = true;
-                    }
+                    lastUpdatedPath = collectionRegistry.getCollectionPath('signedInUsers', 'lastUpdated', { cid: target.cid });
                 } catch (err) {
-                    logger.log('WARN', `[Core Utils] Error checking portfolio for ${target.cid}: ${err.message}`);
-                }
-            }
-            
-            // Also check sync status using user-centric path
-            let syncRef;
-            if (collectionRegistry && collectionRegistry.getCollectionPath) {
-                try {
-                    // syncStatus path is: SignedInUsers/{cid}/syncStatus/latest (document path)
-                    const syncStatusPath = collectionRegistry.getCollectionPath('signedInUsers', 'syncStatus', { cid: target.cid });
-                    syncRef = db.doc(syncStatusPath);
-                } catch (err) {
-                    // Fallback to legacy path
-                    syncRef = db.collection('user_sync_requests')
-                        .doc(target.cid)
-                        .collection('global')
-                        .doc('latest');
+                    logger.log('WARN', `[Core Utils] Failed to get lastUpdated path for ${target.cid}: ${err.message}`);
+                    // Include this user if we can't check (better to update than skip)
+                    return target;
                 }
             } else {
-                // Fallback to legacy path
-                syncRef = db.collection('user_sync_requests')
-                    .doc(target.cid)
-                    .collection('global')
-                    .doc('latest');
+                // Include this user if registry not available
+                return target;
             }
             
-            const syncDoc = await syncRef.get();
-            if (syncDoc.exists) {
-                const syncData = syncDoc.data();
-                const lastRequestedAt = syncData.lastRequestedAt?.toDate?.()?.getTime() || 
-                                      syncData.lastRequestedAt?.toMillis?.() || null;
+            const lastUpdatedRef = db.doc(lastUpdatedPath);
+            const lastUpdatedDoc = await lastUpdatedRef.get();
+            
+            let needsUpdate = false;
+            
+            if (!lastUpdatedDoc.exists) {
+                // No last updated data - needs update
+                needsUpdate = true;
+            } else {
+                const lastUpdatedData = lastUpdatedDoc.data();
                 
-                if (lastRequestedAt && (now - lastRequestedAt) < thresholdMs) {
-                    skippedCount++;
-                    return null; // Skip this user - recently updated
+                // Check each data type - if any is missing or older than threshold, needs update
+                const dataTypes = ['portfolio', 'tradeHistory', 'socialPosts'];
+                for (const dataType of dataTypes) {
+                    const lastUpdated = lastUpdatedData[dataType];
+                    if (!lastUpdated) {
+                        needsUpdate = true;
+                        break;
+                    }
+                    
+                    const lastUpdatedMs = lastUpdated.toDate?.()?.getTime() || lastUpdated.toMillis?.() || null;
+                    if (lastUpdatedMs && (now - lastUpdatedMs) >= thresholdMs) {
+                        needsUpdate = true;
+                        break;
+                    }
                 }
             }
             
-            // If user has data for today, skip them (they're already up-to-date)
-            if (hasTodayData) {
+            if (needsUpdate) {
+                return target;
+            } else {
                 skippedCount++;
-                return null; // Skip this user - already updated today
+                return null;
             }
-            
-            return target; // Include this user - needs update
         });
         
         const results = await Promise.all(checkPromises);
         const finalTargets = results.filter(t => t !== null);
 
-        logger.log('INFO', `[Core Utils] Found ${targets.length} Signed-In Users. Skipped ${skippedCount} recently updated. ${finalTargets.length} will be updated.`);
+        logger.log('INFO', `[Core Utils] Found ${allUsers.length} Signed-In Users. Skipped ${skippedCount} recently updated. ${finalTargets.length} will be updated.`);
         return finalTargets;
 
     } catch (error) {

package/functions/fetch-popular-investors/helpers/fetch_helpers.js

@@ -129,6 +129,64 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
 
             logger.log('SUCCESS', `[PopularInvestorFetch] Stored ${data.TotalRows} rankings into ${finalRankingsCollectionName}/${today}`);
             
+            // Update the master list of Popular Investors
+            try {
+                const { FieldValue } = require('@google-cloud/firestore');
+                let masterListPath = 'system_state/popular_investor_master_list';
+                
+                if (collectionRegistry && collectionRegistry.getCollectionPath) {
+                    try {
+                        // Get the path from registry
+                        const registryPath = collectionRegistry.getCollectionPath('system', 'popularInvestorMasterList', {});
+                        masterListPath = registryPath;
+                    } catch (e) {
+                        logger.log('WARN', `[PopularInvestorFetch] Failed to get master list path from registry, using default: ${e.message}`);
+                    }
+                }
+                
+                const masterListRef = db.doc(masterListPath);
+                const masterListDoc = await masterListRef.get();
+                
+                const now = new Date();
+                const investorsMap = masterListDoc.exists ? (masterListDoc.data().investors || {}) : {};
+                
+                // Update the master list with all PIs from this fetch
+                for (const item of data.Items) {
+                    const cid = String(item.CustomerId);
+                    const username = item.UserName;
+                    
+                    if (!cid || !username) continue;
+                    
+                    if (!investorsMap[cid]) {
+                        // New PI discovered
+                        investorsMap[cid] = {
+                            cid: cid,
+                            username: username,
+                            firstSeenAt: FieldValue.serverTimestamp(),
+                            lastSeenAt: FieldValue.serverTimestamp()
+                        };
+                    } else {
+                        // Existing PI - update lastSeenAt and username if changed
+                        investorsMap[cid].lastSeenAt = FieldValue.serverTimestamp();
+                        if (username && investorsMap[cid].username !== username) {
+                            investorsMap[cid].username = username;
+                        }
+                    }
+                }
+                
+                // Write the updated master list
+                await masterListRef.set({
+                    investors: investorsMap,
+                    lastUpdated: FieldValue.serverTimestamp(),
+                    totalInvestors: Object.keys(investorsMap).length
+                }, { merge: true });
+                
+                logger.log('SUCCESS', `[PopularInvestorFetch] Updated master list with ${data.Items.length} PIs. Total unique PIs: ${Object.keys(investorsMap).length}`);
+            } catch (masterListError) {
+                logger.log('WARN', `[PopularInvestorFetch] Failed to update master list: ${masterListError.message}`);
+                // Non-critical, continue
+            }
+            
             // Update root data indexer for today's date after rankings data is stored
             try {
                 const { runRootDataIndexer } = require('../../root-data-indexer/index');

package/functions/generic-api/user-api/helpers/sync/user_sync_helpers.js

@@ -424,22 +424,102 @@ async function getUserSyncStatus(req, res, dependencies, config) {
         // No request found, check if user can request
         let canRequest = true;
         let rateLimitInfo = null;
+        let lastUpdatedInfo = null;
         
-            if (requestingUserCid) {
+        // Check last updated times for the target user
+        try {
+            const { collectionRegistry } = dependencies;
+            const HOURS_THRESHOLD = 24;
+            const thresholdMs = HOURS_THRESHOLD * 60 * 60 * 1000;
+            const now = Date.now();
+            
+            // Determine user type (PI or signed-in user)
+            const referrer = req.headers.referer || req.headers.referrer || '';
+            const sourcePage = req.query?.sourcePage || '';
+            const isPI = referrer.includes('/popular-investors/') || sourcePage === 'popular-investor';
+            
+            let lastUpdatedPath;
+            if (collectionRegistry && collectionRegistry.getCollectionPath) {
+                try {
+                    if (isPI) {
+                        lastUpdatedPath = collectionRegistry.getCollectionPath('popularInvestors', 'lastUpdated', { piCid: String(targetCidNum) });
+                    } else {
+                        lastUpdatedPath = collectionRegistry.getCollectionPath('signedInUsers', 'lastUpdated', { cid: String(targetCidNum) });
+                    }
+                } catch (err) {
+                    logger.log('WARN', `[getUserSyncStatus] Failed to get lastUpdated path: ${err.message}`);
+                }
+            }
+            
+            if (lastUpdatedPath) {
+                const lastUpdatedRef = db.doc(lastUpdatedPath);
+                const lastUpdatedDoc = await lastUpdatedRef.get();
+                
+                if (lastUpdatedDoc.exists) {
+                    const lastUpdatedData = lastUpdatedDoc.data();
+                    const dataTypes = ['portfolio', 'tradeHistory', 'socialPosts'];
+                    const allRecent = dataTypes.every(dataType => {
+                        const lastUpdated = lastUpdatedData[dataType];
+                        if (!lastUpdated) return false;
+                        
+                        const lastUpdatedMs = lastUpdated.toDate?.()?.getTime() || lastUpdated.toMillis?.() || null;
+                        return lastUpdatedMs && (now - lastUpdatedMs) < thresholdMs;
+                    });
+                    
+                    if (allRecent) {
+                        // All data types were updated recently - disable sync
+                        canRequest = false;
+                        const oldestUpdate = Math.min(...dataTypes.map(dataType => {
+                            const lastUpdated = lastUpdatedData[dataType];
+                            const lastUpdatedMs = lastUpdated.toDate?.()?.getTime() || lastUpdated.toMillis?.() || null;
+                            return lastUpdatedMs || 0;
+                        }).filter(ms => ms > 0));
+                        
+                        const canRequestAgainAt = new Date(oldestUpdate + thresholdMs).toISOString();
+                        lastUpdatedInfo = {
+                            allDataTypesRecent: true,
+                            canRequestAgainAt,
+                            lastUpdated: {
+                                portfolio: lastUpdatedData.portfolio?.toDate?.()?.toISOString() || null,
+                                tradeHistory: lastUpdatedData.tradeHistory?.toDate?.()?.toISOString() || null,
+                                socialPosts: lastUpdatedData.socialPosts?.toDate?.()?.toISOString() || null
+                            }
+                        };
+                    } else {
+                        // Some data types need updating - allow sync
+                        lastUpdatedInfo = {
+                            allDataTypesRecent: false,
+                            lastUpdated: {
+                                portfolio: lastUpdatedData.portfolio?.toDate?.()?.toISOString() || null,
+                                tradeHistory: lastUpdatedData.tradeHistory?.toDate?.()?.toISOString() || null,
+                                socialPosts: lastUpdatedData.socialPosts?.toDate?.()?.toISOString() || null
+                            }
+                        };
+                    }
+                }
+            }
+        } catch (lastUpdatedError) {
+            logger.log('WARN', `[getUserSyncStatus] Error checking last updated times: ${lastUpdatedError.message}`);
+            // Non-critical, continue with rate limit check
+        }
+        
+        if (requestingUserCid) {
             // Check if this is a developer account (bypass rate limits for developers)
             const { isDeveloperAccount } = require('../dev/dev_helpers');
             const isDeveloper = isDeveloperAccount(requestingUserCid);
             
             if (isDeveloper) {
-                // Developer accounts bypass rate limits
-                canRequest = true;
+                // Developer accounts bypass rate limits (but still respect last updated check)
                 rateLimitInfo = {
                     bypassed: true,
                     reason: 'developer_account'
                 };
             } else {
                 const rateLimitCheck = await checkRateLimits(db, targetCidNum, logger);
-                canRequest = rateLimitCheck.allowed;
+                // Combine rate limit and last updated checks - both must pass
+                if (!rateLimitCheck.allowed) {
+                    canRequest = false;
+                }
                 rateLimitInfo = {
                     canRequestAgainAt: rateLimitCheck.canRequestAgainAt
                 };
@@ -450,7 +530,8 @@ async function getUserSyncStatus(req, res, dependencies, config) {
             success: true,
             status: 'not_requested',
             canRequest,
-            rateLimit: rateLimitInfo
+            rateLimit: rateLimitInfo,
+            lastUpdated: lastUpdatedInfo
         });
         
     } catch (error) {

package/functions/task-engine/handler_creator.js

@@ -228,22 +228,23 @@ async function handleRequest(message, context, configObj, dependencies) {
                 await handleUpdate(data, 'single-update', dependencies, config); 
                 break;
             case 'POPULAR_INVESTOR_UPDATE':
-                // For POPULAR_INVESTOR_UPDATE, the entire payload IS the task data
-                // (not wrapped in a 'data' field like other task types)
-                // Extract task data from payload, excluding 'type'
-                const taskData = data || {
-                    cid: payload.cid,
-                    username: payload.username,
-                    requestId: payload.requestId,
-                    source: payload.source,
-                    requestedBy: payload.requestedBy,
-                    actualRequestedBy: payload.actualRequestedBy,
-                    metadata: payload.metadata,
-                    priority: payload.priority
+                // For POPULAR_INVESTOR_UPDATE, fields are at the payload root level
+                // Merge data object with payload root fields
+                const taskData = {
+                    cid: payload.cid || data?.cid,
+                    username: payload.username || data?.username,
+                    requestId: payload.requestId || data?.requestId,
+                    source: payload.source || data?.source,
+                    requestedBy: payload.requestedBy || data?.requestedBy,
+                    effectiveRequestedBy: payload.effectiveRequestedBy || data?.effectiveRequestedBy,
+                    metadata: payload.metadata || data?.metadata || {},
+                    priority: payload.priority || data?.priority,
+                    // Merge nested data object if it exists
+                    ...(data || {})
                 };
                 
-                if (!taskData || (!taskData.cid && !taskData.username)) {
-                    logger.log('ERROR', '[TaskEngine] POPULAR_INVESTOR_UPDATE missing required fields (cid or username)', { payload, taskData });
+                if (!taskData.cid && !taskData.username) {
+                    logger.log('ERROR', '[TaskEngine] POPULAR_INVESTOR_UPDATE missing required fields (cid or username)', { payload, data, taskData });
                     return;
                 }
                 

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -27,6 +27,50 @@ const {
     storeSignedInUserSocialPosts
 } = require('./data_storage_helpers');
 
+/**
+ * Helper function to update last updated timestamp for a user's data type
+ * @param {object} db - Firestore database instance
+ * @param {object} collectionRegistry - Collection registry helper
+ * @param {string} cid - User CID
+ * @param {string} userType - 'popularInvestor' or 'signedInUser'
+ * @param {string} dataType - 'portfolio', 'tradeHistory', or 'socialPosts'
+ * @param {object} logger - Logger instance
+ */
+async function updateLastUpdatedTimestamp(db, collectionRegistry, cid, userType, dataType, logger) {
+    try {
+        const { FieldValue } = require('@google-cloud/firestore');
+        let lastUpdatedPath;
+        
+        if (collectionRegistry && collectionRegistry.getCollectionPath) {
+            try {
+                if (userType === 'popularInvestor') {
+                    lastUpdatedPath = collectionRegistry.getCollectionPath('popularInvestors', 'lastUpdated', { piCid: cid });
+                } else {
+                    lastUpdatedPath = collectionRegistry.getCollectionPath('signedInUsers', 'lastUpdated', { cid: cid });
+                }
+            } catch (e) {
+                logger.log('WARN', `[LastUpdated] Failed to get path from registry: ${e.message}`);
+                return;
+            }
+        } else {
+            logger.log('WARN', '[LastUpdated] Collection registry not available');
+            return;
+        }
+        
+        const lastUpdatedRef = db.doc(lastUpdatedPath);
+        const updateData = {
+            [dataType]: FieldValue.serverTimestamp(),
+            updatedAt: FieldValue.serverTimestamp()
+        };
+        
+        await lastUpdatedRef.set(updateData, { merge: true });
+        logger.log('INFO', `[LastUpdated] Updated ${dataType} timestamp for ${userType} ${cid}`);
+    } catch (error) {
+        logger.log('WARN', `[LastUpdated] Failed to update timestamp for ${userType} ${cid} ${dataType}: ${error.message}`);
+        // Non-critical, continue
+    }
+}
+
 async function handlePopularInvestorUpdate(taskData, config, dependencies) {
     const { logger, proxyManager, batchManager, headerManager, db, pubsub, collectionRegistry } = dependencies;
     
@@ -37,9 +81,13 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
     }
     
     const { cid, username, requestId, source, metadata } = taskData;
+    const data = taskData.data || {}; // Extract data object
     // Extract targetCid from metadata if present (for optimization)
     const targetCid = metadata?.targetCid || cid;
     
+    // Always fetch social data (for both on-demand and scheduled updates)
+    const includeSocial = data.includeSocial !== false; // Default to true, only skip if explicitly false
+    
     // Validate required fields
     if (!cid && !username) {
         logger.log('ERROR', '[PI Update] Missing required fields: cid or username', { taskData });
@@ -192,6 +240,9 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
             portfolioData
         });
         
+        // Update last updated timestamp for portfolio
+        await updateLastUpdatedTimestamp(db, collectionRegistry, cid, 'popularInvestor', 'portfolio', logger);
+        
         // Mark success for header reporting (at least partial)
         fetchSuccess = true;
 
@@ -354,7 +405,99 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
             logger.log('WARN', `[PI Update] History fetch failed for ${username}. Status: ${historyRes?.status || 'unknown'}`);
         }
 
-        logger.log('SUCCESS', `[PI Update] Completed full update for ${username}`);
+        // Fetch social data (always for all PI updates)
+        let socialFetched = false;
+        if (includeSocial && username) {
+            try {
+                logger.log('INFO', `[PI Update] Fetching social data for ${username} (${cid})...`);
+                
+                // Import social helper functions
+                const { getGcidForUser } = require('../../social-task-handler/helpers/handler_helpers');
+                const { FieldValue } = require('@google-cloud/firestore');
+                
+                // Get GCID for user
+                const gcid = await getGcidForUser(dependencies, config.social || {}, cid, username);
+                
+                // Fetch social posts
+                const userFeedApiUrl = config.social?.userFeedApiUrl || 'https://www.etoro.com/api/edm-streams/v1/feed/user/top/';
+                const fetchUrl = `${userFeedApiUrl}${gcid}?take=10&offset=0&reactionsPageSize=20&badgesExperimentIsEnabled=false&client_request_id=${uuid}`;
+                
+                const socialResponse = await proxyManager.fetch(fetchUrl, requestOptions);
+                if (socialResponse.ok) {
+                    const socialData = await socialResponse.json();
+                    const discussions = socialData?.discussions || [];
+                    
+                    // Process and store social posts using new structure
+                    const processedPostsCollection = config.social?.processedPostsCollectionName || 'social_processed_registry';
+                    const processedRef = db.collection(processedPostsCollection);
+                    
+                    const MAX_POSTS = 30;
+                    const postsToStore = [];
+                    
+                    for (const discussion of discussions.slice(0, MAX_POSTS)) {
+                        const post = discussion.post;
+                        if (!post || !post.id) continue;
+                        
+                        // Check if already processed
+                        const existing = await processedRef.doc(post.id).get();
+                        if (existing.exists) continue;
+                        
+                        // Prepare post data
+                        const postData = {
+                            id: post.id,
+                            postId: post.id,
+                            text: post.message?.text || '',
+                            ownerId: post.owner?.id,
+                            username: post.owner?.username,
+                            createdAt: post.created,
+                            fetchedAt: FieldValue.serverTimestamp(),
+                            snippet: (post.message?.text || '').substring(0, 500),
+                            stats: {
+                                likes: discussion.emotionsData?.like?.paging?.totalCount || 0,
+                                comments: discussion.summary?.totalCommentsAndReplies || 0
+                            },
+                            aiAnalysis: {
+                                overallSentiment: "Neutral",
+                                topics: [],
+                                isSpam: false,
+                                qualityScore: 0.5
+                            },
+                            tags: post.tags?.map(t => t.market?.symbolName).filter(Boolean) || []
+                        };
+                        
+                        postsToStore.push(postData);
+                        
+                        // Mark as processed
+                        await processedRef.doc(post.id).set({ processedAt: FieldValue.serverTimestamp() });
+                    }
+                    
+                    // Store posts using new structure
+                    if (postsToStore.length > 0) {
+                        await storePopularInvestorSocialPosts({
+                            db,
+                            logger,
+                            collectionRegistry,
+                            cid: String(cid),
+                            date: today,
+                            posts: postsToStore
+                        });
+                        
+                        // Update last updated timestamp for social posts
+                        await updateLastUpdatedTimestamp(db, collectionRegistry, String(cid), 'popularInvestor', 'socialPosts', logger);
+                        
+                        socialFetched = true;
+                        logger.log('INFO', `[PI Update] Fetched and stored ${postsToStore.length} social posts for ${username}`);
+                    }
+                } else {
+                    logger.log('WARN', `[PI Update] Social fetch failed for ${username} (status: ${socialResponse.status})`);
+                }
+            } catch (socialError) {
+                logger.log('WARN', `[PI Update] Social fetch failed for ${username}`, socialError);
+                // Continue - social failure shouldn't block portfolio/history processing
+            }
+        }
+
+        logger.log('SUCCESS', `[PI Update] Completed full update for ${username} (Portfolio: ✓, History: ✓, Social: ${socialFetched ? '✓' : '✗'})`);
         
         // Update request status and trigger computation if this is an on-demand request (PI fetch or sync)
         if (requestId && (source === 'on_demand' || source === 'on_demand_sync') && db) {
@@ -395,6 +538,10 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
                     // Get rootDataIndexer config from injected config object
                     const rootDataIndexerConfig = config.rootDataIndexer || {};
                     
+                    // Determine which data types ran
+                    const dataTypesRun = ['piPortfolios', 'piHistory'];
+                    if (socialFetched) dataTypesRun.push('piSocial');
+                    
                     const wasIndexed = await conditionallyRunRootDataIndexer({
                         db,
                         logger,
@@ -402,7 +549,7 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
                         rootDataIndexerConfig,
                         dependencies,
                         counterRef: null, // No counter for on-demand PI updates
-                        dataTypesRun: ['piPortfolios', 'piHistory']
+                        dataTypesRun: dataTypesRun
                     });
                     
                     // Update status to indicate indexing is done (or skipped), computation is being triggered
@@ -522,7 +669,8 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
 async function handleOnDemandUserUpdate(taskData, config, dependencies) {
     const { cid, username, requestId, source, metadata } = taskData;
     const data = taskData.data || {}; // Extract data object
-    const includeSocial = data.includeSocial === true; // Check if social should be fetched
+    // All on-demand requests should fetch social data
+    const includeSocial = data.includeSocial !== false; // Default to true, only skip if explicitly false
     const since = data.since || new Date(Date.now() - (7 * 24 * 60 * 60 * 1000)).toISOString(); // Default to 7 days
     const portfolioOnly = data.portfolioOnly === false ? false : true; // Default to true (fetch portfolio)
     
@@ -701,6 +849,7 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
                         date: today,
                         portfolioData
                     });
+                    await updateLastUpdatedTimestamp(db, collectionRegistry, String(cid), 'popularInvestor', 'portfolio', logger);
                 } else {
                     await storeSignedInUserPortfolio({
                         db,
@@ -710,6 +859,7 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
                         date: today,
                         portfolioData
                     });
+                    await updateLastUpdatedTimestamp(db, collectionRegistry, String(cid), 'signedInUser', 'portfolio', logger);
                 }
             } catch (batchErr) {
                 const errorMsg = `Failed to store portfolio data: ${batchErr.message}`;
@@ -783,14 +933,17 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
                 try {
                     // Use correct storage function based on user type
                     if (userType === 'POPULAR_INVESTOR') {
-                        await storePopularInvestorTradeHistory({
-                            db,
-                            logger,
-                            collectionRegistry,
-                            cid: String(cid),
-                            date: today,
-                            historyData
-                        });
+                await storePopularInvestorTradeHistory({
+                    db,
+                    logger,
+                    collectionRegistry,
+                    cid: String(cid),
+                    date: today,
+                    historyData
+                });
+                
+                // Update last updated timestamp for trade history
+                await updateLastUpdatedTimestamp(db, collectionRegistry, String(cid), 'popularInvestor', 'tradeHistory', logger);
                     } else {
                         await storeSignedInUserTradeHistory({
                             db,
@@ -800,6 +953,7 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
                             date: today,
                             historyData
                         });
+                        await updateLastUpdatedTimestamp(db, collectionRegistry, String(cid), 'signedInUser', 'tradeHistory', logger);
                     }
                     historyFetched = true;
                 } catch (batchErr) {
@@ -832,9 +986,9 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
         logger.log('INFO', `[On-Demand Update] Skipping history fetch (portfolioOnly=false) for ${username}`);
     }
 
-        // Fetch social data - always for on-demand syncs
+        // Fetch social data for on-demand syncs
         let socialFetched = false;
-        if (username) {
+        if (includeSocial && username) {
             try {
                 logger.log('INFO', `[On-Demand Update] Fetching social data for ${username} (${cid})...`);
                 
@@ -919,6 +1073,9 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
                                 date: today,
                                 posts: postsToStore
                             });
+                            
+                            // Update last updated timestamp for social posts
+                            await updateLastUpdatedTimestamp(db, collectionRegistry, String(cid), 'signedInUser', 'socialPosts', logger);
                         }
                         
                         // Update date tracking document (for root data indexer)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.516",
+  "version": "1.0.518",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [