bulltrackers-module 1.0.539 → 1.0.540

package/functions/alert-system/helpers/alert_helpers.js

@@ -186,20 +186,17 @@ function shouldTriggerAlert(subscription, alertTypeId) {
 }
 
 /**
- * Get PI username from rankings or subscriptions
+ * Get PI username from master list (single source of truth)
+ * Falls back to subscriptions if not in master list
  */
 async function getPIUsername(db, piCid) {
   try {
-    // Try to get from rankings first
-    const rankingsRef = db.collection('rankings')
-      .doc('latest')
-      .collection('popular_investors')
-      .doc(String(piCid));
+    // Try to get from master list first (single source of truth)
+    const { getPIUsernameFromMasterList } = require('../../generic-api/user-api/helpers/core/user_status_helpers');
+    const username = await getPIUsernameFromMasterList(db, piCid, null, null);
     
-    const rankingsDoc = await rankingsRef.get();
-    if (rankingsDoc.exists) {
-      const data = rankingsDoc.data();
-      return data.username || `PI-${piCid}`;
+    if (username) {
+      return username;
     }
     
     // Fallback: try to get from any subscription

package/functions/fetch-popular-investors/helpers/fetch_helpers.js

@@ -174,6 +174,7 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
             logger.log('SUCCESS', `[PopularInvestorFetch] Stored ${data.TotalRows} rankings into ${finalRankingsCollectionName}/${today}${firestorePayload._compressed ? ' (compressed)' : ''}`);
             
             // Update the master list of Popular Investors
+            // Use batched writes to avoid 500 field transform limit
             try {
                 const { FieldValue } = require('@google-cloud/firestore');
                 let masterListPath = 'system_state/popular_investor_master_list';
@@ -192,40 +193,97 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
                 const masterListDoc = await masterListRef.get();
                 
                 const now = new Date();
-                const investorsMap = masterListDoc.exists ? (masterListDoc.data().investors || {}) : {};
+                const existingInvestors = masterListDoc.exists ? (masterListDoc.data().investors || {}) : {};
+                const investorsToUpdate = {};
+                let newInvestorsCount = 0;
+                let updatedInvestorsCount = 0;
                 
-                // Update the master list with all PIs from this fetch
+                // Identify which investors need updating (only those in current fetch)
                 for (const item of data.Items) {
                     const cid = String(item.CustomerId);
                     const username = item.UserName;
                     
                     if (!cid || !username) continue;
                     
-                    if (!investorsMap[cid]) {
+                    if (!existingInvestors[cid]) {
                         // New PI discovered
-                        investorsMap[cid] = {
+                        investorsToUpdate[cid] = {
                             cid: cid,
                             username: username,
-                            firstSeenAt: FieldValue.serverTimestamp(),
-                            lastSeenAt: FieldValue.serverTimestamp()
+                            firstSeenAt: now,
+                            lastSeenAt: now
                         };
+                        newInvestorsCount++;
                     } else {
-                        // Existing PI - update lastSeenAt and username if changed
-                        investorsMap[cid].lastSeenAt = FieldValue.serverTimestamp();
-                        if (username && investorsMap[cid].username !== username) {
-                            investorsMap[cid].username = username;
+                        // Existing PI - check if username changed or needs lastSeenAt update
+                        const needsUpdate = existingInvestors[cid].username !== username;
+                        if (needsUpdate) {
+                            investorsToUpdate[cid] = {
+                                ...existingInvestors[cid],
+                                username: username,
+                                lastSeenAt: now
+                            };
+                            updatedInvestorsCount++;
+                        } else {
+                            // Just update lastSeenAt timestamp
+                            investorsToUpdate[cid] = {
+                                ...existingInvestors[cid],
+                                lastSeenAt: now
+                            };
+                            updatedInvestorsCount++;
                         }
                     }
                 }
                 
-                // Write the updated master list
-                await masterListRef.set({
-                    investors: investorsMap,
-                    lastUpdated: FieldValue.serverTimestamp(),
-                    totalInvestors: Object.keys(investorsMap).length
-                }, { merge: true });
+                // Use batched writes to update only changed investors
+                // Since we're using regular Date objects (not serverTimestamp()), we avoid field transform limits
+                // But we still batch to handle large numbers of investors efficiently
+                const BATCH_SIZE = 450; // Firestore allows 500 operations per batch, leave room for metadata
+                const investorsToUpdateEntries = Object.entries(investorsToUpdate);
+                const totalBatches = Math.ceil(investorsToUpdateEntries.length / BATCH_SIZE);
                 
-                logger.log('SUCCESS', `[PopularInvestorFetch] Updated master list with ${data.Items.length} PIs. Total unique PIs: ${Object.keys(investorsMap).length}`);
+                // If document doesn't exist, create it with all investors in first batch
+                if (!masterListDoc.exists && investorsToUpdateEntries.length > 0) {
+                    const batch = db.batch();
+                    const finalInvestorsMap = { ...existingInvestors, ...investorsToUpdate };
+                    batch.set(masterListRef, {
+                        investors: finalInvestorsMap,
+                        lastUpdated: FieldValue.serverTimestamp(),
+                        totalInvestors: Object.keys(finalInvestorsMap).length
+                    }, { merge: true });
+                    await batch.commit();
+                    logger.log('INFO', `[PopularInvestorFetch] Created master list with ${Object.keys(finalInvestorsMap).length} investors`);
+                } else if (investorsToUpdateEntries.length > 0) {
+                    // Document exists - update only changed investors in batches
+                    for (let batchIdx = 0; batchIdx < totalBatches; batchIdx++) {
+                        const batch = db.batch();
+                        const startIdx = batchIdx * BATCH_SIZE;
+                        const endIdx = Math.min(startIdx + BATCH_SIZE, investorsToUpdateEntries.length);
+                        const batchEntries = investorsToUpdateEntries.slice(startIdx, endIdx);
+                        
+                        // Build update object with all investors in this batch
+                        const batchUpdateData = {};
+                        for (const [cid, investorData] of batchEntries) {
+                            batchUpdateData[`investors.${cid}`] = investorData;
+                        }
+                        
+                        // Update metadata (lastUpdated, totalInvestors) only in the last batch
+                        if (batchIdx === totalBatches - 1) {
+                            const finalInvestorsMap = { ...existingInvestors, ...investorsToUpdate };
+                            batchUpdateData.lastUpdated = FieldValue.serverTimestamp();
+                            batchUpdateData.totalInvestors = Object.keys(finalInvestorsMap).length;
+                        }
+                        
+                        batch.update(masterListRef, batchUpdateData);
+                        await batch.commit();
+                        
+                        if (totalBatches > 1) {
+                            logger.log('INFO', `[PopularInvestorFetch] Updated master list batch ${batchIdx + 1}/${totalBatches} (${batchEntries.length} investors)`);
+                        }
+                    }
+                }
+                
+                logger.log('SUCCESS', `[PopularInvestorFetch] Updated master list: ${newInvestorsCount} new, ${updatedInvestorsCount} updated. Total unique PIs: ${Object.keys({ ...existingInvestors, ...investorsToUpdate }).length}`);
             } catch (masterListError) {
                 logger.log('WARN', `[PopularInvestorFetch] Failed to update master list: ${masterListError.message}`);
                 // Non-critical, continue

package/functions/generic-api/user-api/helpers/core/user_status_helpers.js

@@ -5,6 +5,120 @@
 
 const { findLatestRankingsDate } = require('./data_lookup_helpers');
 
+/**
+ * Get Popular Investor master list from Firestore
+ * @param {Firestore} db - Firestore instance
+ * @param {object} collectionRegistry - Collection registry (optional)
+ * @param {object} logger - Logger instance (optional)
+ * @returns {Promise<object>} - Master list investors map { cid: { cid, username, firstSeenAt, lastSeenAt } }
+ */
+async function getPIMasterList(db, collectionRegistry = null, logger = null) {
+    try {
+        let masterListPath = 'system_state/popular_investor_master_list';
+        
+        if (collectionRegistry && collectionRegistry.getCollectionPath) {
+            try {
+                masterListPath = collectionRegistry.getCollectionPath('system', 'popularInvestorMasterList', {});
+            } catch (err) {
+                if (logger) {
+                    logger.log('WARN', `[getPIMasterList] Failed to get master list path from registry, using default: ${err.message}`);
+                }
+            }
+        }
+        
+        const masterListRef = db.doc(masterListPath);
+        const masterListDoc = await masterListRef.get();
+        
+        if (!masterListDoc.exists) {
+            return {};
+        }
+        
+        const masterListData = masterListDoc.data();
+        return masterListData.investors || {};
+    } catch (error) {
+        if (logger) {
+            logger.log('ERROR', `[getPIMasterList] Error loading master list: ${error.message}`);
+        }
+        return {};
+    }
+}
+
+/**
+ * Get PI username from CID using master list
+ * @param {Firestore} db - Firestore instance
+ * @param {string|number} piCid - Popular Investor CID
+ * @param {object} collectionRegistry - Collection registry (optional)
+ * @param {object} logger - Logger instance (optional)
+ * @returns {Promise<string|null>} - Username or null if not found
+ */
+async function getPIUsernameFromMasterList(db, piCid, collectionRegistry = null, logger = null) {
+    try {
+        const investors = await getPIMasterList(db, collectionRegistry, logger);
+        const cidStr = String(piCid);
+        const investor = investors[cidStr];
+        
+        if (investor && investor.username) {
+            return investor.username;
+        }
+        
+        return null;
+    } catch (error) {
+        if (logger) {
+            logger.log('ERROR', `[getPIUsernameFromMasterList] Error getting username for PI ${piCid}: ${error.message}`);
+        }
+        return null;
+    }
+}
+
+/**
+ * Get PI CID from username using master list
+ * @param {Firestore} db - Firestore instance
+ * @param {string} username - Popular Investor username
+ * @param {object} collectionRegistry - Collection registry (optional)
+ * @param {object} logger - Logger instance (optional)
+ * @returns {Promise<string|null>} - CID or null if not found
+ */
+async function getPICidFromMasterList(db, username, collectionRegistry = null, logger = null) {
+    try {
+        const investors = await getPIMasterList(db, collectionRegistry, logger);
+        const searchUsername = username.toLowerCase().trim();
+        
+        for (const [cid, investor] of Object.entries(investors)) {
+            if (investor.username && investor.username.toLowerCase().trim() === searchUsername) {
+                return cid;
+            }
+        }
+        
+        return null;
+    } catch (error) {
+        if (logger) {
+            logger.log('ERROR', `[getPICidFromMasterList] Error getting CID for username ${username}: ${error.message}`);
+        }
+        return null;
+    }
+}
+
+/**
+ * Check if a CID is a Popular Investor using master list
+ * @param {Firestore} db - Firestore instance
+ * @param {string|number} cid - User CID
+ * @param {object} collectionRegistry - Collection registry (optional)
+ * @param {object} logger - Logger instance (optional)
+ * @returns {Promise<boolean>} - True if PI, false otherwise
+ */
+async function isPopularInvestor(db, cid, collectionRegistry = null, logger = null) {
+    try {
+        const investors = await getPIMasterList(db, collectionRegistry, logger);
+        const cidStr = String(cid);
+        return investors.hasOwnProperty(cidStr);
+    } catch (error) {
+        if (logger) {
+            logger.log('ERROR', `[isPopularInvestor] Error checking if ${cid} is PI: ${error.message}`);
+        }
+        return false;
+    }
+}
+
 /**
  * Check if a signed-in user is also a Popular Investor
  * Returns ranking entry if found, null otherwise
@@ -43,31 +157,28 @@ async function checkIfUserIsPI(db, userCid, config, logger = null) {
             return fakeRankEntry;
         }
         
-        // Otherwise, check real rankings
-        const rankingsCollection = config.popularInvestorRankingsCollection || process.env.FIRESTORE_COLLECTION_PI_RANKINGS || 'popular_investor_rankings';
-        const rankingsDate = await findLatestRankingsDate(db, rankingsCollection, 30);
+        // Otherwise, check master list (single source of truth)
+        const collectionRegistry = config.collectionRegistry || null;
+        const isPI = await isPopularInvestor(db, userCid, collectionRegistry, logger);
         
-        if (!rankingsDate) {
+        if (!isPI) {
             return null;
         }
         
-        const rankingsRef = db.collection(rankingsCollection).doc(rankingsDate);
-        const rankingsDoc = await rankingsRef.get();
+        // Get username from master list
+        const username = await getPIUsernameFromMasterList(db, userCid, collectionRegistry, logger);
         
-        if (!rankingsDoc.exists) {
+        if (!username) {
             return null;
         }
         
-        const rawRankingsData = rankingsDoc.data();
-        // Decompress if needed
-        const { tryDecompress } = require('./compression_helpers');
-        const rankingsData = tryDecompress(rawRankingsData);
-        const rankingsItems = rankingsData.Items || [];
-        
-        // Find user in rankings
-        const userRankEntry = rankingsItems.find(item => String(item.CustomerId) === String(userCid));
-        
-        return userRankEntry || null;
+        // Return a ranking-like entry for compatibility (minimal data)
+        // Note: Full ranking data (AUM, copiers, etc.) is not in master list
+        // If full ranking data is needed, it should be fetched from rankings collection separately
+        return {
+            CustomerId: Number(userCid),
+            UserName: username
+        };
     } catch (error) {
         console.error('[checkIfUserIsPI] Error checking if user is PI:', error);
         return null;
@@ -75,6 +186,10 @@ async function checkIfUserIsPI(db, userCid, config, logger = null) {
 }
 
 module.exports = {
-    checkIfUserIsPI
+    checkIfUserIsPI,
+    getPIMasterList,
+    getPIUsernameFromMasterList,
+    getPICidFromMasterList,
+    isPopularInvestor
 };
 

package/functions/generic-api/user-api/helpers/fetch/on_demand_fetch_helpers.js

@@ -501,49 +501,23 @@ async function checkRateLimits(db, userCid, piCid, logger) {
 }
 
 /**
- * Get PI username from rankings
- * Uses latest available rankings date (with fallback)
+ * Get PI username from master list (single source of truth)
+ * Uses the popular investor master list instead of rankings
  */
 async function getPiUsername(db, piCid, config, logger) {
     try {
-        // Import the helper function from data_helpers
-        const { findLatestRankingsDate } = require('../data_helpers');
+        // Use master list helper instead of rankings
+        const { getPIUsernameFromMasterList } = require('../core/user_status_helpers');
+        const collectionRegistry = config.collectionRegistry || null;
         
-        const rankingsCollection = config.popularInvestorRankingsCollection || process.env.FIRESTORE_COLLECTION_PI_RANKINGS || 'popular_investor_rankings';
-        const rankingsDate = await findLatestRankingsDate(db, rankingsCollection, 30);
+        const username = await getPIUsernameFromMasterList(db, piCid, collectionRegistry, logger);
         
-        if (!rankingsDate) {
-            logger.log('WARN', `[getPiUsername] No rankings data found (checked last 30 days) for PI ${piCid}`);
-            return null;
-        }
-        
-        // Fetch rankings document for the latest available date
-        const rankingsRef = db.collection(rankingsCollection).doc(rankingsDate);
-        const rankingsDoc = await rankingsRef.get();
-        
-        if (!rankingsDoc.exists) {
-            logger.log('WARN', `[getPiUsername] Rankings doc does not exist for date ${rankingsDate}`);
-            return null;
-        }
-        
-        const rawRankingsData = rankingsDoc.data();
-        // Decompress if needed
-        const rankingsData = tryDecompress(rawRankingsData);
-        const rankingsItems = rankingsData.Items || [];
-        
-        // Search for the PI in the rankings items
-        const piCidNum = Number(piCid);
-        const rankingEntry = rankingsItems.find(item => Number(item.CustomerId) === piCidNum);
-        
-        if (rankingEntry) {
-            const username = rankingEntry.UserName || rankingEntry.username || null;
-            if (username) {
-                logger.log('INFO', `[getPiUsername] Found username "${username}" for PI ${piCid} in rankings date ${rankingsDate}`);
-            }
+        if (username) {
+            logger.log('INFO', `[getPiUsername] Found username "${username}" for PI ${piCid} from master list`);
             return username;
         }
         
-        logger.log('WARN', `[getPiUsername] PI ${piCid} not found in rankings date ${rankingsDate}`);
+        logger.log('WARN', `[getPiUsername] PI ${piCid} not found in master list`);
         return null;
     } catch (error) {
         logger.log('ERROR', `[getPiUsername] Error fetching username for PI ${piCid}`, error);

package/functions/generic-api/user-api/helpers/search/pi_search_helpers.js

@@ -3,8 +3,7 @@
  * Handles PI search endpoints
  */
 
-const { findLatestRankingsDate } = require('../core/data_lookup_helpers');
-const { tryDecompress } = require('../core/compression_helpers');
+const { getPIMasterList } = require('../core/user_status_helpers');
 
 /**
  * GET /user/search/pis
@@ -19,41 +18,35 @@ async function searchPopularInvestors(req, res, dependencies, config) {
     }
     
     try {
-        const rankingsCollection = config.popularInvestorRankingsCollection || process.env.FIRESTORE_COLLECTION_PI_RANKINGS || 'popular_investor_rankings';
-        const rankingsDate = await findLatestRankingsDate(db, rankingsCollection, 30);
+        // Use master list instead of rankings (single source of truth)
+        const collectionRegistry = dependencies.collectionRegistry || null;
+        const investors = await getPIMasterList(db, collectionRegistry, logger);
         
-        if (!rankingsDate) {
-            return res.status(404).json({ error: "Rankings data not available" });
+        if (Object.keys(investors).length === 0) {
+            return res.status(404).json({ error: "Popular investor data not available" });
         }
         
-        const rankingsRef = db.collection(rankingsCollection).doc(rankingsDate);
-        const rankingsDoc = await rankingsRef.get();
-        
-        if (!rankingsDoc.exists) {
-            return res.status(404).json({ error: "Rankings data not available" });
-        }
-        
-        const rawRankingsData = rankingsDoc.data();
-        // Decompress if needed
-        const rankingsData = tryDecompress(rawRankingsData);
-        const rankingsItems = rankingsData.Items || [];
-        
         // Search by username (case-insensitive, partial match)
         const searchQuery = query.toLowerCase().trim();
-        const matches = rankingsItems
-            .filter(item => {
-                const username = (item.UserName || '').toLowerCase();
-                return username.includes(searchQuery);
-            })
-            .slice(0, parseInt(limit))
-            .map(item => ({
-                cid: item.CustomerId,
-                username: item.UserName,
-                aum: item.AUMValue,
-                riskScore: item.RiskScore,
-                gain: item.Gain,
-                copiers: item.Copiers
-            }));
+        const matches = [];
+        
+        for (const [cid, investor] of Object.entries(investors)) {
+            if (investor.username) {
+                const username = investor.username.toLowerCase();
+                if (username.includes(searchQuery)) {
+                    matches.push({
+                        cid: Number(cid),
+                        username: investor.username
+                        // Note: AUM, riskScore, gain, copiers are not in master list
+                        // If needed, they should be fetched from rankings collection separately
+                    });
+                    
+                    if (matches.length >= parseInt(limit)) {
+                        break;
+                    }
+                }
+            }
+        }
         
         return res.status(200).json({
             results: matches,

package/functions/generic-api/user-api/helpers/watchlist/watchlist_generation_helpers.js

@@ -4,10 +4,11 @@
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
-const { findLatestPortfolioDate, findLatestComputationDate, findLatestRankingsDate } = require('../core/data_lookup_helpers');
+const { findLatestPortfolioDate, findLatestComputationDate } = require('../core/data_lookup_helpers');
 const { tryDecompress } = require('../core/compression_helpers');
 const { getEffectiveCid, getCopiedPIsWithDevOverride } = require('../dev/dev_helpers');
 const { writeWithMigration } = require('../core/path_resolution_helpers');
+const { getPIMasterList, getPIUsernameFromMasterList } = require('../core/user_status_helpers');
 
 /**
  * POST /user/me/watchlist/auto-generate
@@ -146,32 +147,9 @@ async function autoGenerateWatchlist(req, res, dependencies, config) {
             });
         }
         
-        // Fetch rankings to get usernames
-        const rankingsCollection = config.popularInvestorRankingsCollection || process.env.FIRESTORE_COLLECTION_PI_RANKINGS || 'popular_investor_rankings';
-        const rankingsDate = await findLatestRankingsDate(db, rankingsCollection, 30);
-        
-        if (!rankingsDate) {
-            return res.status(404).json({ error: "Rankings data not available" });
-        }
-        
-        const rankingsRef = db.collection(rankingsCollection).doc(rankingsDate);
-        const rankingsDoc = await rankingsRef.get();
-        
-        if (!rankingsDoc.exists) {
-            return res.status(404).json({ error: "Rankings data not available" });
-        }
-        
-        const rawRankingsData = rankingsDoc.data();
-        // Decompress if needed
-        const { tryDecompress } = require('../core/compression_helpers');
-        const rankingsData = tryDecompress(rawRankingsData);
-        const rankingsItems = rankingsData.Items || [];
-        const rankingsMap = new Map();
-        for (const item of rankingsItems) {
-            if (item.CustomerId) {
-                rankingsMap.set(String(item.CustomerId), item);
-            }
-        }
+        // Get usernames from master list (single source of truth)
+        const collectionRegistry = dependencies.collectionRegistry || null;
+        const investors = await getPIMasterList(db, collectionRegistry, logger);
         
         // Create watchlist items
         let matchedCount = 0;
@@ -179,10 +157,15 @@ async function autoGenerateWatchlist(req, res, dependencies, config) {
         
         for (const copiedPI of copiedPIs) {
             const cidStr = String(copiedPI.cid);
-            const rankEntry = rankingsMap.get(cidStr);
-            const username = rankEntry?.UserName || copiedPI.username || 'Unknown';
+            // Try to get username from master list
+            const username = await getPIUsernameFromMasterList(db, cidStr, collectionRegistry, logger) 
+                || copiedPI.username 
+                || 'Unknown';
             
-            if (rankEntry) matchedCount++;
+            // Check if PI exists in master list
+            if (investors[cidStr]) {
+                matchedCount++;
+            }
             
             watchlistItems.push({
                 cid: Number(cidStr),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.539",
+  "version": "1.0.540",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [