bulltrackers-module 1.0.535 → 1.0.537

package/functions/computation-system/context/ContextFactory.js

@@ -27,7 +27,8 @@ class ContextFactory {
             allRankings, allRankingsYesterday,   // Global rank lists
             allVerifications,
             // [NEW] New Root Data Types for Profile Metrics
-            ratings, pageViews, watchlistMembership, alertHistory
+            ratings, pageViews, watchlistMembership, alertHistory,
+            piMasterList, // [NEW]
         } = options;
         
         return {
@@ -56,7 +57,8 @@ class ContextFactory {
                 ratings: ratings || {},
                 pageViews: pageViews || {},
                 watchlistMembership: watchlistMembership || {},
-                alertHistory: alertHistory || {}
+                alertHistory: alertHistory || {},
+                piMasterList: piMasterList || {}, // [NEW]
             }
         };
     }

package/functions/computation-system/data/AvailabilityChecker.js

@@ -229,7 +229,10 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
                     piRatings:          !!details.piRatings,
                     piPageViews:        !!details.piPageViews,
                     watchlistMembership: !!details.watchlistMembership,
-                    piAlertHistory:     !!details.piAlertHistory
+                    piAlertHistory:     !!details.piAlertHistory,
+
+                    // [NEW] Global Helper Data
+                    piMasterList: !!details.piMasterList
                 },
                 portfolioRefs: null, 
                 historyRefs: null,

package/functions/computation-system/data/CachedDataLoader.js

@@ -14,7 +14,8 @@ const {
     loadPIRatings,                // [NEW]
     loadPIPageViews,              // [NEW]
     loadWatchlistMembership: loadWatchlistMembershipData, // [NEW] Renamed to avoid conflict
-    loadPIAlertHistory            // [NEW]
+    loadPIAlertHistory,            // [NEW]
+    loadPopularInvestorMasterList // [NEW]
 } = require('../utils/data_loader');
 const zlib = require('zlib'); 
 
@@ -31,7 +32,8 @@ class CachedDataLoader {
             ratings: new Map(),   // [NEW]
             pageViews: new Map(), // [NEW]
             watchlistMembership: new Map(), // [NEW]
-            alertHistory: new Map() // [NEW]
+            alertHistory: new Map(),// [NEW]
+            piMasterList: null // [NEW] Singleton cache (not date dependent)
         }; 
     }
 
@@ -141,6 +143,13 @@ class CachedDataLoader {
             return {}; 
         } 
     }
+    // [NEW] Load PI Master List (Global, Cached)
+    async loadPIMasterList() {
+        if (this.cache.piMasterList) return this.cache.piMasterList;
+        const data = await loadPopularInvestorMasterList(this.config, this.deps);
+        this.cache.piMasterList = data;
+        return data;
+    }
 }
 
 module.exports = { CachedDataLoader };

package/functions/computation-system/executors/StandardExecutor.js

@@ -5,7 +5,7 @@
  * }
  */
 const { normalizeName, getEarliestDataDates }                          = require('../utils/utils');
-const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs } = require('../utils/data_loader');
+const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs, getHistoryPartRefs } = require('../utils/data_loader');
 const { CachedDataLoader }                                             = require('../data/CachedDataLoader');
 const { ContextFactory }                                               = require('../context/ContextFactory');
 const { commitResults }                                                = require('../persistence/ResultCommitter');
@@ -24,15 +24,31 @@ class StandardExecutor {
             const type = (c.userType || 'ALL').toUpperCase();
             requiredUserTypes.add(type);
         });
-        // If any calc requires 'ALL' (or has no userType), we fetch everything
         const userTypeArray = requiredUserTypes.has('ALL') ? null : Array.from(requiredUserTypes);
         
+        // [OPTIMIZATION] Check for Target CID in manifests (On-Demand Optimization)
+        // If present, we will filter all data streams to strictly this user
+        const targetCid = calcs.find(c => c.targetCid)?.targetCid || calcs.find(c => c.manifest?.targetCid)?.manifest?.targetCid;
+        if (targetCid) {
+            logger.log('INFO', `[StandardExecutor] Running in Targeted Mode for CID: ${targetCid}`);
+        }
+
         const fullRoot = { ...rootData };
         if (calcs.some(c => c.isHistorical)) {
             const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
             const prevStr = prev.toISOString().slice(0, 10);
-            // [FIX] Pass userTypeArray to filter yesterday's data loading
-            fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr, userTypeArray);
+            
+            // Fetch yesterday's refs
+            let yRefs = await getPortfolioPartRefs(config, deps, prevStr, userTypeArray);
+            
+            // [OPTIMIZATION] Filter Yesterday's Refs if targetCid is set
+            if (targetCid && yRefs) {
+                const originalCount = yRefs.length;
+                yRefs = yRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
+                logger.log('INFO', `[StandardExecutor] Filtered Yesterday's Refs: ${originalCount} -> ${yRefs.length}`);
+            }
+            
+            fullRoot.yesterdayPortfolioRefs = yRefs;
         }
 
         const state = {};
@@ -46,18 +62,48 @@ class StandardExecutor {
             } catch (e) { logger.log('WARN', `Failed to init ${c.name}`); } 
         }
 
-        return await StandardExecutor.streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite, userTypeArray);
+        // Pass targetCid to streamAndProcess
+        return await StandardExecutor.streamAndProcess(
+            dStr, state, passName, config, deps, fullRoot, 
+            rootData.portfolioRefs, rootData.historyRefs, 
+            fetchedDeps, previousFetchedDeps, skipStatusWrite, 
+            userTypeArray, targetCid
+        );
     }
 
-    // [UPDATED] Accepts requiredUserTypes param
-    static async streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite, requiredUserTypes = null) {
+    // [UPDATED] Added targetCid param
+    static async streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite, requiredUserTypes = null, targetCid = null) {
         const { logger } = deps;
         const calcs = Object.values(state).filter(c => c && c.manifest);
         const streamingCalcs = calcs.filter(c => c.manifest.rootDataDependencies.includes('portfolio') || c.manifest.rootDataDependencies.includes('history'));
         
         if (streamingCalcs.length === 0) return { successUpdates: {}, failureReport: [] };
+
+        // --- 1. Resolve and Filter Portfolio Refs (Today) ---
+        let effectivePortfolioRefs = portfolioRefs;
+        if (!effectivePortfolioRefs) {
+            // If refs weren't provided by AvailabilityChecker, fetch them now
+            effectivePortfolioRefs = await getPortfolioPartRefs(config, deps, dateStr, requiredUserTypes);
+        }
+        if (targetCid && effectivePortfolioRefs) {
+            // Filter: Keep only refs that match the CID (or Legacy refs without CID)
+            effectivePortfolioRefs = effectivePortfolioRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
+        }
+
+        // --- 2. Resolve and Filter History Refs ---
+        let effectiveHistoryRefs = historyRefs;
+        const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
         
-        let totalReadOps = (portfolioRefs?.length || 0) + (historyRefs?.length || 0);
+        if (needsTradingHistory) {
+            if (!effectiveHistoryRefs) {
+                effectiveHistoryRefs = await getHistoryPartRefs(config, deps, dateStr, requiredUserTypes);
+            }
+            if (targetCid && effectiveHistoryRefs) {
+                effectiveHistoryRefs = effectiveHistoryRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
+            }
+        }
+        
+        let totalReadOps = (effectivePortfolioRefs?.length || 0) + (effectiveHistoryRefs?.length || 0);
         if (rootData.yesterdayPortfolioRefs) totalReadOps += rootData.yesterdayPortfolioRefs.length;
         totalReadOps += 2; 
 
@@ -83,6 +129,7 @@ class StandardExecutor {
         const setupDuration = performance.now() - startSetup;
         Object.keys(executionStats).forEach(name => executionStats[name].timings.setup += setupDuration);
 
+        // Yesterday's Refs are already filtered in run()
         const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
         const prevDateStr = prevDate.toISOString().slice(0, 10);
         
@@ -91,17 +138,13 @@ class StandardExecutor {
             earliestDates = await getEarliestDataDates(config, deps);
         }
 
-        // [FIX] Pass requiredUserTypes to streamPortfolioData
-        // Note: portfolioRefs (for today) might be null if not pre-fetched, allowing streamPortfolioData to fetch filtered refs
-        const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs, requiredUserTypes);
+        // [FIX] Use effective/filtered refs
+        const tP_iter = streamPortfolioData(config, deps, dateStr, effectivePortfolioRefs, requiredUserTypes);
         
         const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
-        // Yesterday's refs were already filtered in run(), so we pass them directly
         const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs)  ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
         
-        const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-        // [FIX] Pass requiredUserTypes to streamHistoryData
-        const tH_iter = (needsTradingHistory) ? streamHistoryData(config, deps, dateStr, historyRefs, requiredUserTypes) : null;
+        const tH_iter = (needsTradingHistory) ? streamHistoryData(config, deps, dateStr, effectiveHistoryRefs, requiredUserTypes) : null;
 
         let yP_chunk = {}, tH_chunk = {};
         let usersSinceLastFlush = 0;
@@ -159,7 +202,7 @@ class StandardExecutor {
     
     // ... rest of the file (flushBuffer, mergeReports, executePerUser) ...
     static async flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, mode, skipStatusWrite, isInitialWrite = false) {
-        // ... (No changes to flushBuffer)
+        const { logger } = deps;
         const transformedState = {};
         for (const [name, inst] of Object.entries(state)) {
             const rawResult = inst.results || {};
@@ -197,7 +240,6 @@ class StandardExecutor {
     }
 
     static mergeReports(successAcc, failureAcc, newResult) {
-        // ... (No changes to mergeReports)
         if (!newResult) return;
         for (const [name, update] of Object.entries(newResult.successUpdates)) {
             if (!successAcc[name]) {
@@ -230,24 +272,25 @@ class StandardExecutor {
    static async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps, config, deps, loader, stats, earliestDates) {
         const { logger } = deps;
         const targetUserType = metadata.userType; 
+        // [NEW] Always load Global Helpers
         const mappings = await loader.loadMappings();
+        const piMasterList = await loader.loadPopularInvestorMasterList(config, deps); // [NEW] Loaded globally
         const SCHEMAS = mathLayer.SCHEMAS;
 
         // 1. Load Root Data
         const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
-        const verifications = metadata.rootDataDependencies?.includes('verification') ? await loader.loadVerifications() : null;
-        const rankings = metadata.rootDataDependencies?.includes('rankings') ? await loader.loadRankings(dateStr) : null;
+        const verifications = metadata.rootDataDependencies?.includes('verification') ? await loader.loadVerificationProfiles(config, deps) : null;
+        const rankings = metadata.rootDataDependencies?.includes('rankings') ? await loader.loadPopularInvestorRankings(config, deps, dateStr) : null;
         
         // [FIX] Load Yesterday's Rankings if isHistorical is true
         let yesterdayRankings = null;
         if (metadata.rootDataDependencies?.includes('rankings') && metadata.isHistorical) {
              const prevDate = new Date(dateStr); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
              const prevStr = prevDate.toISOString().slice(0, 10);
-             // Assuming CachedDataLoader handles caching for efficiency
-             yesterdayRankings = await loader.loadRankings(prevStr); 
+             yesterdayRankings = await loader.loadPopularInvestorRankings(config, deps, prevStr); 
         }
 
-        const socialContainer = metadata.rootDataDependencies?.includes('social') ? await loader.loadSocial(dateStr) : null;
+        const socialContainer = metadata.rootDataDependencies?.includes('social') ? await loader.loadDailySocialPostInsights(config, deps, dateStr) : null;
         
         // [NEW] Load New Root Data Types for Profile Metrics
         // [FIX] Enforce canHaveMissingRoots
@@ -256,14 +299,13 @@ class StandardExecutor {
         let ratings = null;
         if (metadata.rootDataDependencies?.includes('ratings')) {
             try {
-                ratings = await loader.loadRatings(dateStr);
+                ratings = await loader.loadPIRatings(config, deps, dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'ratings' failed to load for ${metadata.name}: ${e.message}`);
                 }
                 ratings = null;
             }
-            // Check if result is null (clean fetch but empty/missing doc)
             if (!ratings && !allowMissing) {
                 throw new Error(`[StandardExecutor] Required root 'ratings' is missing for ${metadata.name}`);
             }
@@ -272,7 +314,7 @@ class StandardExecutor {
         let pageViews = null;
         if (metadata.rootDataDependencies?.includes('pageViews')) {
             try {
-                pageViews = await loader.loadPageViews(dateStr);
+                pageViews = await loader.loadPIPageViews(config, deps, dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'pageViews' failed to load for ${metadata.name}: ${e.message}`);
@@ -287,7 +329,7 @@ class StandardExecutor {
         let watchlistMembership = null;
         if (metadata.rootDataDependencies?.includes('watchlist')) {
             try {
-                watchlistMembership = await loader.loadWatchlistMembership(dateStr);
+                watchlistMembership = await loader.loadWatchlistMembership(config, deps, dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'watchlist' failed to load for ${metadata.name}: ${e.message}`);
@@ -302,7 +344,7 @@ class StandardExecutor {
         let alertHistory = null;
         if (metadata.rootDataDependencies?.includes('alerts')) {
             try {
-                alertHistory = await loader.loadAlertHistory(dateStr);
+                alertHistory = await loader.loadPIAlertHistory(config, deps, dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'alerts' failed to load for ${metadata.name}: ${e.message}`);
@@ -348,7 +390,6 @@ class StandardExecutor {
             
             const userVerification = verifications ? verifications[userId] : null;
             
-            // [FIX] Extract current AND yesterday's rank entry for this user
             const userRanking = rankings ? (rankings.find(r => String(r.CustomerId) === String(userId)) || null) : null;
             const userRankingYesterday = yesterdayRankings ? (yesterdayRankings.find(r => String(r.CustomerId) === String(userId)) || null) : null;
 
@@ -371,21 +412,20 @@ class StandardExecutor {
                 config, deps,
                 verification: userVerification, 
                 
-                // [FIX] Pass both ranking entries
                 rankings: userRanking,
                 yesterdayRankings: userRankingYesterday, 
                 
-                // [FIX] Pass both global lists
                 allRankings: rankings,
                 allRankingsYesterday: yesterdayRankings,
                 
                 allVerifications: verifications,
                 
-                // [NEW] Pass New Root Data Types for Profile Metrics
                 ratings: ratings || {},
                 pageViews: pageViews || {},
                 watchlistMembership: watchlistMembership || {},
-                alertHistory: alertHistory || {}
+                alertHistory: alertHistory || {},
+
+                piMasterList,
             });
 
             if (metadata.requiresEarliestDataDate && earliestDates) {

package/functions/computation-system/layers/extractors.js

@@ -947,6 +947,39 @@ class AlertHistoryExtractor {
     }
 }
 
+/**
+ * [NEW] Extractor for Popular Investor Master List (Global Backup)
+ * Access via: context.globalData.piMasterList
+ */
+class PIMasterListExtractor {
+    /**
+     * Get the master record for a PI
+     * @param {Object} masterList - context.globalData.piMasterList
+     * @param {string} cid - The User CID
+     */
+    static getRecord(masterList, cid) {
+        if (!masterList || !cid) return null;
+        return masterList[String(cid)] || null;
+    }
+
+    /**
+     * Resolve Username from Master List
+     * @param {Object} masterList 
+     * @param {string} cid 
+     */
+    static getUsername(masterList, cid) {
+        const record = this.getRecord(masterList, cid);
+        return record ? record.username : null;
+    }
+
+    /**
+     * Check if CID exists in master list
+     */
+    static exists(masterList, cid) {
+        return !!this.getRecord(masterList, cid);
+    }
+}
+
 module.exports = { 
     DataExtractor, 
     priceExtractor, 
@@ -962,5 +995,6 @@ module.exports = {
     RatingsExtractor,
     PageViewsExtractor,
     WatchlistMembershipExtractor,
-    AlertHistoryExtractor
+    AlertHistoryExtractor,
+    PIMasterListExtractor
 };

package/functions/computation-system/utils/data_loader.js

@@ -774,6 +774,37 @@ async function loadPIAlertHistory(config, deps, dateString) {
     }
 }
 
+// [NEW] Load Popular Investor Master List
+async function loadPopularInvestorMasterList(config, deps) {
+    const { db, logger, calculationUtils } = deps;
+    const { withRetry } = calculationUtils;
+    
+    // Default to 'system_state' collection, 'popular_investor_master_list' doc
+    const collectionName = config.piMasterListCollection || 'system_state';
+    const docId = config.piMasterListDocId || 'popular_investor_master_list';
+
+    logger.log('INFO', `Loading Popular Investor Master List from ${collectionName}/${docId}`);
+
+    try {
+        const docRef = db.collection(collectionName).doc(docId);
+        const docSnap = await withRetry(() => docRef.get(), 'getPIMasterList');
+
+        if (!docSnap.exists) {
+            logger.log('WARN', 'Popular Investor Master List not found.');
+            return {};
+        }
+
+        const data = tryDecompress(docSnap.data());
+        // Structure is { investors: { cid: { username, ... } } } or direct map { cid: { ... } }
+        // Based on user input, it looks like a direct map of CIDs or a field holding the map.
+        // We return the raw object which acts as the map.
+        return data.investors || data; 
+    } catch (error) {
+        logger.log('ERROR', `Failed to load PI Master List: ${error.message}`);
+        return {};
+    }
+}
+
 module.exports = {
     getPortfolioPartRefs, 
     loadDataByRefs, 
@@ -791,5 +822,6 @@ module.exports = {
     loadPIRatings,
     loadPIPageViews,
     loadWatchlistMembership,
-    loadPIAlertHistory
+    loadPIAlertHistory,
+    loadPopularInvestorMasterList // [NEW]
 };

package/functions/root-data-indexer/index.js

@@ -93,6 +93,7 @@ exports.runRootDataIndexer = async (config, dependencies) => {
         piPageViews: collections.piPageViews || 'PIPageViewsData',
         watchlistMembership: collections.watchlistMembership || 'WatchlistMembershipData',
         piAlertHistory: collections.piAlertHistory || 'PIAlertHistoryData',
+        piMasterList: collections.piMasterList || 'system_state', // [NEW] Collection for master list
         ...collections // Allow overrides
     };
 
@@ -237,7 +238,8 @@ exports.runRootDataIndexer = async (config, dependencies) => {
                     piRatings:             false,
                     piPageViews:           false,
                     watchlistMembership:   false,
-                    piAlertHistory:        false
+                    piAlertHistory:        false,
+                    piMasterList:          false // [NEW] Flag for Master List
                 }
             };
 
@@ -312,6 +314,10 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             // Path: PIAlertHistoryData/{YYYY-MM-DD}
             const piAlertHistoryRef = db.collection(safeCollections.piAlertHistory).doc(dateStr);
 
+            // [NEW] Master List Ref (Single Global Document)
+            // Path: system_state/popular_investor_master_list
+            const piMasterListRef = db.collection(safeCollections.piMasterList).doc('popular_investor_master_list');
+
             // 4. Social Data Checks - Use date tracking documents (NEW STRUCTURE)
             // Single tracking documents at root level:
             // - PopularInvestorSocialPostData/_dates -> fetchedDates.{date}
@@ -387,7 +393,8 @@ exports.runRootDataIndexer = async (config, dependencies) => {
                 piRatingsSnap,
                 piPageViewsSnap,
                 watchlistMembershipSnap,
-                piAlertHistorySnap
+                piAlertHistorySnap,
+                piMasterListSnap // [NEW]
             ] = await Promise.all([
                 checkAnyPartExists(normPortPartsRef),
                 checkAnyPartExists(specPortPartsRef),
@@ -396,6 +403,7 @@ exports.runRootDataIndexer = async (config, dependencies) => {
                 insightsRef.get(),
                 Promise.resolve(!genericSocialSnap.empty),
                 piRankingsRef.get(),
+                piMasterListRef.get(), // [NEW]
                 // Check new structure first, fallback to legacy
                 checkDateCollectionHasDocs(piPortfoliosCollectionRef).then(exists => exists || checkAnyPartExists(piPortfoliosPartsRef)),
                 checkAnyPartExists(piDeepPartsRef), // Legacy only
@@ -459,6 +467,9 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             availability.hasHistory   = normHistExists || specHistExists || piHistExists || signedInHistExists;
             availability.hasInsights  = insightsSnap.exists;
             availability.hasSocial    = foundPISocial || foundSignedInSocial || genericSocialExists; 
+
+            // [NEW] Assign Master List Availability
+            availability.details.piMasterList = piMasterListSnap.exists;
             
             // Price Check
             // Check if the target date exists in the price availability set

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.535",
+  "version": "1.0.537",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [