bulltrackers-module 1.0.702 → 1.0.704

package/functions/computation-system/data/CachedDataLoader.js

@@ -119,7 +119,8 @@ class CachedDataLoader {
     // =========================================================================
     // GENERIC LOADER HELPER
     // =========================================================================
-    async _loadGeneric(methodName, key) {
+    // [FIX] Accepts ...args to pass down filters (like requiredUserTypes)
+    async _loadGeneric(methodName, key, ...args) {
         const def = LOADER_DEFINITIONS[methodName];
         if (!def) throw new Error(`Unknown loader method: ${methodName}`);
 
@@ -132,7 +133,7 @@ class CachedDataLoader {
             this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading '${def.cache}' from: ${collection}/${key}`);
         }
 
-        const promise = def.fn(this.config, this.deps, key);
+        const promise = def.fn(this.config, this.deps, key, ...args);
         cacheMap.set(key, promise);
         return promise;
     }
@@ -151,8 +152,8 @@ class CachedDataLoader {
     async loadWatchlistMembership(dateStr)  { return this._loadGeneric('loadWatchlistMembership', dateStr); }
     async loadAlertHistory(dateStr)         { return this._loadGeneric('loadAlertHistory'       , dateStr); }
     async loadPIWatchlistData(piCid)        { return this._loadGeneric('loadPIWatchlistData'    , String(piCid)); }
-    // <--- ADDED PORTFOLIOS ACCESSOR
-    async loadPortfolios(dateStr)           { return this._loadGeneric('loadPortfolios'         , dateStr); }
+    // <--- ADDED PORTFOLIOS ACCESSOR with extra arg
+    async loadPortfolios(dateStr, userTypes) { return this._loadGeneric('loadPortfolios', dateStr, userTypes); }
 
     // =========================================================================
     // SPECIALIZED LOADERS (Non-Standard Patterns)
@@ -204,13 +205,14 @@ class CachedDataLoader {
     /**
      * Optimistically loads data series using Batch Reads (db.getAll).
      * Uses Availability Index to minimize costs.
+     * [FIX] Now accepts ...args to pass context (e.g. requiredUserTypes)
      */
-    async loadSeries(loaderMethod, dateStr, lookbackDays) {
+    async loadSeries(loaderMethod, dateStr, lookbackDays, ...args) {
         const def = LOADER_DEFINITIONS[loaderMethod];
         if (!def) throw new Error(`[CachedDataLoader] Unknown series method ${loaderMethod}`);
         
         // Fallback to legacy loop if method isn't configured for batching (missing config/flag)
-        if (!def.configKey || !def.flag) return this._loadSeriesLegacy(loaderMethod, dateStr, lookbackDays);
+        if (!def.configKey || !def.flag) return this._loadSeriesLegacy(loaderMethod, dateStr, lookbackDays, ...args);
 
         // 1. Calculate Date Range
         const endDate = new Date(dateStr);
@@ -265,7 +267,7 @@ class CachedDataLoader {
                 });
             } catch (err) {
                 console.warn(`[CachedDataLoader] Batch failed: ${err.message}. Legacy fallback.`);
-                return this._loadSeriesLegacy(loaderMethod, dateStr, lookbackDays);
+                return this._loadSeriesLegacy(loaderMethod, dateStr, lookbackDays, ...args);
             }
         }
 
@@ -277,7 +279,7 @@ class CachedDataLoader {
         };
     }
 
-    async _loadSeriesLegacy(loaderMethod, dateStr, lookbackDays) {
+    async _loadSeriesLegacy(loaderMethod, dateStr, lookbackDays, ...args) {
         const results = {};
         const promises = [];
         const endDate = new Date(dateStr);
@@ -286,7 +288,8 @@ class CachedDataLoader {
             const d = new Date(endDate);
             d.setUTCDate(d.getUTCDate() - i);
             const dStr = d.toISOString().slice(0, 10);
-            promises.push(this[loaderMethod](dStr).then(data => data ? results[dStr] = data : null).catch(() => null));
+            // [FIX] Pass args (e.g. requiredUserTypes) to the loader method
+            promises.push(this[loaderMethod](dStr, ...args).then(data => data ? results[dStr] = data : null).catch(() => null));
         }
 
         await Promise.all(promises);

package/functions/computation-system/executors/StandardExecutor.js

@@ -1,10 +1,9 @@
 /**
  * @fileoverview Executor for "Standard" (User-Level) calculations.
  * REFACTORED: Hoisted data loading, centralized Series/Root logic.
- * UPDATED: Added Master List Driver for PI calculations to ensure full coverage.
  */
 const { normalizeName, getEarliestDataDates } = require('../utils/utils');
-const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs, getHistoryPartRefs, loadFullDayMap } = require('../utils/data_loader');
+const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs, getHistoryPartRefs } = require('../utils/data_loader');
 const { CachedDataLoader } = require('../data/CachedDataLoader');
 const { ContextFactory } = require('../context/ContextFactory');
 const { commitResults } = require('../persistence/ResultCommitter');
@@ -81,6 +80,8 @@ class StandardExecutor {
         const loader = new CachedDataLoader(config, deps);
         
         // --- 1. PRE-LOAD GLOBAL DATA (Hoisted) ---
+        // Load all "Singleton" datasets once (Ratings, Rankings, Series, Mappings)
+        // instead of checking/loading per-user inside the loop.
         const startSetup = performance.now();
         
         const [
@@ -116,37 +117,8 @@ class StandardExecutor {
             shardMap[normalizeName(c.manifest.name)] = 0;
         });
 
-        // --- 3. DETERMINE ITERATION STRATEGY ---
-        let tP_iter;
-        const isPiOnly = requiredUserTypes && requiredUserTypes.length === 1 && requiredUserTypes[0] === 'POPULAR_INVESTOR';
-        
-        // STRATEGY A: MASTER LIST DRIVER (For PIs, ensuring we hit every ID in the list, even if missing today)
-        if (isPiOnly && globalRoots.piMasterList && !targetCid) {
-            logger.log('INFO', `[StandardExecutor] 🚀 Driving execution via Master List (${Object.keys(globalRoots.piMasterList).length} PIs)`);
-            
-            // Load Today's Map Fully (Small enough for PIs)
-            const todayMap = await loadFullDayMap(config, deps, effPortRefs, dateStr);
-            
-            // Create a generator that yields chunks from the Master List keys
-            tP_iter = (async function* () {
-                const allCids = Object.keys(globalRoots.piMasterList);
-                const batchSize = config.partRefBatchSize || 50;
-                for (let i = 0; i < allCids.length; i += batchSize) {
-                    const batchCids = allCids.slice(i, i + batchSize);
-                    const chunk = {};
-                    batchCids.forEach(cid => {
-                        // If missing today, provide a stub so the Calc can look back in History
-                        chunk[cid] = todayMap[cid] || { _userType: 'POPULAR_INVESTOR', _isMissingToday: true };
-                    });
-                    yield chunk;
-                }
-            })();
-        } 
-        // STRATEGY B: STANDARD STREAM (Iterate whatever is in Firestore/GCS for today)
-        else {
-            tP_iter = streamPortfolioData(config, deps, dateStr, effPortRefs, requiredUserTypes);
-        }
-
+        // --- 3. STREAM EXECUTION ---
+        const tP_iter = streamPortfolioData(config, deps, dateStr, effPortRefs, requiredUserTypes);
         const yP_iter = (streamingCalcs.some(c => c.manifest.isHistorical) && rootData.yesterdayPortfolioRefs) 
                         ? streamPortfolioData(config, deps, new Date(new Date(dateStr).getTime() - 86400000).toISOString().slice(0, 10), rootData.yesterdayPortfolioRefs) 
                         : null;
@@ -175,7 +147,7 @@ class StandardExecutor {
                         stats[normalizeName(calc.manifest.name)],
                         earliestDates,
                         seriesData,
-                        globalRoots
+                        globalRoots // <--- PASSED DOWN
                     )
                 ));
                 
@@ -196,8 +168,8 @@ class StandardExecutor {
                 }
             }
         } finally {
-            if (yP_iter?.return && typeof yP_iter.return === 'function') await yP_iter.return();
-            if (tH_iter?.return && typeof tH_iter.return === 'function') await tH_iter.return();
+            if (yP_iter?.return) await yP_iter.return();
+            if (tH_iter?.return) await tH_iter.return();
         }
 
         const finalRes = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardMap, stats, 'FINAL', skipStatusWrite, !hasFlushed);
@@ -207,7 +179,7 @@ class StandardExecutor {
     }
 
     // =========================================================================
-    // PER-USER EXECUTION
+    // PER-USER EXECUTION (Pure Logic)
     // =========================================================================
     static async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps, config, deps, stats, earliestDates, seriesData, globalRoots) {
         const { logger } = deps;
@@ -217,21 +189,18 @@ class StandardExecutor {
         let success = 0, failures = 0;
 
         for (const [userId, todayPortfolio] of Object.entries(portfolioData)) {
+            // 1. Filter User
             if (metadata.targetCid && String(userId) !== String(metadata.targetCid)) { stats.skippedUsers++; continue; }
             
-            // 2. Determine Type (Handle Stub for Missing Users)
+            // 2. Determine Type
             let actualType = todayPortfolio._userType;
             if (!actualType) {
-                // If driving from Master List, we know they are PIs even if missing today
                 const isRanked = globalRoots.rankings && globalRoots.rankings.some(r => String(r.CustomerId) === String(userId));
-                if (todayPortfolio._isMissingToday && isRanked) {
-                    actualType = 'POPULAR_INVESTOR';
-                } else {
-                    actualType = isRanked ? 'POPULAR_INVESTOR' : (todayPortfolio.PublicPositions ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL);
-                }
+                actualType = isRanked ? 'POPULAR_INVESTOR' : (todayPortfolio.PublicPositions ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL);
             }
             if (targetUserType && targetUserType !== 'all' && targetUserType !== actualType) { stats.skippedUsers++; continue; }
 
+            // 3. Resolve User Specifics from Global Data
             const userRank = globalRoots.rankings?.find(r => String(r.CustomerId) === String(userId)) || null;
             const userRankYest = globalRoots.rankingsYesterday?.find(r => String(r.CustomerId) === String(userId)) || null;
             const userVerify = globalRoots.verifications?.[userId] || null;
@@ -242,21 +211,30 @@ class StandardExecutor {
                          (actualType === 'SIGNED_IN_USER' ? globalRoots.social.signedIn[userId] : globalRoots.social.generic)) || {};
             }
 
+            // 4. Build Context
             const context = ContextFactory.buildPerUserContext({
-                todayPortfolio: todayPortfolio._isMissingToday ? null : todayPortfolio, 
+                todayPortfolio, 
                 yesterdayPortfolio: yesterdayPortfolioData?.[userId] || null, 
                 todayHistory: historyData?.[userId] || null,
                 userId, userType: actualType, dateStr, metadata,
+                
+                // Injected Global Data
                 mappings: globalRoots.mappings,
                 piMasterList: globalRoots.piMasterList,
                 insights: globalRoots.insights,
                 socialData: social ? { today: social } : null,
+                
+                // Dependency Data
                 computedDependencies: computedDeps, 
                 previousComputedDependencies: prevDeps,
                 config, deps,
+                
+                // Specific Lookups
                 verification: userVerify,
                 rankings: userRank,
                 yesterdayRankings: userRankYest,
+                
+                // Full Access (if needed by calc)
                 allRankings: globalRoots.rankings,
                 allRankingsYesterday: globalRoots.rankingsYesterday,
                 allVerifications: globalRoots.verifications,
@@ -264,6 +242,7 @@ class StandardExecutor {
                 pageViews: globalRoots.pageViews || {},
                 watchlistMembership: globalRoots.watchlistMembership || {},
                 alertHistory: globalRoots.alertHistory || {},
+                
                 seriesData
             });
 
@@ -285,12 +264,13 @@ class StandardExecutor {
     }
 
     // =========================================================================
-    // FLUSH & MERGE
+    // HELPERS (Flush & Merge)
     // =========================================================================
     static async flushBuffer(state, dateStr, passName, config, deps, shardMap, stats, mode, skipStatus, isInitial) {
         const transformedState = {};
         for (const [name, inst] of Object.entries(state)) {
             let data = inst.results || {};
+            // Pivot user-date structure if needed
             const first = Object.keys(data)[0];
             if (first && data[first] && typeof data[first] === 'object' && /^\d{4}-\d{2}-\d{2}$/.test(Object.keys(data[first])[0])) {
                 const pivoted = {};
@@ -303,7 +283,7 @@ class StandardExecutor {
                 data = pivoted;
             }
             transformedState[name] = { manifest: inst.manifest, getResult: async () => data, _executionStats: stats[name] };
-            inst.results = {};
+            inst.results = {}; // Clear buffer
         }
         const res = await commitResults(transformedState, dateStr, passName, config, deps, skipStatus, { flushMode: mode, shardIndexes: shardMap, isInitialWrite: isInitial });
         if (res.shardIndexes) Object.assign(shardMap, res.shardIndexes);
@@ -330,13 +310,31 @@ class StandardExecutor {
     }
 }
 
+// =============================================================================
+// SHARED LOADING HELPERS
+// =============================================================================
+
+/**
+ * Pre-loads all shared global datasets required by the active calculations.
+ * Returns a consolidated object of { ratings, rankings, insights, ... }
+ */
 async function loadGlobalRoots(loader, dateStr, calcs, deps) {
     const { logger } = deps;
     const roots = {};
+    
+    // 1. Identify Requirements
     const reqs = {
-        mappings: true, piMasterList: true, rankings: false, rankingsYesterday: false,
-        verifications: false, insights: false, social: false, ratings: false,
-        pageViews: false, watchlist: false, alerts: false
+        mappings: true,
+        piMasterList: true,
+        rankings: false,
+        rankingsYesterday: false,
+        verifications: false,
+        insights: false,
+        social: false,
+        ratings: false,
+        pageViews: false,
+        watchlist: false,
+        alerts: false
     };
 
     for (const c of calcs) {
@@ -352,6 +350,7 @@ async function loadGlobalRoots(loader, dateStr, calcs, deps) {
         if (deps.includes('alerts')) reqs.alerts = true;
     }
 
+    // 2. Fetch Helper
     const fetch = async (key, method, dateArg, optional = true) => {
         if (!reqs[key]) return;
         try {
@@ -363,8 +362,9 @@ async function loadGlobalRoots(loader, dateStr, calcs, deps) {
         }
     };
 
+    // 3. Execute Loads
     await Promise.all([
-        fetch('mappings', 'loadMappings', null, false),
+        fetch('mappings', 'loadMappings', null, false), // Always required
         fetch('piMasterList', 'loadPIMasterList', null, false),
         fetch('rankings', 'loadRankings', dateStr),
         fetch('verifications', 'loadVerifications', dateStr),
@@ -381,14 +381,20 @@ async function loadGlobalRoots(loader, dateStr, calcs, deps) {
         await fetch('rankingsYesterday', 'loadRankings', prev);
     }
     
+    // Map internal names to match loadGlobalRoots structure if needed
     roots.watchlistMembership = roots.watchlist;
     roots.alertHistory = roots.alerts;
+    
     return roots;
 }
 
 async function loadSeriesData(loader, dateStr, calcs, config, deps) {
     const rootReqs = {};
-    const depReqs = {};
+    const depReqs = {};  // norm -> { days, originalName, category }
+
+    // [FIX] Calculate User Types here to pass to loadSeries
+    const requiredUserTypes = new Set(calcs.map(c => (c.userType || 'ALL').toUpperCase()));
+    const userTypeArray = requiredUserTypes.has('ALL') ? null : Array.from(requiredUserTypes);
 
     calcs.forEach(c => {
         if (c.manifest.rootDataSeries) {
@@ -409,21 +415,30 @@ async function loadSeriesData(loader, dateStr, calcs, config, deps) {
     const rootMap = {
         alerts: 'loadAlertHistory', insights: 'loadInsights', ratings: 'loadRatings',
         watchlist: 'loadWatchlistMembership', rankings: 'loadRankings',
+        // <--- ADDED MAPPING
         portfolios: 'loadPortfolios' 
     };
 
     await Promise.all(Object.entries(rootReqs).map(async ([key, days]) => {
-        if (rootMap[key]) series.root[key] = (await loader.loadSeries(rootMap[key], dateStr, days)).data;
+        if (rootMap[key]) {
+            // [FIX] Pass userTypeArray specifically if loading portfolios
+            const extraArgs = (key === 'portfolios') ? [userTypeArray] : [];
+            series.root[key] = (await loader.loadSeries(rootMap[key], dateStr, days, ...extraArgs)).data;
+        }
     }));
 
+    // Build lookup from ALL computations using config.calculations
     const depEntries = Object.values(depReqs);
     if (depEntries.length) {
         const depOriginalNames = depEntries.map(e => e.originalName);
         const maxDays = Math.max(...depEntries.map(e => e.days));
+        
         const allManifests = getManifest(config.activeProductLines || [], getCalculations(config), deps);
         const lookup = Object.fromEntries(allManifests.map(m => [normalizeName(m.name), m.category]));
+        
         series.results = await fetchResultSeries(dateStr, depOriginalNames, lookup, config, deps, maxDays);
     }
+
     return series;
 }
 

package/functions/computation-system/utils/data_loader.js

@@ -229,13 +229,14 @@ async function loadFullDayMap(config, deps, partRefs, dateString) {
 }
 
 /** Stage 3.5: Load Daily Portfolios (Wrapper for Series Loading) */
-async function loadDailyPortfolios(config, deps, dateString) {
+async function loadDailyPortfolios(config, deps, dateString, requiredUserTypes = null) {
     // 1. GCS FAST PATH
     const cached = await tryLoadFromGCS(config, dateString, 'portfolios', deps.logger);
     if (cached) return cached;
 
     // 2. FIRESTORE FALLBACK
-    const partRefs = await getPortfolioPartRefs(config, deps, dateString);
+    // [FIX] Now passing requiredUserTypes to prevent fetching all users (e.g. NormalUserPortfolios)
+    const partRefs = await getPortfolioPartRefs(config, deps, dateString, requiredUserTypes);
     if (partRefs.length === 0) return {};
     return loadDataByRefs(config, deps, partRefs);
 }
@@ -813,7 +814,7 @@ module.exports = {
     getPortfolioPartRefs, 
     loadDataByRefs, 
     loadFullDayMap,
-    loadDailyPortfolios, // <--- EXPORTED NEW WRAPPER
+    loadDailyPortfolios, 
     loadDailyInsights, 
     loadDailySocialPostInsights, 
     getHistoryPartRefs, 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.702",
+  "version": "1.0.704",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [