bulltrackers-module 1.0.700 → 1.0.702

package/functions/computation-system/data/CachedDataLoader.js

@@ -17,7 +17,8 @@ const {
     loadWatchlistMembership: loadWatchlistMembershipData,
     loadPIAlertHistory,
     loadPIWatchlistData,
-    loadPopularInvestorMasterList
+    loadPopularInvestorMasterList,
+    loadDailyPortfolios // <--- IMPORTED
 } = require('../utils/data_loader');
 const { getAvailabilityWindow } = require('./AvailabilityChecker');
 const zlib = require('zlib'); 
@@ -81,6 +82,12 @@ const LOADER_DEFINITIONS = {
         // No collection key needed for direct implementation, but handled by fn
         fn: loadPIWatchlistData,
         isIdBased: true // Uses ID instead of Date
+    },
+    // <--- ADDED SUPPORT FOR PORTFOLIO SERIES
+    loadPortfolios: {
+        cache: 'portfolios',
+        fn: loadDailyPortfolios
+        // No configKey/flag ensures legacy loop (required for complex multi-collection fetch)
     }
 };
 
@@ -144,6 +151,8 @@ class CachedDataLoader {
     async loadWatchlistMembership(dateStr)  { return this._loadGeneric('loadWatchlistMembership', dateStr); }
     async loadAlertHistory(dateStr)         { return this._loadGeneric('loadAlertHistory'       , dateStr); }
     async loadPIWatchlistData(piCid)        { return this._loadGeneric('loadPIWatchlistData'    , String(piCid)); }
+    // <--- ADDED PORTFOLIOS ACCESSOR
+    async loadPortfolios(dateStr)           { return this._loadGeneric('loadPortfolios'         , dateStr); }
 
     // =========================================================================
     // SPECIALIZED LOADERS (Non-Standard Patterns)

package/functions/computation-system/data/DependencyFetcher.js

@@ -226,6 +226,7 @@ async function fetchDependencies(date, calcs, config, deps, manifestLookup = {},
 async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config, deps, lookbackDays) {
     const { db, logger } = deps;
     const results = {}; // normalizedName -> { date -> data }
+    const { resultsCollection = 'computation_results', resultsSubcollection = 'results', computationsSubcollection = 'computations' } = config;
     
     // Initialize results structure
     calcNames.forEach(n => results[normalizeName(n)] = {});
@@ -238,6 +239,15 @@ async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config,
         dates.push(d.toISOString().slice(0, 10));
     }
 
+    // [DEBUG] Log the manifest lookup and resolved categories
+    logger.log('INFO', `[DependencyFetcher] 🔍 ManifestLookup has ${Object.keys(manifestLookup).length} entries`);
+    for (const rawName of calcNames) {
+        const norm = normalizeName(rawName);
+        const category = manifestLookup[norm] || 'analytics';
+        const samplePath = `${resultsCollection}/${dates[0]}/${resultsSubcollection}/${category}/${computationsSubcollection}/${rawName}`;
+        logger.log('INFO', `[DependencyFetcher] 📍 '${rawName}' -> category='${category}' -> Path: ${samplePath}`);
+    }
+
     // Build Fetch Operations
     const ops = [];
     for (const dateStr of dates) {
@@ -260,6 +270,13 @@ async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config,
     for (let i = 0; i < ops.length; i += BATCH_SIZE) {
         await Promise.all(ops.slice(i, i + BATCH_SIZE).map(fn => fn()));
     }
+    
+    // [DEBUG] Log results summary
+    for (const rawName of calcNames) {
+        const norm = normalizeName(rawName);
+        const foundDates = Object.keys(results[norm] || {});
+        logger.log('INFO', `[DependencyFetcher] ✅ '${rawName}' found data for ${foundDates.length}/${lookbackDays} days`);
+    }
 
     return results;
 }

package/functions/computation-system/executors/MetaExecutor.js

@@ -9,14 +9,11 @@ const { commitResults }     = require('../persistence/ResultCommitter');
 const { fetchResultSeries } = require('../data/DependencyFetcher');
 const { getManifest }       = require('../topology/ManifestLoader');
 
-// [FIX] Load calculations package directly for manifest building during force runs
-let _calculations = null;
-function getCalculations() {
-    if (!_calculations) {
-        try { _calculations = require('aiden-shared-calculations-unified').calculations; } 
-        catch (e) { _calculations = {}; }
-    }
-    return _calculations;
+// Helper to get calculations - prefer config.calculations, fallback to direct require
+function getCalculations(config) {
+    if (config && config.calculations) return config.calculations;
+    try { return require('aiden-shared-calculations-unified').calculations; } 
+    catch (e) { return {}; }
 }
 
 class MetaExecutor {
@@ -29,8 +26,8 @@ class MetaExecutor {
         const dStr = date.toISOString().slice(0, 10);
         const loader = new CachedDataLoader(config, deps);
 
-        // 1. Setup Manifest Lookup
-        const allManifests = getManifest(config.productLines, config.calculationsDirectory, deps); 
+        // 1. Setup Manifest Lookup (use activeProductLines and calculations from config)
+        const allManifests = getManifest(config.activeProductLines || [], getCalculations(config), deps); 
         const manifestLookup = Object.fromEntries(allManifests.map(m => [normalizeName(m.name), m.category]));
 
         // 2. Load Base Data (Always Required)
@@ -104,8 +101,8 @@ class MetaExecutor {
         const { logger } = deps;
         const calcs = [metadata]; // Treat single as list for helpers
 
-        // [FIX] Build manifestLookup using the actual calculations package (not config.calculationsDirectory)
-        const allManifests = getManifest([], getCalculations(), deps);
+        // Build manifestLookup using calculations from config (set in index.js)
+        const allManifests = getManifest(config.activeProductLines || [], getCalculations(config), deps);
         const manifestLookup = Object.fromEntries(allManifests.map(m => [normalizeName(m.name), m.category]));
 
         // 1. Load Data using Shared Helpers

package/functions/computation-system/executors/StandardExecutor.js

@@ -1,9 +1,10 @@
 /**
  * @fileoverview Executor for "Standard" (User-Level) calculations.
  * REFACTORED: Hoisted data loading, centralized Series/Root logic.
+ * UPDATED: Added Master List Driver for PI calculations to ensure full coverage.
  */
 const { normalizeName, getEarliestDataDates } = require('../utils/utils');
-const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs, getHistoryPartRefs } = require('../utils/data_loader');
+const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs, getHistoryPartRefs, loadFullDayMap } = require('../utils/data_loader');
 const { CachedDataLoader } = require('../data/CachedDataLoader');
 const { ContextFactory } = require('../context/ContextFactory');
 const { commitResults } = require('../persistence/ResultCommitter');
@@ -13,14 +14,11 @@ const mathLayer = require('../layers/index');
 const { performance } = require('perf_hooks');
 const v8 = require('v8');
 
-// [FIX] Load calculations package directly for manifest building
-let _calculations = null;
-function getCalculations() {
-    if (!_calculations) {
-        try { _calculations = require('aiden-shared-calculations-unified').calculations; } 
-        catch (e) { _calculations = {}; }
-    }
-    return _calculations;
+// Helper to get calculations - prefer config.calculations, fallback to direct require
+function getCalculations(config) {
+    if (config && config.calculations) return config.calculations;
+    try { return require('aiden-shared-calculations-unified').calculations; } 
+    catch (e) { return {}; }
 }
 
 class StandardExecutor {
@@ -83,8 +81,6 @@ class StandardExecutor {
         const loader = new CachedDataLoader(config, deps);
         
         // --- 1. PRE-LOAD GLOBAL DATA (Hoisted) ---
-        // Load all "Singleton" datasets once (Ratings, Rankings, Series, Mappings)
-        // instead of checking/loading per-user inside the loop.
         const startSetup = performance.now();
         
         const [
@@ -120,8 +116,37 @@ class StandardExecutor {
             shardMap[normalizeName(c.manifest.name)] = 0;
         });
 
-        // --- 3. STREAM EXECUTION ---
-        const tP_iter = streamPortfolioData(config, deps, dateStr, effPortRefs, requiredUserTypes);
+        // --- 3. DETERMINE ITERATION STRATEGY ---
+        let tP_iter;
+        const isPiOnly = requiredUserTypes && requiredUserTypes.length === 1 && requiredUserTypes[0] === 'POPULAR_INVESTOR';
+        
+        // STRATEGY A: MASTER LIST DRIVER (For PIs, ensuring we hit every ID in the list, even if missing today)
+        if (isPiOnly && globalRoots.piMasterList && !targetCid) {
+            logger.log('INFO', `[StandardExecutor] 🚀 Driving execution via Master List (${Object.keys(globalRoots.piMasterList).length} PIs)`);
+            
+            // Load Today's Map Fully (Small enough for PIs)
+            const todayMap = await loadFullDayMap(config, deps, effPortRefs, dateStr);
+            
+            // Create a generator that yields chunks from the Master List keys
+            tP_iter = (async function* () {
+                const allCids = Object.keys(globalRoots.piMasterList);
+                const batchSize = config.partRefBatchSize || 50;
+                for (let i = 0; i < allCids.length; i += batchSize) {
+                    const batchCids = allCids.slice(i, i + batchSize);
+                    const chunk = {};
+                    batchCids.forEach(cid => {
+                        // If missing today, provide a stub so the Calc can look back in History
+                        chunk[cid] = todayMap[cid] || { _userType: 'POPULAR_INVESTOR', _isMissingToday: true };
+                    });
+                    yield chunk;
+                }
+            })();
+        } 
+        // STRATEGY B: STANDARD STREAM (Iterate whatever is in Firestore/GCS for today)
+        else {
+            tP_iter = streamPortfolioData(config, deps, dateStr, effPortRefs, requiredUserTypes);
+        }
+
         const yP_iter = (streamingCalcs.some(c => c.manifest.isHistorical) && rootData.yesterdayPortfolioRefs) 
                         ? streamPortfolioData(config, deps, new Date(new Date(dateStr).getTime() - 86400000).toISOString().slice(0, 10), rootData.yesterdayPortfolioRefs) 
                         : null;
@@ -150,7 +175,7 @@ class StandardExecutor {
                         stats[normalizeName(calc.manifest.name)],
                         earliestDates,
                         seriesData,
-                        globalRoots // <--- PASSED DOWN
+                        globalRoots
                     )
                 ));
                 
@@ -171,8 +196,8 @@ class StandardExecutor {
                 }
             }
         } finally {
-            if (yP_iter?.return) await yP_iter.return();
-            if (tH_iter?.return) await tH_iter.return();
+            if (yP_iter?.return && typeof yP_iter.return === 'function') await yP_iter.return();
+            if (tH_iter?.return && typeof tH_iter.return === 'function') await tH_iter.return();
         }
 
         const finalRes = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardMap, stats, 'FINAL', skipStatusWrite, !hasFlushed);
@@ -182,7 +207,7 @@ class StandardExecutor {
     }
 
     // =========================================================================
-    // PER-USER EXECUTION (Pure Logic)
+    // PER-USER EXECUTION
     // =========================================================================
     static async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps, config, deps, stats, earliestDates, seriesData, globalRoots) {
         const { logger } = deps;
@@ -192,18 +217,21 @@ class StandardExecutor {
         let success = 0, failures = 0;
 
         for (const [userId, todayPortfolio] of Object.entries(portfolioData)) {
-            // 1. Filter User
             if (metadata.targetCid && String(userId) !== String(metadata.targetCid)) { stats.skippedUsers++; continue; }
             
-            // 2. Determine Type
+            // 2. Determine Type (Handle Stub for Missing Users)
             let actualType = todayPortfolio._userType;
             if (!actualType) {
+                // If driving from Master List, we know they are PIs even if missing today
                 const isRanked = globalRoots.rankings && globalRoots.rankings.some(r => String(r.CustomerId) === String(userId));
-                actualType = isRanked ? 'POPULAR_INVESTOR' : (todayPortfolio.PublicPositions ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL);
+                if (todayPortfolio._isMissingToday && isRanked) {
+                    actualType = 'POPULAR_INVESTOR';
+                } else {
+                    actualType = isRanked ? 'POPULAR_INVESTOR' : (todayPortfolio.PublicPositions ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL);
+                }
             }
             if (targetUserType && targetUserType !== 'all' && targetUserType !== actualType) { stats.skippedUsers++; continue; }
 
-            // 3. Resolve User Specifics from Global Data
             const userRank = globalRoots.rankings?.find(r => String(r.CustomerId) === String(userId)) || null;
             const userRankYest = globalRoots.rankingsYesterday?.find(r => String(r.CustomerId) === String(userId)) || null;
             const userVerify = globalRoots.verifications?.[userId] || null;
@@ -214,30 +242,21 @@ class StandardExecutor {
                          (actualType === 'SIGNED_IN_USER' ? globalRoots.social.signedIn[userId] : globalRoots.social.generic)) || {};
             }
 
-            // 4. Build Context
             const context = ContextFactory.buildPerUserContext({
-                todayPortfolio, 
+                todayPortfolio: todayPortfolio._isMissingToday ? null : todayPortfolio, 
                 yesterdayPortfolio: yesterdayPortfolioData?.[userId] || null, 
                 todayHistory: historyData?.[userId] || null,
                 userId, userType: actualType, dateStr, metadata,
-                
-                // Injected Global Data
                 mappings: globalRoots.mappings,
                 piMasterList: globalRoots.piMasterList,
                 insights: globalRoots.insights,
                 socialData: social ? { today: social } : null,
-                
-                // Dependency Data
                 computedDependencies: computedDeps, 
                 previousComputedDependencies: prevDeps,
                 config, deps,
-                
-                // Specific Lookups
                 verification: userVerify,
                 rankings: userRank,
                 yesterdayRankings: userRankYest,
-                
-                // Full Access (if needed by calc)
                 allRankings: globalRoots.rankings,
                 allRankingsYesterday: globalRoots.rankingsYesterday,
                 allVerifications: globalRoots.verifications,
@@ -245,7 +264,6 @@ class StandardExecutor {
                 pageViews: globalRoots.pageViews || {},
                 watchlistMembership: globalRoots.watchlistMembership || {},
                 alertHistory: globalRoots.alertHistory || {},
-                
                 seriesData
             });
 
@@ -267,13 +285,12 @@ class StandardExecutor {
     }
 
     // =========================================================================
-    // HELPERS (Flush & Merge)
+    // FLUSH & MERGE
     // =========================================================================
     static async flushBuffer(state, dateStr, passName, config, deps, shardMap, stats, mode, skipStatus, isInitial) {
         const transformedState = {};
         for (const [name, inst] of Object.entries(state)) {
             let data = inst.results || {};
-            // Pivot user-date structure if needed
             const first = Object.keys(data)[0];
             if (first && data[first] && typeof data[first] === 'object' && /^\d{4}-\d{2}-\d{2}$/.test(Object.keys(data[first])[0])) {
                 const pivoted = {};
@@ -286,7 +303,7 @@ class StandardExecutor {
                 data = pivoted;
             }
             transformedState[name] = { manifest: inst.manifest, getResult: async () => data, _executionStats: stats[name] };
-            inst.results = {}; // Clear buffer
+            inst.results = {};
         }
         const res = await commitResults(transformedState, dateStr, passName, config, deps, skipStatus, { flushMode: mode, shardIndexes: shardMap, isInitialWrite: isInitial });
         if (res.shardIndexes) Object.assign(shardMap, res.shardIndexes);
@@ -313,31 +330,13 @@ class StandardExecutor {
     }
 }
 
-// =============================================================================
-// SHARED LOADING HELPERS
-// =============================================================================
-
-/**
- * Pre-loads all shared global datasets required by the active calculations.
- * Returns a consolidated object of { ratings, rankings, insights, ... }
- */
 async function loadGlobalRoots(loader, dateStr, calcs, deps) {
     const { logger } = deps;
     const roots = {};
-    
-    // 1. Identify Requirements
     const reqs = {
-        mappings: true,
-        piMasterList: true,
-        rankings: false,
-        rankingsYesterday: false,
-        verifications: false,
-        insights: false,
-        social: false,
-        ratings: false,
-        pageViews: false,
-        watchlist: false,
-        alerts: false
+        mappings: true, piMasterList: true, rankings: false, rankingsYesterday: false,
+        verifications: false, insights: false, social: false, ratings: false,
+        pageViews: false, watchlist: false, alerts: false
     };
 
     for (const c of calcs) {
@@ -353,7 +352,6 @@ async function loadGlobalRoots(loader, dateStr, calcs, deps) {
         if (deps.includes('alerts')) reqs.alerts = true;
     }
 
-    // 2. Fetch Helper
     const fetch = async (key, method, dateArg, optional = true) => {
         if (!reqs[key]) return;
         try {
@@ -365,9 +363,8 @@ async function loadGlobalRoots(loader, dateStr, calcs, deps) {
         }
     };
 
-    // 3. Execute Loads
     await Promise.all([
-        fetch('mappings', 'loadMappings', null, false), // Always required
+        fetch('mappings', 'loadMappings', null, false),
         fetch('piMasterList', 'loadPIMasterList', null, false),
         fetch('rankings', 'loadRankings', dateStr),
         fetch('verifications', 'loadVerifications', dateStr),
@@ -384,16 +381,14 @@ async function loadGlobalRoots(loader, dateStr, calcs, deps) {
         await fetch('rankingsYesterday', 'loadRankings', prev);
     }
     
-    // Map internal names to match loadGlobalRoots structure if needed
     roots.watchlistMembership = roots.watchlist;
     roots.alertHistory = roots.alerts;
-    
     return roots;
 }
 
 async function loadSeriesData(loader, dateStr, calcs, config, deps) {
     const rootReqs = {};
-    const depReqs = {};  // norm -> { days, originalName, category }
+    const depReqs = {};
 
     calcs.forEach(c => {
         if (c.manifest.rootDataSeries) {
@@ -413,26 +408,22 @@ async function loadSeriesData(loader, dateStr, calcs, config, deps) {
     const series = { root: {}, results: {} };
     const rootMap = {
         alerts: 'loadAlertHistory', insights: 'loadInsights', ratings: 'loadRatings',
-        watchlist: 'loadWatchlistMembership', rankings: 'loadRankings'
+        watchlist: 'loadWatchlistMembership', rankings: 'loadRankings',
+        portfolios: 'loadPortfolios' 
     };
 
     await Promise.all(Object.entries(rootReqs).map(async ([key, days]) => {
         if (rootMap[key]) series.root[key] = (await loader.loadSeries(rootMap[key], dateStr, days)).data;
     }));
 
-    // [FIX] Use getCalculations() to load ALL computations for proper category lookup
     const depEntries = Object.values(depReqs);
     if (depEntries.length) {
         const depOriginalNames = depEntries.map(e => e.originalName);
         const maxDays = Math.max(...depEntries.map(e => e.days));
-        
-        // Build lookup from ALL computations (empty productLines = load all)
-        const allManifests = getManifest([], getCalculations(), deps);
+        const allManifests = getManifest(config.activeProductLines || [], getCalculations(config), deps);
         const lookup = Object.fromEntries(allManifests.map(m => [normalizeName(m.name), m.category]));
-        
         series.results = await fetchResultSeries(dateStr, depOriginalNames, lookup, config, deps, maxDays);
     }
-
     return series;
 }
 

package/functions/computation-system/utils/data_loader.js

@@ -228,6 +228,18 @@ async function loadFullDayMap(config, deps, partRefs, dateString) {
     return fullMap;
 }
 
+/** Stage 3.5: Load Daily Portfolios (Wrapper for Series Loading) */
+async function loadDailyPortfolios(config, deps, dateString) {
+    // 1. GCS FAST PATH
+    const cached = await tryLoadFromGCS(config, dateString, 'portfolios', deps.logger);
+    if (cached) return cached;
+
+    // 2. FIRESTORE FALLBACK
+    const partRefs = await getPortfolioPartRefs(config, deps, dateString);
+    if (partRefs.length === 0) return {};
+    return loadDataByRefs(config, deps, partRefs);
+}
+
 /** Stage 4: Load daily instrument insights */
 async function loadDailyInsights(config, deps, dateString) {
     const { db, logger, calculationUtils } = deps;
@@ -800,7 +812,8 @@ async function loadPopularInvestorMasterList(config, deps, dateString = null) {
 module.exports = {
     getPortfolioPartRefs, 
     loadDataByRefs, 
-    loadFullDayMap, 
+    loadFullDayMap,
+    loadDailyPortfolios, // <--- EXPORTED NEW WRAPPER
     loadDailyInsights, 
     loadDailySocialPostInsights, 
     getHistoryPartRefs, 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.700",
+  "version": "1.0.702",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [