bulltrackers-module 1.0.536 → 1.0.537

package/functions/computation-system/executors/StandardExecutor.js

@@ -5,7 +5,7 @@
  * }
  */
 const { normalizeName, getEarliestDataDates }                          = require('../utils/utils');
-const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs } = require('../utils/data_loader');
+const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs, getHistoryPartRefs } = require('../utils/data_loader');
 const { CachedDataLoader }                                             = require('../data/CachedDataLoader');
 const { ContextFactory }                                               = require('../context/ContextFactory');
 const { commitResults }                                                = require('../persistence/ResultCommitter');
@@ -24,15 +24,31 @@ class StandardExecutor {
             const type = (c.userType || 'ALL').toUpperCase();
             requiredUserTypes.add(type);
         });
-        // If any calc requires 'ALL' (or has no userType), we fetch everything
         const userTypeArray = requiredUserTypes.has('ALL') ? null : Array.from(requiredUserTypes);
         
+        // [OPTIMIZATION] Check for Target CID in manifests (On-Demand Optimization)
+        // If present, we will filter all data streams to strictly this user
+        const targetCid = calcs.find(c => c.targetCid)?.targetCid || calcs.find(c => c.manifest?.targetCid)?.manifest?.targetCid;
+        if (targetCid) {
+            logger.log('INFO', `[StandardExecutor] Running in Targeted Mode for CID: ${targetCid}`);
+        }
+
         const fullRoot = { ...rootData };
         if (calcs.some(c => c.isHistorical)) {
             const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
             const prevStr = prev.toISOString().slice(0, 10);
-            // [FIX] Pass userTypeArray to filter yesterday's data loading
-            fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr, userTypeArray);
+            
+            // Fetch yesterday's refs
+            let yRefs = await getPortfolioPartRefs(config, deps, prevStr, userTypeArray);
+            
+            // [OPTIMIZATION] Filter Yesterday's Refs if targetCid is set
+            if (targetCid && yRefs) {
+                const originalCount = yRefs.length;
+                yRefs = yRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
+                logger.log('INFO', `[StandardExecutor] Filtered Yesterday's Refs: ${originalCount} -> ${yRefs.length}`);
+            }
+            
+            fullRoot.yesterdayPortfolioRefs = yRefs;
         }
 
         const state = {};
@@ -46,18 +62,48 @@ class StandardExecutor {
             } catch (e) { logger.log('WARN', `Failed to init ${c.name}`); } 
         }
 
-        return await StandardExecutor.streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite, userTypeArray);
+        // Pass targetCid to streamAndProcess
+        return await StandardExecutor.streamAndProcess(
+            dStr, state, passName, config, deps, fullRoot, 
+            rootData.portfolioRefs, rootData.historyRefs, 
+            fetchedDeps, previousFetchedDeps, skipStatusWrite, 
+            userTypeArray, targetCid
+        );
     }
 
-    // [UPDATED] Accepts requiredUserTypes param
-    static async streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite, requiredUserTypes = null) {
+    // [UPDATED] Added targetCid param
+    static async streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite, requiredUserTypes = null, targetCid = null) {
         const { logger } = deps;
         const calcs = Object.values(state).filter(c => c && c.manifest);
         const streamingCalcs = calcs.filter(c => c.manifest.rootDataDependencies.includes('portfolio') || c.manifest.rootDataDependencies.includes('history'));
         
         if (streamingCalcs.length === 0) return { successUpdates: {}, failureReport: [] };
+
+        // --- 1. Resolve and Filter Portfolio Refs (Today) ---
+        let effectivePortfolioRefs = portfolioRefs;
+        if (!effectivePortfolioRefs) {
+            // If refs weren't provided by AvailabilityChecker, fetch them now
+            effectivePortfolioRefs = await getPortfolioPartRefs(config, deps, dateStr, requiredUserTypes);
+        }
+        if (targetCid && effectivePortfolioRefs) {
+            // Filter: Keep only refs that match the CID (or Legacy refs without CID)
+            effectivePortfolioRefs = effectivePortfolioRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
+        }
+
+        // --- 2. Resolve and Filter History Refs ---
+        let effectiveHistoryRefs = historyRefs;
+        const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
+        
+        if (needsTradingHistory) {
+            if (!effectiveHistoryRefs) {
+                effectiveHistoryRefs = await getHistoryPartRefs(config, deps, dateStr, requiredUserTypes);
+            }
+            if (targetCid && effectiveHistoryRefs) {
+                effectiveHistoryRefs = effectiveHistoryRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
+            }
+        }
         
-        let totalReadOps = (portfolioRefs?.length || 0) + (historyRefs?.length || 0);
+        let totalReadOps = (effectivePortfolioRefs?.length || 0) + (effectiveHistoryRefs?.length || 0);
         if (rootData.yesterdayPortfolioRefs) totalReadOps += rootData.yesterdayPortfolioRefs.length;
         totalReadOps += 2; 
 
@@ -83,6 +129,7 @@ class StandardExecutor {
         const setupDuration = performance.now() - startSetup;
         Object.keys(executionStats).forEach(name => executionStats[name].timings.setup += setupDuration);
 
+        // Yesterday's Refs are already filtered in run()
         const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
         const prevDateStr = prevDate.toISOString().slice(0, 10);
         
@@ -91,17 +138,13 @@ class StandardExecutor {
             earliestDates = await getEarliestDataDates(config, deps);
         }
 
-        // [FIX] Pass requiredUserTypes to streamPortfolioData
-        // Note: portfolioRefs (for today) might be null if not pre-fetched, allowing streamPortfolioData to fetch filtered refs
-        const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs, requiredUserTypes);
+        // [FIX] Use effective/filtered refs
+        const tP_iter = streamPortfolioData(config, deps, dateStr, effectivePortfolioRefs, requiredUserTypes);
         
         const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
-        // Yesterday's refs were already filtered in run(), so we pass them directly
         const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs)  ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
         
-        const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-        // [FIX] Pass requiredUserTypes to streamHistoryData
-        const tH_iter = (needsTradingHistory) ? streamHistoryData(config, deps, dateStr, historyRefs, requiredUserTypes) : null;
+        const tH_iter = (needsTradingHistory) ? streamHistoryData(config, deps, dateStr, effectiveHistoryRefs, requiredUserTypes) : null;
 
         let yP_chunk = {}, tH_chunk = {};
         let usersSinceLastFlush = 0;
@@ -159,7 +202,7 @@ class StandardExecutor {
     
     // ... rest of the file (flushBuffer, mergeReports, executePerUser) ...
     static async flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, mode, skipStatusWrite, isInitialWrite = false) {
-        // ... (No changes to flushBuffer)
+        const { logger } = deps;
         const transformedState = {};
         for (const [name, inst] of Object.entries(state)) {
             const rawResult = inst.results || {};
@@ -197,7 +240,6 @@ class StandardExecutor {
     }
 
     static mergeReports(successAcc, failureAcc, newResult) {
-        // ... (No changes to mergeReports)
         if (!newResult) return;
         for (const [name, update] of Object.entries(newResult.successUpdates)) {
             if (!successAcc[name]) {
@@ -232,24 +274,23 @@ class StandardExecutor {
         const targetUserType = metadata.userType; 
         // [NEW] Always load Global Helpers
         const mappings = await loader.loadMappings();
-        const piMasterList = await loader.loadPIMasterList(); // [NEW] Loaded globally
+        const piMasterList = await loader.loadPopularInvestorMasterList(config, deps); // [NEW] Loaded globally
         const SCHEMAS = mathLayer.SCHEMAS;
 
         // 1. Load Root Data
         const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
-        const verifications = metadata.rootDataDependencies?.includes('verification') ? await loader.loadVerifications() : null;
-        const rankings = metadata.rootDataDependencies?.includes('rankings') ? await loader.loadRankings(dateStr) : null;
+        const verifications = metadata.rootDataDependencies?.includes('verification') ? await loader.loadVerificationProfiles(config, deps) : null;
+        const rankings = metadata.rootDataDependencies?.includes('rankings') ? await loader.loadPopularInvestorRankings(config, deps, dateStr) : null;
         
         // [FIX] Load Yesterday's Rankings if isHistorical is true
         let yesterdayRankings = null;
         if (metadata.rootDataDependencies?.includes('rankings') && metadata.isHistorical) {
              const prevDate = new Date(dateStr); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
              const prevStr = prevDate.toISOString().slice(0, 10);
-             // Assuming CachedDataLoader handles caching for efficiency
-             yesterdayRankings = await loader.loadRankings(prevStr); 
+             yesterdayRankings = await loader.loadPopularInvestorRankings(config, deps, prevStr); 
         }
 
-        const socialContainer = metadata.rootDataDependencies?.includes('social') ? await loader.loadSocial(dateStr) : null;
+        const socialContainer = metadata.rootDataDependencies?.includes('social') ? await loader.loadDailySocialPostInsights(config, deps, dateStr) : null;
         
         // [NEW] Load New Root Data Types for Profile Metrics
         // [FIX] Enforce canHaveMissingRoots
@@ -258,14 +299,13 @@ class StandardExecutor {
         let ratings = null;
         if (metadata.rootDataDependencies?.includes('ratings')) {
             try {
-                ratings = await loader.loadRatings(dateStr);
+                ratings = await loader.loadPIRatings(config, deps, dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'ratings' failed to load for ${metadata.name}: ${e.message}`);
                 }
                 ratings = null;
             }
-            // Check if result is null (clean fetch but empty/missing doc)
             if (!ratings && !allowMissing) {
                 throw new Error(`[StandardExecutor] Required root 'ratings' is missing for ${metadata.name}`);
             }
@@ -274,7 +314,7 @@ class StandardExecutor {
         let pageViews = null;
         if (metadata.rootDataDependencies?.includes('pageViews')) {
             try {
-                pageViews = await loader.loadPageViews(dateStr);
+                pageViews = await loader.loadPIPageViews(config, deps, dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'pageViews' failed to load for ${metadata.name}: ${e.message}`);
@@ -289,7 +329,7 @@ class StandardExecutor {
         let watchlistMembership = null;
         if (metadata.rootDataDependencies?.includes('watchlist')) {
             try {
-                watchlistMembership = await loader.loadWatchlistMembership(dateStr);
+                watchlistMembership = await loader.loadWatchlistMembership(config, deps, dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'watchlist' failed to load for ${metadata.name}: ${e.message}`);
@@ -304,7 +344,7 @@ class StandardExecutor {
         let alertHistory = null;
         if (metadata.rootDataDependencies?.includes('alerts')) {
             try {
-                alertHistory = await loader.loadAlertHistory(dateStr);
+                alertHistory = await loader.loadPIAlertHistory(config, deps, dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'alerts' failed to load for ${metadata.name}: ${e.message}`);
@@ -350,7 +390,6 @@ class StandardExecutor {
             
             const userVerification = verifications ? verifications[userId] : null;
             
-            // [FIX] Extract current AND yesterday's rank entry for this user
             const userRanking = rankings ? (rankings.find(r => String(r.CustomerId) === String(userId)) || null) : null;
             const userRankingYesterday = yesterdayRankings ? (yesterdayRankings.find(r => String(r.CustomerId) === String(userId)) || null) : null;
 
@@ -373,23 +412,19 @@ class StandardExecutor {
                 config, deps,
                 verification: userVerification, 
                 
-                // [FIX] Pass both ranking entries
                 rankings: userRanking,
                 yesterdayRankings: userRankingYesterday, 
                 
-                // [FIX] Pass both global lists
                 allRankings: rankings,
                 allRankingsYesterday: yesterdayRankings,
                 
                 allVerifications: verifications,
                 
-                // [NEW] Pass New Root Data Types for Profile Metrics
                 ratings: ratings || {},
                 pageViews: pageViews || {},
                 watchlistMembership: watchlistMembership || {},
                 alertHistory: alertHistory || {},
 
-                // [NEW] Pass Master List
                 piMasterList,
             });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.536",
+  "version": "1.0.537",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [