bulltrackers-module 1.0.658 → 1.0.659

package/functions/computation-system/executors/StandardExecutor.js

@@ -1,457 +1,241 @@
-const { normalizeName, getEarliestDataDates }                          = require('../utils/utils');
+/**
+ * @fileoverview Executor for "Standard" (User-Level) calculations.
+ * REFACTORED: Hoisted data loading, centralized Series/Root logic.
+ */
+const { normalizeName, getEarliestDataDates } = require('../utils/utils');
 const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs, getHistoryPartRefs } = require('../utils/data_loader');
-const { CachedDataLoader }                                             = require('../data/CachedDataLoader');
-const { ContextFactory }                                               = require('../context/ContextFactory');
-const { commitResults }                                                = require('../persistence/ResultCommitter');
-const { fetchResultSeries }                                            = require('../data/DependencyFetcher');
-const { getManifest }                                                  = require('../topology/ManifestLoader'); 
-const mathLayer                                                        = require('../layers/index');
-const { performance }                                                  = require('perf_hooks');
-const v8                                                               = require('v8'); 
+const { CachedDataLoader } = require('../data/CachedDataLoader');
+const { ContextFactory } = require('../context/ContextFactory');
+const { commitResults } = require('../persistence/ResultCommitter');
+const { fetchResultSeries } = require('../data/DependencyFetcher');
+const { getManifest } = require('../topology/ManifestLoader');
+const mathLayer = require('../layers/index');
+const { performance } = require('perf_hooks');
+const v8 = require('v8');
 
 class StandardExecutor {
+
+    // =========================================================================
+    // ENTRY POINT
+    // =========================================================================
     static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
-        const dStr   = date.toISOString().slice(0, 10);
-        const logger = deps.logger;
-
-        // Determine required user types for this batch of calculations
-        const requiredUserTypes = new Set();
-        calcs.forEach(c => {
-            const type = (c.userType || 'ALL').toUpperCase();
-            requiredUserTypes.add(type);
-        });
+        const dStr = date.toISOString().slice(0, 10);
+        const { logger } = deps;
+
+        // 1. Setup Scope (User Types & Target CID)
+        const requiredUserTypes = new Set(calcs.map(c => (c.userType || 'ALL').toUpperCase()));
         const userTypeArray = requiredUserTypes.has('ALL') ? null : Array.from(requiredUserTypes);
         
-        // [OPTIMIZATION] Check for Target CID in manifests (On-Demand Optimization)
         const targetCid = calcs.find(c => c.targetCid)?.targetCid || calcs.find(c => c.manifest?.targetCid)?.manifest?.targetCid;
-        if (targetCid) {
-            logger.log('INFO', `[StandardExecutor] Running in Targeted Mode for CID: ${targetCid}`);
-        }
+        if (targetCid) logger.log('INFO', `[StandardExecutor] 🎯 Targeted Mode: CID ${targetCid}`);
 
+        // 2. Prepare History Context (Refs)
         const fullRoot = { ...rootData };
         if (calcs.some(c => c.isHistorical)) {
-            const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
-            const prevStr = prev.toISOString().slice(0, 10);
-            
-            // Fetch yesterday's refs
+            const prevStr = new Date(date.getTime() - 86400000).toISOString().slice(0, 10);
             let yRefs = await getPortfolioPartRefs(config, deps, prevStr, userTypeArray);
             
-            // [OPTIMIZATION] Filter Yesterday's Refs if targetCid is set
             if (targetCid && yRefs) {
-                const originalCount = yRefs.length;
                 yRefs = yRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
-                logger.log('INFO', `[StandardExecutor] Filtered Yesterday's Refs: ${originalCount} -> ${yRefs.length}`);
             }
-            
             fullRoot.yesterdayPortfolioRefs = yRefs;
         }
 
+        // 3. Initialize Instances
         const state = {};
-        for (const c of calcs) { 
-            try { 
-                const inst = new c.class(); 
-                inst.manifest = c; 
-                inst.results = {}; 
-                state[normalizeName(c.name)] = inst; 
-                logger.log('INFO', `${c.name} calculation running for ${dStr}`); 
-            } catch (e) { logger.log('WARN', `Failed to init ${c.name}`); } 
+        for (const c of calcs) {
+            try {
+                const inst = new c.class();
+                inst.manifest = c;
+                inst.results = {};
+                state[normalizeName(c.name)] = inst;
+            } catch (e) { logger.log('WARN', `Failed to init ${c.name}`); }
         }
 
         return await StandardExecutor.streamAndProcess(
-            dStr, state, passName, config, deps, fullRoot, 
-            rootData.portfolioRefs, rootData.historyRefs, 
-            fetchedDeps, previousFetchedDeps, skipStatusWrite, 
+            dStr, state, passName, config, deps, fullRoot,
+            rootData.portfolioRefs, rootData.historyRefs,
+            fetchedDeps, previousFetchedDeps, skipStatusWrite,
             userTypeArray, targetCid
         );
     }
 
+    // =========================================================================
+    // STREAMING LOOP
+    // =========================================================================
     static async streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite, requiredUserTypes = null, targetCid = null) {
         const { logger } = deps;
-        const calcs = Object.values(state).filter(c => c && c.manifest);
+        const calcs = Object.values(state).filter(c => c?.manifest);
         const streamingCalcs = calcs.filter(c => c.manifest.rootDataDependencies.includes('portfolio') || c.manifest.rootDataDependencies.includes('history'));
         
-        if (streamingCalcs.length === 0) return { successUpdates: {}, failureReport: [] };
-
-        // --- 0. [NEW] Build Global Manifest Lookup ---
-        const allManifests = getManifest(config.productLines, config.calculationsDirectory, deps); 
-        const manifestLookup = {};
-        allManifests.forEach(m => { manifestLookup[normalizeName(m.name)] = m.category; });
+        if (!streamingCalcs.length) return { successUpdates: {}, failureReport: [] };
 
-        // --- 1. Resolve and Filter Portfolio Refs (Today) ---
-        let effectivePortfolioRefs = portfolioRefs;
-        if (!effectivePortfolioRefs) {
-            effectivePortfolioRefs = await getPortfolioPartRefs(config, deps, dateStr, requiredUserTypes);
-        }
-        if (targetCid && effectivePortfolioRefs) {
-            effectivePortfolioRefs = effectivePortfolioRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
-        }
-
-        // --- 2. Resolve and Filter History Refs ---
-        let effectiveHistoryRefs = historyRefs;
-        const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-        
-        if (needsTradingHistory) {
-            if (!effectiveHistoryRefs) {
-                effectiveHistoryRefs = await getHistoryPartRefs(config, deps, dateStr, requiredUserTypes);
-            }
-            if (targetCid && effectiveHistoryRefs) {
-                effectiveHistoryRefs = effectiveHistoryRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
-            }
-        }
+        const loader = new CachedDataLoader(config, deps);
         
-        let totalReadOps = (effectivePortfolioRefs?.length || 0) + (effectiveHistoryRefs?.length || 0);
-        if (rootData.yesterdayPortfolioRefs) totalReadOps += rootData.yesterdayPortfolioRefs.length;
-        totalReadOps += 2; 
-
-        const readOpsPerCalc = Math.ceil(totalReadOps / streamingCalcs.length);
-
-        const executionStats = {};
-        const shardIndexMap  = {}; 
-        const aggregatedSuccess = {};
-        const aggregatedFailures = [];
-        const errorStats = { count: 0, total: 0 };
-
-        Object.keys(state).forEach(name => {
-            executionStats[name] = { 
-                processedUsers: 0, skippedUsers: 0, timings: { setup: 0, stream: 0, processing: 0 }
-            };
-            shardIndexMap[name] = 0;
-        });
-
-        let hasFlushed = false;
-        const cachedLoader = new CachedDataLoader(config, deps);
+        // --- 1. PRE-LOAD GLOBAL DATA (Hoisted) ---
+        // Load all "Singleton" datasets once (Ratings, Rankings, Series, Mappings)
+        // instead of checking/loading per-user inside the loop.
         const startSetup = performance.now();
         
-        // [OPTIMIZATION] Hoist Static Data Load out of User Loop
-        const mappings = await cachedLoader.loadMappings();
-        // Pre-load Master List to cache it once
-        const piMasterList = await cachedLoader.loadPIMasterList();
+        const [
+            globalRoots, 
+            seriesData, 
+            earliestDates
+        ] = await Promise.all([
+            loadGlobalRoots(loader, dateStr, streamingCalcs, deps),
+            loadSeriesData(loader, dateStr, streamingCalcs, config, deps),
+            streamingCalcs.some(c => c.manifest.requiresEarliestDataDate) ? getEarliestDataDates(config, deps) : null
+        ]);
 
         const setupDuration = performance.now() - startSetup;
-        Object.keys(executionStats).forEach(name => executionStats[name].timings.setup += setupDuration);
 
-        // --- [NEW] Series / Lookback Loading Logic ---
-        const rootSeriesRequests = {};
-        const dependencySeriesRequests = {};
+        // --- 2. PREPARE REFS & STATS ---
+        let effPortRefs = portfolioRefs || await getPortfolioPartRefs(config, deps, dateStr, requiredUserTypes);
+        let effHistRefs = historyRefs;
         
-        // 1. Identify requirements from manifests
-        streamingCalcs.forEach(c => {
-            // Check for Root Data Series (e.g., { insights: 7, alerts: 30 })
-            if (c.manifest.rootDataSeries) {
-                Object.entries(c.manifest.rootDataSeries).forEach(([type, conf]) => {
-                    const days = typeof conf === 'object' ? conf.lookback : conf;
-                    if (!rootSeriesRequests[type] || days > rootSeriesRequests[type]) {
-                        rootSeriesRequests[type] = days;
-                    }
-                });
-            }
-            // Check for Computation Result Series (e.g., { 'RiskScore': 7 })
-            if (c.manifest.dependencySeries) {
-                Object.entries(c.manifest.dependencySeries).forEach(([depName, conf]) => {
-                    const days = typeof conf === 'object' ? conf.lookback : conf;
-                    const normalized = normalizeName(depName);
-                    if (!dependencySeriesRequests[normalized] || days > dependencySeriesRequests[normalized]) {
-                        dependencySeriesRequests[normalized] = days;
-                    }
-                });
-            }
-        });
-
-        const seriesData = { root: {}, results: {} };
-
-        // 2. Load Root Series
-        for (const [type, days] of Object.entries(rootSeriesRequests)) {
-            let loaderMethod = null;
-            if (type === 'alerts') loaderMethod = 'loadAlertHistory';
-            else if (type === 'insights') loaderMethod = 'loadInsights';
-            else if (type === 'ratings') loaderMethod = 'loadRatings';
-            else if (type === 'watchlist') loaderMethod = 'loadWatchlistMembership';
-            // [CRITICAL UPDATE] Add rankings support for AUM lookback
-            else if (type === 'rankings') loaderMethod = 'loadRankings';
-
-            if (loaderMethod) {
-                logger.log('INFO', `[StandardExecutor] Loading ${days}-day series for Root Data '${type}'...`);
-                // Assume CachedDataLoader has loadSeries method (added in previous step)
-                const series = await cachedLoader.loadSeries(loaderMethod, dateStr, days);
-                seriesData.root[type] = series.data; // map of date -> data
-            }
-        }
+        if (targetCid) effPortRefs = effPortRefs?.filter(r => !r.cid || String(r.cid) === String(targetCid));
 
-        // 3. Load Computation Result Series
-        const calcNamesToFetch = Object.keys(dependencySeriesRequests);
-        if (calcNamesToFetch.length > 0) {
-            const maxDays = Math.max(...Object.values(dependencySeriesRequests));
-            logger.log('INFO', `[StandardExecutor] Loading up to ${maxDays}-day series for Dependencies: ${calcNamesToFetch.join(', ')}`);
-            
-            // Pass manifestLookup to fetcher
-            const resultsSeries = await fetchResultSeries(dateStr, calcNamesToFetch, manifestLookup, config, deps, maxDays);
-            seriesData.results = resultsSeries; 
+        const needsHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
+        if (needsHistory) {
+            effHistRefs = effHistRefs || await getHistoryPartRefs(config, deps, dateStr, requiredUserTypes);
+            if (targetCid) effHistRefs = effHistRefs?.filter(r => !r.cid || String(r.cid) === String(targetCid));
         }
-        // ---------------------------------------------
 
-        const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-        const prevDateStr = prevDate.toISOString().slice(0, 10);
-        
-        let earliestDates = null;
-        if (streamingCalcs.some(c => c.manifest.requiresEarliestDataDate)) {
-            earliestDates = await getEarliestDataDates(config, deps);
-        }
+        const stats = {}; 
+        const shardMap = {};
+        calcs.forEach(c => { 
+            stats[normalizeName(c.manifest.name)] = { 
+                processedUsers: 0, skippedUsers: 0, timings: { setup: setupDuration, stream: 0, processing: 0 } 
+            };
+            shardMap[normalizeName(c.manifest.name)] = 0;
+        });
 
-        const tP_iter = streamPortfolioData(config, deps, dateStr, effectivePortfolioRefs, requiredUserTypes);
-        
-        const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
-        const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs)  ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
-        
-        const tH_iter = (needsTradingHistory) ? streamHistoryData(config, deps, dateStr, effectiveHistoryRefs, requiredUserTypes) : null;
+        // --- 3. STREAM EXECUTION ---
+        const tP_iter = streamPortfolioData(config, deps, dateStr, effPortRefs, requiredUserTypes);
+        const yP_iter = (streamingCalcs.some(c => c.manifest.isHistorical) && rootData.yesterdayPortfolioRefs) 
+                        ? streamPortfolioData(config, deps, new Date(new Date(dateStr).getTime() - 86400000).toISOString().slice(0, 10), rootData.yesterdayPortfolioRefs) 
+                        : null;
+        const tH_iter = needsHistory ? streamHistoryData(config, deps, dateStr, effHistRefs, requiredUserTypes) : null;
 
         let yP_chunk = {}, tH_chunk = {};
-        let usersSinceLastFlush = 0;
+        let usersSinceFlush = 0;
+        let hasFlushed = false;
+        const aggregatedSuccess = {}; 
+        const aggregatedFailures = [];
 
         try {
             for await (const tP_chunk of tP_iter) {
-                const startStream = performance.now();
+                const t0 = performance.now();
                 if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
                 if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
-                const streamDuration = performance.now() - startStream;
-                Object.keys(executionStats).forEach(name => executionStats[name].timings.stream += streamDuration);
+                const tStream = performance.now() - t0;
 
-                const chunkSize = Object.keys(tP_chunk).length;
-                const startProcessing = performance.now();
+                const t1 = performance.now();
                 
-                const batchResults = await Promise.all(streamingCalcs.map(calc =>  
+                // Parallelize Calc Execution for this Chunk
+                await Promise.all(streamingCalcs.map(calc => 
                     StandardExecutor.executePerUser(
-                        calc, calc.manifest, dateStr, tP_chunk, yP_chunk, tH_chunk, 
-                        fetchedDeps, previousFetchedDeps, config, deps, cachedLoader,
-                        executionStats[normalizeName(calc.manifest.name)],
+                        calc, calc.manifest, dateStr, tP_chunk, yP_chunk, tH_chunk,
+                        fetchedDeps, previousFetchedDeps, config, deps,
+                        stats[normalizeName(calc.manifest.name)],
                         earliestDates,
                         seriesData,
-                        // [NEW] Pass Hoisted Data
-                        mappings,
-                        piMasterList
-                    ) 
+                        globalRoots // <--- PASSED DOWN
+                    )
                 ));
                 
-                const procDuration = performance.now() - startProcessing;
-                Object.keys(executionStats).forEach(name => executionStats[name].timings.processing += procDuration);
-
-                batchResults.forEach(r => { errorStats.total += (r.success + r.failures); errorStats.count += r.failures; });
-                if (errorStats.total > 100 && (errorStats.count / errorStats.total) > 0.10) { throw new Error(`[Circuit Breaker] High failure rate detected.`); }
+                const tProc = performance.now() - t1;
+                streamingCalcs.forEach(c => {
+                    const s = stats[normalizeName(c.manifest.name)];
+                    s.timings.stream += tStream;
+                    s.timings.processing += tProc;
+                });
 
-                usersSinceLastFlush += chunkSize;
-                const heapStats = v8.getHeapStatistics();
-                if (usersSinceLastFlush >= 500 || (heapStats.used_heap_size / heapStats.heap_size_limit) > 0.70) {
-                    const flushResult = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, 'INTERMEDIATE', true, !hasFlushed);
+                usersSinceFlush += Object.keys(tP_chunk).length;
+                const heap = v8.getHeapStatistics();
+                if (usersSinceFlush >= 500 || (heap.used_heap_size / heap.heap_size_limit) > 0.7) {
+                    const res = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardMap, stats, 'INTERMEDIATE', true, !hasFlushed);
+                    StandardExecutor.mergeReports(aggregatedSuccess, aggregatedFailures, res);
                     hasFlushed = true;
-                    StandardExecutor.mergeReports(aggregatedSuccess, aggregatedFailures, flushResult);
-                    usersSinceLastFlush = 0;
+                    usersSinceFlush = 0;
                 }
             }
         } finally {
-            if (yP_iter && yP_iter.return) await yP_iter.return();
-            if (tH_iter && tH_iter.return) await tH_iter.return();
+            if (yP_iter?.return) await yP_iter.return();
+            if (tH_iter?.return) await tH_iter.return();
         }
 
-        const finalResult = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, 'FINAL', skipStatusWrite, !hasFlushed);
-        StandardExecutor.mergeReports(aggregatedSuccess, aggregatedFailures, finalResult);
-
-        Object.values(aggregatedSuccess).forEach(update => {
-            if (!update.metrics.io) update.metrics.io = { reads: 0, writes: 0, deletes: 0 };
-            update.metrics.io.reads = readOpsPerCalc;
-        });
+        const finalRes = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardMap, stats, 'FINAL', skipStatusWrite, !hasFlushed);
+        StandardExecutor.mergeReports(aggregatedSuccess, aggregatedFailures, finalRes);
 
         return { successUpdates: aggregatedSuccess, failureReport: aggregatedFailures };
     }
-    
-    static async flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, mode, skipStatusWrite, isInitialWrite = false) {
-        const { logger } = deps;
-        const transformedState = {};
-        for (const [name, inst] of Object.entries(state)) {
-            const rawResult = inst.results || {};
-            const firstUser = Object.keys(rawResult)[0];
-            let dataToCommit = rawResult;
-
-            if (firstUser && rawResult[firstUser] && typeof rawResult[firstUser] === 'object') {
-                const innerKeys = Object.keys(rawResult[firstUser]);
-                if (innerKeys.length > 0 && innerKeys.every(k => /^\d{4}-\d{2}-\d{2}$/.test(k))) {
-                    const transposed = {};
-                    for (const [userId, dateMap] of Object.entries(rawResult)) {
-                        for (const [dateKey, dailyData] of Object.entries(dateMap)) {
-                            if (!transposed[dateKey]) transposed[dateKey] = {};
-                            transposed[dateKey][userId] = dailyData;
-                        }
-                    }
-                    dataToCommit = transposed;
-                }
-            }
 
-            transformedState[name] = {
-                manifest: inst.manifest,
-                getResult: async () => dataToCommit,
-                _executionStats: executionStats[name]
-            };
-            inst.results = {};
-        }
-
-        const result = await commitResults(transformedState, dateStr, passName, config, deps, skipStatusWrite, {
-            flushMode: mode, shardIndexes: shardIndexMap, isInitialWrite: isInitialWrite 
-        });
-
-        if (result.shardIndexes) Object.assign(shardIndexMap, result.shardIndexes);
-        return result;
-    }
-
-    static mergeReports(successAcc, failureAcc, newResult) {
-        if (!newResult) return;
-        for (const [name, update] of Object.entries(newResult.successUpdates)) {
-            if (!successAcc[name]) {
-                successAcc[name] = update;
-            } else {
-                successAcc[name].metrics.storage.sizeBytes += (update.metrics.storage.sizeBytes || 0);
-                successAcc[name].metrics.storage.keys      += (update.metrics.storage.keys || 0);
-                successAcc[name].metrics.storage.shardCount = Math.max(successAcc[name].metrics.storage.shardCount, update.metrics.storage.shardCount || 1);
-                
-                if (update.metrics.io) {
-                    if (!successAcc[name].metrics.io) successAcc[name].metrics.io = { writes: 0, deletes: 0, reads: 0 };
-                    successAcc[name].metrics.io.writes += (update.metrics.io.writes || 0);
-                    successAcc[name].metrics.io.deletes += (update.metrics.io.deletes || 0);
-                }
-
-                if (update.metrics?.execution?.timings) {
-                     if (!successAcc[name].metrics.execution) successAcc[name].metrics.execution = { timings: { setup:0, stream:0, processing:0 }};
-                     const tDest = successAcc[name].metrics.execution.timings;
-                     const tSrc  = update.metrics.execution.timings;
-                     tDest.setup += (tSrc.setup || 0);
-                     tDest.stream += (tSrc.stream || 0);
-                     tDest.processing += (tSrc.processing || 0);
-                }
-                successAcc[name].hash = update.hash;
-            }
-        }
-        if (newResult.failureReport) failureAcc.push(...newResult.failureReport);
-    }
-    
-   static async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps, config, deps, loader, stats, earliestDates, seriesData = {}, mappings = null, piMasterList = null) {
+    // =========================================================================
+    // PER-USER EXECUTION (Pure Logic)
+    // =========================================================================
+    static async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps, config, deps, stats, earliestDates, seriesData, globalRoots) {
         const { logger } = deps;
-        const targetUserType = metadata.userType; 
-        
-        // [OPTIMIZATION] Use passed mappings/list if available, else load (fallback)
-        const mappingsToUse = mappings || await loader.loadMappings();
-        const piMasterListToUse = piMasterList || await loader.loadPIMasterList();
-        
+        const targetUserType = metadata.userType;
         const SCHEMAS = mathLayer.SCHEMAS;
-
-        // 1. Load Root Data (CachedLoader handles memoization for these)
-        const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
-        const verifications = metadata.rootDataDependencies?.includes('verification') ? await loader.loadVerifications() : null;
-        const rankings = metadata.rootDataDependencies?.includes('rankings') ? await loader.loadRankings(dateStr) : null;
-        
-        let yesterdayRankings = null;
-        if (metadata.rootDataDependencies?.includes('rankings') && metadata.isHistorical) {
-             const prevDate = new Date(dateStr); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-             const prevStr = prevDate.toISOString().slice(0, 10);
-             yesterdayRankings = await loader.loadRankings(prevStr); 
-        }
-
-        const socialContainer = metadata.rootDataDependencies?.includes('social') ? await loader.loadSocial(dateStr) : null;
         
-        const allowMissing = metadata.canHaveMissingRoots === true;
-        
-        // Helper to safely load roots
-        const safeLoad = async (method, name) => {
-             if (!metadata.rootDataDependencies?.includes(name)) return null;
-             try {
-                 return await loader[method](dateStr);
-             } catch (e) {
-                 if (!allowMissing) throw new Error(`[StandardExecutor] Required root '${name}' failed: ${e.message}`);
-                 return null;
-             }
-        };
-
-        const ratings = await safeLoad('loadRatings', 'ratings');
-        const pageViews = await safeLoad('loadPageViews', 'pageViews');
-        const watchlistMembership = await safeLoad('loadWatchlistMembership', 'watchlist');
-        const alertHistory = await safeLoad('loadAlertHistory', 'alerts');
-
-        if (!allowMissing) {
-             if (metadata.rootDataDependencies?.includes('ratings') && !ratings) throw new Error("Missing ratings");
-             if (metadata.rootDataDependencies?.includes('pageViews') && !pageViews) throw new Error("Missing pageViews");
-             if (metadata.rootDataDependencies?.includes('watchlist') && !watchlistMembership) throw new Error("Missing watchlist");
-             if (metadata.rootDataDependencies?.includes('alerts') && !alertHistory) throw new Error("Missing alerts");
-        }
-
-        let chunkSuccess = 0;
-        let chunkFailures = 0;
+        let success = 0, failures = 0;
 
         for (const [userId, todayPortfolio] of Object.entries(portfolioData)) {
-            // --- OPTIMIZATION: TARGET SPECIFIC USER ---
-            // If the request contains a targetCid, skip all other users immediately
-            if (metadata.targetCid && String(userId) !== String(metadata.targetCid)) {
-                if (stats) stats.skippedUsers++;
-                continue;
-            }
-            // ------------------------------------------
+            // 1. Filter User
+            if (metadata.targetCid && String(userId) !== String(metadata.targetCid)) { stats.skippedUsers++; continue; }
             
-            const yesterdayPortfolio = yesterdayPortfolioData ? yesterdayPortfolioData[userId] : null;
-            const todayHistory = historyData ? historyData[userId] : null;
-            
-            let actualUserType = todayPortfolio._userType;
-            if (!actualUserType) {
-                if (todayPortfolio.PublicPositions) {
-                    const isRanked = rankings && rankings.some(r => String(r.CustomerId) === String(userId));
-                    actualUserType = isRanked ? 'POPULAR_INVESTOR' : SCHEMAS.USER_TYPES.SPECULATOR;
-                } else {
-                    actualUserType = SCHEMAS.USER_TYPES.NORMAL;
-                }
-            }
-            
-            if (targetUserType && targetUserType !== 'all') {
-                 if (targetUserType !== actualUserType) {
-                     if (stats) stats.skippedUsers++;
-                     continue;
-                 }
+            // 2. Determine Type
+            let actualType = todayPortfolio._userType;
+            if (!actualType) {
+                const isRanked = globalRoots.rankings && globalRoots.rankings.some(r => String(r.CustomerId) === String(userId));
+                actualType = isRanked ? 'POPULAR_INVESTOR' : (todayPortfolio.PublicPositions ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL);
             }
-            
-            const userVerification = verifications ? verifications[userId] : null;
-            
-            const userRanking = rankings ? (rankings.find(r => String(r.CustomerId) === String(userId)) || null) : null;
-            const userRankingYesterday = yesterdayRankings ? (yesterdayRankings.find(r => String(r.CustomerId) === String(userId)) || null) : null;
-
-            let effectiveSocialData = null;
-            if (socialContainer) {
-                if (actualUserType === 'POPULAR_INVESTOR') {
-                    effectiveSocialData = socialContainer.pi[userId] || {}; 
-                } else if (actualUserType === 'SIGNED_IN_USER') {
-                    effectiveSocialData = socialContainer.signedIn[userId] || {};
-                } else {
-                    effectiveSocialData = socialContainer.generic || {};
-                }
+            if (targetUserType && targetUserType !== 'all' && targetUserType !== actualType) { stats.skippedUsers++; continue; }
+
+            // 3. Resolve User Specifics from Global Data
+            const userRank = globalRoots.rankings?.find(r => String(r.CustomerId) === String(userId)) || null;
+            const userRankYest = globalRoots.rankingsYesterday?.find(r => String(r.CustomerId) === String(userId)) || null;
+            const userVerify = globalRoots.verifications?.[userId] || null;
+
+            let social = null;
+            if (globalRoots.social) {
+                social = (actualType === 'POPULAR_INVESTOR' ? globalRoots.social.pi[userId] : 
+                         (actualType === 'SIGNED_IN_USER' ? globalRoots.social.signedIn[userId] : globalRoots.social.generic)) || {};
             }
 
-            const context = ContextFactory.buildPerUserContext({ 
-                todayPortfolio, yesterdayPortfolio, todayHistory, userId, 
-                userType: actualUserType, dateStr, metadata, 
-                mappings: mappingsToUse, 
-                insights, 
-                socialData: effectiveSocialData ? { today: effectiveSocialData } : null,
-                computedDependencies: computedDeps, previousComputedDependencies: prevDeps, 
-                config, deps,
-                verification: userVerification, 
+            // 4. Build Context
+            const context = ContextFactory.buildPerUserContext({
+                todayPortfolio, 
+                yesterdayPortfolio: yesterdayPortfolioData?.[userId] || null, 
+                todayHistory: historyData?.[userId] || null,
+                userId, userType: actualType, dateStr, metadata,
                 
-                rankings: userRanking,
-                yesterdayRankings: userRankingYesterday, 
+                // Injected Global Data
+                mappings: globalRoots.mappings,
+                piMasterList: globalRoots.piMasterList,
+                insights: globalRoots.insights,
+                socialData: social ? { today: social } : null,
                 
-                allRankings: rankings,
-                allRankingsYesterday: yesterdayRankings,
+                // Dependency Data
+                computedDependencies: computedDeps, 
+                previousComputedDependencies: prevDeps,
+                config, deps,
                 
-                allVerifications: verifications,
+                // Specific Lookups
+                verification: userVerify,
+                rankings: userRank,
+                yesterdayRankings: userRankYest,
+                
+                // Full Access (if needed by calc)
+                allRankings: globalRoots.rankings,
+                allRankingsYesterday: globalRoots.rankingsYesterday,
+                allVerifications: globalRoots.verifications,
+                ratings: globalRoots.ratings || {},
+                pageViews: globalRoots.pageViews || {},
+                watchlistMembership: globalRoots.watchlistMembership || {},
+                alertHistory: globalRoots.alertHistory || {},
                 
-                ratings: ratings || {},
-                pageViews: pageViews || {},
-                watchlistMembership: watchlistMembership || {},
-                alertHistory: alertHistory || {},
-
-                piMasterList: piMasterListToUse,
-                // [NEW] Pass Series Data
                 seriesData
             });
 
@@ -459,18 +243,176 @@ class StandardExecutor {
                 if (!context.system) context.system = {};
                 context.system.earliestHistoryDate = earliestDates.history;
             }
-            
-            try { 
-                await calcInstance.process(context); 
-                if (stats) stats.processedUsers++;
-                chunkSuccess++;
-            } catch (e) { 
-                logger.log('WARN', `Calc ${metadata.name} failed for user ${userId}: ${e.message}`); 
-                chunkFailures++;
+
+            try {
+                await calcInstance.process(context);
+                stats.processedUsers++;
+                success++;
+            } catch (e) {
+                logger.log('WARN', `Calc ${metadata.name} failed for user ${userId}: ${e.message}`);
+                failures++;
             }
         }
-        return { success: chunkSuccess, failures: chunkFailures };
+        return { success, failures };
     }
+
+    // =========================================================================
+    // HELPERS (Flush & Merge)
+    // =========================================================================
+    static async flushBuffer(state, dateStr, passName, config, deps, shardMap, stats, mode, skipStatus, isInitial) {
+        const transformedState = {};
+        for (const [name, inst] of Object.entries(state)) {
+            let data = inst.results || {};
+            // Pivot user-date structure if needed
+            const first = Object.keys(data)[0];
+            if (first && data[first] && typeof data[first] === 'object' && /^\d{4}-\d{2}-\d{2}$/.test(Object.keys(data[first])[0])) {
+                const pivoted = {};
+                for (const [uid, dMap] of Object.entries(data)) {
+                    for (const [dKey, val] of Object.entries(dMap)) {
+                        if (!pivoted[dKey]) pivoted[dKey] = {};
+                        pivoted[dKey][uid] = val;
+                    }
+                }
+                data = pivoted;
+            }
+            transformedState[name] = { manifest: inst.manifest, getResult: async () => data, _executionStats: stats[name] };
+            inst.results = {}; // Clear buffer
+        }
+        const res = await commitResults(transformedState, dateStr, passName, config, deps, skipStatus, { flushMode: mode, shardIndexes: shardMap, isInitialWrite: isInitial });
+        if (res.shardIndexes) Object.assign(shardMap, res.shardIndexes);
+        return res;
+    }
+
+    static mergeReports(success, failure, result) {
+        if (!result) return;
+        if (result.failureReport) failure.push(...result.failureReport);
+        for (const [name, update] of Object.entries(result.successUpdates)) {
+            if (!success[name]) success[name] = update;
+            else {
+                const m = success[name].metrics;
+                const u = update.metrics;
+                m.storage.sizeBytes += u.storage.sizeBytes;
+                m.storage.keys += u.storage.keys;
+                m.storage.shardCount = Math.max(m.storage.shardCount, u.storage.shardCount);
+                if (u.execution) {
+                    m.execution.timings.stream += u.execution.timings.stream;
+                    m.execution.timings.processing += u.execution.timings.processing;
+                }
+            }
+        }
+    }
+}
+
+// =============================================================================
+// SHARED LOADING HELPERS
+// =============================================================================
+
+/**
+ * Pre-loads all shared global datasets required by the active calculations.
+ * Returns a consolidated object of { ratings, rankings, insights, ... }
+ */
+async function loadGlobalRoots(loader, dateStr, calcs, deps) {
+    const { logger } = deps;
+    const roots = {};
+    
+    // 1. Identify Requirements
+    const reqs = {
+        mappings: true,
+        piMasterList: true,
+        rankings: false,
+        rankingsYesterday: false,
+        verifications: false,
+        insights: false,
+        social: false,
+        ratings: false,
+        pageViews: false,
+        watchlist: false,
+        alerts: false
+    };
+
+    for (const c of calcs) {
+        const deps = c.manifest.rootDataDependencies || [];
+        if (deps.includes('rankings')) reqs.rankings = true;
+        if (deps.includes('rankings') && c.manifest.isHistorical) reqs.rankingsYesterday = true;
+        if (deps.includes('verification')) reqs.verifications = true;
+        if (deps.includes('insights')) reqs.insights = true;
+        if (deps.includes('social')) reqs.social = true;
+        if (deps.includes('ratings')) reqs.ratings = true;
+        if (deps.includes('pageViews')) reqs.pageViews = true;
+        if (deps.includes('watchlist')) reqs.watchlist = true;
+        if (deps.includes('alerts')) reqs.alerts = true;
+    }
+
+    // 2. Fetch Helper
+    const fetch = async (key, method, dateArg, optional = true) => {
+        if (!reqs[key]) return;
+        try {
+            roots[key] = await loader[method](dateArg);
+        } catch (e) {
+            if (!optional) throw e;
+            logger.log('WARN', `[StandardExecutor] Optional root '${key}' failed to load: ${e.message}`);
+            roots[key] = null;
+        }
+    };
+
+    // 3. Execute Loads
+    await Promise.all([
+        fetch('mappings', 'loadMappings', null, false), // Always required
+        fetch('piMasterList', 'loadPIMasterList', null, false),
+        fetch('rankings', 'loadRankings', dateStr),
+        fetch('verifications', 'loadVerifications', dateStr),
+        fetch('insights', 'loadInsights', dateStr),
+        fetch('social', 'loadSocial', dateStr),
+        fetch('ratings', 'loadRatings', dateStr),
+        fetch('pageViews', 'loadPageViews', dateStr),
+        fetch('watchlist', 'loadWatchlistMembership', dateStr),
+        fetch('alerts', 'loadAlertHistory', dateStr)
+    ]);
+
+    if (reqs.rankingsYesterday) {
+        const prev = new Date(new Date(dateStr).getTime() - 86400000).toISOString().slice(0, 10);
+        await fetch('rankingsYesterday', 'loadRankings', prev);
+    }
+    
+    // Map internal names to match loadGlobalRoots structure if needed
+    roots.watchlistMembership = roots.watchlist;
+    roots.alertHistory = roots.alerts;
+    
+    return roots;
+}
+
+async function loadSeriesData(loader, dateStr, calcs, config, deps) {
+    const rootReqs = {};
+    const depReqs = {};
+
+    calcs.forEach(c => {
+        if (c.manifest.rootDataSeries) {
+            Object.entries(c.manifest.rootDataSeries).forEach(([k, v]) => rootReqs[k] = Math.max(rootReqs[k]||0, v.lookback||v));
+        }
+        if (c.manifest.dependencySeries) {
+            Object.entries(c.manifest.dependencySeries).forEach(([k, v]) => depReqs[normalizeName(k)] = Math.max(depReqs[normalizeName(k)]||0, v.lookback||v));
+        }
+    });
+
+    const series = { root: {}, results: {} };
+    const rootMap = {
+        alerts: 'loadAlertHistory', insights: 'loadInsights', ratings: 'loadRatings',
+        watchlist: 'loadWatchlistMembership', rankings: 'loadRankings'
+    };
+
+    await Promise.all(Object.entries(rootReqs).map(async ([key, days]) => {
+        if (rootMap[key]) series.root[key] = (await loader.loadSeries(rootMap[key], dateStr, days)).data;
+    }));
+
+    const depNames = Object.keys(depReqs);
+    if (depNames.length) {
+        // Construct manifest lookup on the fly for fetched names
+        const allManifests = getManifest(config.productLines, config.calculationsDirectory, deps);
+        const lookup = Object.fromEntries(allManifests.map(m => [normalizeName(m.name), m.category]));
+        series.results = await fetchResultSeries(dateStr, depNames, lookup, config, deps, Math.max(...Object.values(depReqs)));
+    }
+
+    return series;
 }
 
 module.exports = { StandardExecutor };