bulltrackers-module 1.0.732 → 1.0.733

package/functions/computation-system/legacy/StandardExecutorold.js

@@ -1,476 +0,0 @@
-const { normalizeName, getEarliestDataDates }                          = require('../utils/utils');
-const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs, getHistoryPartRefs } = require('../utils/data_loader');
-const { CachedDataLoader }                                             = require('../data/CachedDataLoader');
-const { ContextFactory }                                               = require('../context/ContextFactory');
-const { commitResults }                                                = require('../persistence/ResultCommitter');
-const { fetchResultSeries }                                            = require('../data/DependencyFetcher');
-const { getManifest }                                                  = require('../topology/ManifestLoader'); 
-const mathLayer                                                        = require('../layers/index');
-const { performance }                                                  = require('perf_hooks');
-const v8                                                               = require('v8'); 
-
-class StandardExecutor {
-    static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
-        const dStr   = date.toISOString().slice(0, 10);
-        const logger = deps.logger;
-
-        // Determine required user types for this batch of calculations
-        const requiredUserTypes = new Set();
-        calcs.forEach(c => {
-            const type = (c.userType || 'ALL').toUpperCase();
-            requiredUserTypes.add(type);
-        });
-        const userTypeArray = requiredUserTypes.has('ALL') ? null : Array.from(requiredUserTypes);
-        
-        // [OPTIMIZATION] Check for Target CID in manifests (On-Demand Optimization)
-        const targetCid = calcs.find(c => c.targetCid)?.targetCid || calcs.find(c => c.manifest?.targetCid)?.manifest?.targetCid;
-        if (targetCid) {
-            logger.log('INFO', `[StandardExecutor] Running in Targeted Mode for CID: ${targetCid}`);
-        }
-
-        const fullRoot = { ...rootData };
-        if (calcs.some(c => c.isHistorical)) {
-            const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
-            const prevStr = prev.toISOString().slice(0, 10);
-            
-            // Fetch yesterday's refs
-            let yRefs = await getPortfolioPartRefs(config, deps, prevStr, userTypeArray);
-            
-            // [OPTIMIZATION] Filter Yesterday's Refs if targetCid is set
-            if (targetCid && yRefs) {
-                const originalCount = yRefs.length;
-                yRefs = yRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
-                logger.log('INFO', `[StandardExecutor] Filtered Yesterday's Refs: ${originalCount} -> ${yRefs.length}`);
-            }
-            
-            fullRoot.yesterdayPortfolioRefs = yRefs;
-        }
-
-        const state = {};
-        for (const c of calcs) { 
-            try { 
-                const inst = new c.class(); 
-                inst.manifest = c; 
-                inst.results = {}; 
-                state[normalizeName(c.name)] = inst; 
-                logger.log('INFO', `${c.name} calculation running for ${dStr}`); 
-            } catch (e) { logger.log('WARN', `Failed to init ${c.name}`); } 
-        }
-
-        return await StandardExecutor.streamAndProcess(
-            dStr, state, passName, config, deps, fullRoot, 
-            rootData.portfolioRefs, rootData.historyRefs, 
-            fetchedDeps, previousFetchedDeps, skipStatusWrite, 
-            userTypeArray, targetCid
-        );
-    }
-
-    static async streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite, requiredUserTypes = null, targetCid = null) {
-        const { logger } = deps;
-        const calcs = Object.values(state).filter(c => c && c.manifest);
-        const streamingCalcs = calcs.filter(c => c.manifest.rootDataDependencies.includes('portfolio') || c.manifest.rootDataDependencies.includes('history'));
-        
-        if (streamingCalcs.length === 0) return { successUpdates: {}, failureReport: [] };
-
-        // --- 0. [NEW] Build Global Manifest Lookup ---
-        const allManifests = getManifest(config.productLines, config.calculationsDirectory, deps); 
-        const manifestLookup = {};
-        allManifests.forEach(m => { manifestLookup[normalizeName(m.name)] = m.category; });
-
-        // --- 1. Resolve and Filter Portfolio Refs (Today) ---
-        let effectivePortfolioRefs = portfolioRefs;
-        if (!effectivePortfolioRefs) {
-            effectivePortfolioRefs = await getPortfolioPartRefs(config, deps, dateStr, requiredUserTypes);
-        }
-        if (targetCid && effectivePortfolioRefs) {
-            effectivePortfolioRefs = effectivePortfolioRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
-        }
-
-        // --- 2. Resolve and Filter History Refs ---
-        let effectiveHistoryRefs = historyRefs;
-        const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-        
-        if (needsTradingHistory) {
-            if (!effectiveHistoryRefs) {
-                effectiveHistoryRefs = await getHistoryPartRefs(config, deps, dateStr, requiredUserTypes);
-            }
-            if (targetCid && effectiveHistoryRefs) {
-                effectiveHistoryRefs = effectiveHistoryRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
-            }
-        }
-        
-        let totalReadOps = (effectivePortfolioRefs?.length || 0) + (effectiveHistoryRefs?.length || 0);
-        if (rootData.yesterdayPortfolioRefs) totalReadOps += rootData.yesterdayPortfolioRefs.length;
-        totalReadOps += 2; 
-
-        const readOpsPerCalc = Math.ceil(totalReadOps / streamingCalcs.length);
-
-        const executionStats = {};
-        const shardIndexMap  = {}; 
-        const aggregatedSuccess = {};
-        const aggregatedFailures = [];
-        const errorStats = { count: 0, total: 0 };
-
-        Object.keys(state).forEach(name => {
-            executionStats[name] = { 
-                processedUsers: 0, skippedUsers: 0, timings: { setup: 0, stream: 0, processing: 0 }
-            };
-            shardIndexMap[name] = 0;
-        });
-
-        let hasFlushed = false;
-        const cachedLoader = new CachedDataLoader(config, deps);
-        const startSetup = performance.now();
-        
-        // [OPTIMIZATION] Hoist Static Data Load out of User Loop
-        const mappings = await cachedLoader.loadMappings();
-        // Pre-load Master List to cache it once
-        const piMasterList = await cachedLoader.loadPIMasterList();
-
-        const setupDuration = performance.now() - startSetup;
-        Object.keys(executionStats).forEach(name => executionStats[name].timings.setup += setupDuration);
-
-        // --- [NEW] Series / Lookback Loading Logic ---
-        const rootSeriesRequests = {};
-        const dependencySeriesRequests = {};
-        
-        // 1. Identify requirements from manifests
-        streamingCalcs.forEach(c => {
-            // Check for Root Data Series (e.g., { insights: 7, alerts: 30 })
-            if (c.manifest.rootDataSeries) {
-                Object.entries(c.manifest.rootDataSeries).forEach(([type, conf]) => {
-                    const days = typeof conf === 'object' ? conf.lookback : conf;
-                    if (!rootSeriesRequests[type] || days > rootSeriesRequests[type]) {
-                        rootSeriesRequests[type] = days;
-                    }
-                });
-            }
-            // Check for Computation Result Series (e.g., { 'RiskScore': 7 })
-            if (c.manifest.dependencySeries) {
-                Object.entries(c.manifest.dependencySeries).forEach(([depName, conf]) => {
-                    const days = typeof conf === 'object' ? conf.lookback : conf;
-                    const normalized = normalizeName(depName);
-                    if (!dependencySeriesRequests[normalized] || days > dependencySeriesRequests[normalized]) {
-                        dependencySeriesRequests[normalized] = days;
-                    }
-                });
-            }
-        });
-
-        const seriesData = { root: {}, results: {} };
-
-        // 2. Load Root Series
-        for (const [type, days] of Object.entries(rootSeriesRequests)) {
-            let loaderMethod = null;
-            if (type === 'alerts') loaderMethod = 'loadAlertHistory';
-            else if (type === 'insights') loaderMethod = 'loadInsights';
-            else if (type === 'ratings') loaderMethod = 'loadRatings';
-            else if (type === 'watchlist') loaderMethod = 'loadWatchlistMembership';
-            // [CRITICAL UPDATE] Add rankings support for AUM lookback
-            else if (type === 'rankings') loaderMethod = 'loadRankings';
-
-            if (loaderMethod) {
-                logger.log('INFO', `[StandardExecutor] Loading ${days}-day series for Root Data '${type}'...`);
-                // Assume CachedDataLoader has loadSeries method (added in previous step)
-                const series = await cachedLoader.loadSeries(loaderMethod, dateStr, days);
-                seriesData.root[type] = series.data; // map of date -> data
-            }
-        }
-
-        // 3. Load Computation Result Series
-        const calcNamesToFetch = Object.keys(dependencySeriesRequests);
-        if (calcNamesToFetch.length > 0) {
-            const maxDays = Math.max(...Object.values(dependencySeriesRequests));
-            logger.log('INFO', `[StandardExecutor] Loading up to ${maxDays}-day series for Dependencies: ${calcNamesToFetch.join(', ')}`);
-            
-            // Pass manifestLookup to fetcher
-            const resultsSeries = await fetchResultSeries(dateStr, calcNamesToFetch, manifestLookup, config, deps, maxDays);
-            seriesData.results = resultsSeries; 
-        }
-        // ---------------------------------------------
-
-        const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-        const prevDateStr = prevDate.toISOString().slice(0, 10);
-        
-        let earliestDates = null;
-        if (streamingCalcs.some(c => c.manifest.requiresEarliestDataDate)) {
-            earliestDates = await getEarliestDataDates(config, deps);
-        }
-
-        const tP_iter = streamPortfolioData(config, deps, dateStr, effectivePortfolioRefs, requiredUserTypes);
-        
-        const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
-        const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs)  ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
-        
-        const tH_iter = (needsTradingHistory) ? streamHistoryData(config, deps, dateStr, effectiveHistoryRefs, requiredUserTypes) : null;
-
-        let yP_chunk = {}, tH_chunk = {};
-        let usersSinceLastFlush = 0;
-
-        try {
-            for await (const tP_chunk of tP_iter) {
-                const startStream = performance.now();
-                if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
-                if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
-                const streamDuration = performance.now() - startStream;
-                Object.keys(executionStats).forEach(name => executionStats[name].timings.stream += streamDuration);
-
-                const chunkSize = Object.keys(tP_chunk).length;
-                const startProcessing = performance.now();
-                
-                const batchResults = await Promise.all(streamingCalcs.map(calc =>  
-                    StandardExecutor.executePerUser(
-                        calc, calc.manifest, dateStr, tP_chunk, yP_chunk, tH_chunk, 
-                        fetchedDeps, previousFetchedDeps, config, deps, cachedLoader,
-                        executionStats[normalizeName(calc.manifest.name)],
-                        earliestDates,
-                        seriesData,
-                        // [NEW] Pass Hoisted Data
-                        mappings,
-                        piMasterList
-                    ) 
-                ));
-                
-                const procDuration = performance.now() - startProcessing;
-                Object.keys(executionStats).forEach(name => executionStats[name].timings.processing += procDuration);
-
-                batchResults.forEach(r => { errorStats.total += (r.success + r.failures); errorStats.count += r.failures; });
-                if (errorStats.total > 100 && (errorStats.count / errorStats.total) > 0.10) { throw new Error(`[Circuit Breaker] High failure rate detected.`); }
-
-                usersSinceLastFlush += chunkSize;
-                const heapStats = v8.getHeapStatistics();
-                if (usersSinceLastFlush >= 500 || (heapStats.used_heap_size / heapStats.heap_size_limit) > 0.70) {
-                    const flushResult = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, 'INTERMEDIATE', true, !hasFlushed);
-                    hasFlushed = true;
-                    StandardExecutor.mergeReports(aggregatedSuccess, aggregatedFailures, flushResult);
-                    usersSinceLastFlush = 0;
-                }
-            }
-        } finally {
-            if (yP_iter && yP_iter.return) await yP_iter.return();
-            if (tH_iter && tH_iter.return) await tH_iter.return();
-        }
-
-        const finalResult = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, 'FINAL', skipStatusWrite, !hasFlushed);
-        StandardExecutor.mergeReports(aggregatedSuccess, aggregatedFailures, finalResult);
-
-        Object.values(aggregatedSuccess).forEach(update => {
-            if (!update.metrics.io) update.metrics.io = { reads: 0, writes: 0, deletes: 0 };
-            update.metrics.io.reads = readOpsPerCalc;
-        });
-
-        return { successUpdates: aggregatedSuccess, failureReport: aggregatedFailures };
-    }
-    
-    static async flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, mode, skipStatusWrite, isInitialWrite = false) {
-        const { logger } = deps;
-        const transformedState = {};
-        for (const [name, inst] of Object.entries(state)) {
-            const rawResult = inst.results || {};
-            const firstUser = Object.keys(rawResult)[0];
-            let dataToCommit = rawResult;
-
-            if (firstUser && rawResult[firstUser] && typeof rawResult[firstUser] === 'object') {
-                const innerKeys = Object.keys(rawResult[firstUser]);
-                if (innerKeys.length > 0 && innerKeys.every(k => /^\d{4}-\d{2}-\d{2}$/.test(k))) {
-                    const transposed = {};
-                    for (const [userId, dateMap] of Object.entries(rawResult)) {
-                        for (const [dateKey, dailyData] of Object.entries(dateMap)) {
-                            if (!transposed[dateKey]) transposed[dateKey] = {};
-                            transposed[dateKey][userId] = dailyData;
-                        }
-                    }
-                    dataToCommit = transposed;
-                }
-            }
-
-            transformedState[name] = {
-                manifest: inst.manifest,
-                getResult: async () => dataToCommit,
-                _executionStats: executionStats[name]
-            };
-            inst.results = {};
-        }
-
-        const result = await commitResults(transformedState, dateStr, passName, config, deps, skipStatusWrite, {
-            flushMode: mode, shardIndexes: shardIndexMap, isInitialWrite: isInitialWrite 
-        });
-
-        if (result.shardIndexes) Object.assign(shardIndexMap, result.shardIndexes);
-        return result;
-    }
-
-    static mergeReports(successAcc, failureAcc, newResult) {
-        if (!newResult) return;
-        for (const [name, update] of Object.entries(newResult.successUpdates)) {
-            if (!successAcc[name]) {
-                successAcc[name] = update;
-            } else {
-                successAcc[name].metrics.storage.sizeBytes += (update.metrics.storage.sizeBytes || 0);
-                successAcc[name].metrics.storage.keys      += (update.metrics.storage.keys || 0);
-                successAcc[name].metrics.storage.shardCount = Math.max(successAcc[name].metrics.storage.shardCount, update.metrics.storage.shardCount || 1);
-                
-                if (update.metrics.io) {
-                    if (!successAcc[name].metrics.io) successAcc[name].metrics.io = { writes: 0, deletes: 0, reads: 0 };
-                    successAcc[name].metrics.io.writes += (update.metrics.io.writes || 0);
-                    successAcc[name].metrics.io.deletes += (update.metrics.io.deletes || 0);
-                }
-
-                if (update.metrics?.execution?.timings) {
-                     if (!successAcc[name].metrics.execution) successAcc[name].metrics.execution = { timings: { setup:0, stream:0, processing:0 }};
-                     const tDest = successAcc[name].metrics.execution.timings;
-                     const tSrc  = update.metrics.execution.timings;
-                     tDest.setup += (tSrc.setup || 0);
-                     tDest.stream += (tSrc.stream || 0);
-                     tDest.processing += (tSrc.processing || 0);
-                }
-                successAcc[name].hash = update.hash;
-            }
-        }
-        if (newResult.failureReport) failureAcc.push(...newResult.failureReport);
-    }
-    
-   static async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps, config, deps, loader, stats, earliestDates, seriesData = {}, mappings = null, piMasterList = null) {
-        const { logger } = deps;
-        const targetUserType = metadata.userType; 
-        
-        // [OPTIMIZATION] Use passed mappings/list if available, else load (fallback)
-        const mappingsToUse = mappings || await loader.loadMappings();
-        const piMasterListToUse = piMasterList || await loader.loadPIMasterList();
-        
-        const SCHEMAS = mathLayer.SCHEMAS;
-
-        // 1. Load Root Data (CachedLoader handles memoization for these)
-        const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
-        const verifications = metadata.rootDataDependencies?.includes('verification') ? await loader.loadVerifications() : null;
-        const rankings = metadata.rootDataDependencies?.includes('rankings') ? await loader.loadRankings(dateStr) : null;
-        
-        let yesterdayRankings = null;
-        if (metadata.rootDataDependencies?.includes('rankings') && metadata.isHistorical) {
-             const prevDate = new Date(dateStr); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-             const prevStr = prevDate.toISOString().slice(0, 10);
-             yesterdayRankings = await loader.loadRankings(prevStr); 
-        }
-
-        const socialContainer = metadata.rootDataDependencies?.includes('social') ? await loader.loadSocial(dateStr) : null;
-        
-        const allowMissing = metadata.canHaveMissingRoots === true;
-        
-        // Helper to safely load roots
-        const safeLoad = async (method, name) => {
-             if (!metadata.rootDataDependencies?.includes(name)) return null;
-             try {
-                 return await loader[method](dateStr);
-             } catch (e) {
-                 if (!allowMissing) throw new Error(`[StandardExecutor] Required root '${name}' failed: ${e.message}`);
-                 return null;
-             }
-        };
-
-        const ratings = await safeLoad('loadRatings', 'ratings');
-        const pageViews = await safeLoad('loadPageViews', 'pageViews');
-        const watchlistMembership = await safeLoad('loadWatchlistMembership', 'watchlist');
-        const alertHistory = await safeLoad('loadAlertHistory', 'alerts');
-
-        if (!allowMissing) {
-             if (metadata.rootDataDependencies?.includes('ratings') && !ratings) throw new Error("Missing ratings");
-             if (metadata.rootDataDependencies?.includes('pageViews') && !pageViews) throw new Error("Missing pageViews");
-             if (metadata.rootDataDependencies?.includes('watchlist') && !watchlistMembership) throw new Error("Missing watchlist");
-             if (metadata.rootDataDependencies?.includes('alerts') && !alertHistory) throw new Error("Missing alerts");
-        }
-
-        let chunkSuccess = 0;
-        let chunkFailures = 0;
-
-        for (const [userId, todayPortfolio] of Object.entries(portfolioData)) {
-            // --- OPTIMIZATION: TARGET SPECIFIC USER ---
-            // If the request contains a targetCid, skip all other users immediately
-            if (metadata.targetCid && String(userId) !== String(metadata.targetCid)) {
-                if (stats) stats.skippedUsers++;
-                continue;
-            }
-            // ------------------------------------------
-            
-            const yesterdayPortfolio = yesterdayPortfolioData ? yesterdayPortfolioData[userId] : null;
-            const todayHistory = historyData ? historyData[userId] : null;
-            
-            let actualUserType = todayPortfolio._userType;
-            if (!actualUserType) {
-                if (todayPortfolio.PublicPositions) {
-                    const isRanked = rankings && rankings.some(r => String(r.CustomerId) === String(userId));
-                    actualUserType = isRanked ? 'POPULAR_INVESTOR' : SCHEMAS.USER_TYPES.SPECULATOR;
-                } else {
-                    actualUserType = SCHEMAS.USER_TYPES.NORMAL;
-                }
-            }
-            
-            if (targetUserType && targetUserType !== 'all') {
-                 if (targetUserType !== actualUserType) {
-                     if (stats) stats.skippedUsers++;
-                     continue;
-                 }
-            }
-            
-            const userVerification = verifications ? verifications[userId] : null;
-            
-            const userRanking = rankings ? (rankings.find(r => String(r.CustomerId) === String(userId)) || null) : null;
-            const userRankingYesterday = yesterdayRankings ? (yesterdayRankings.find(r => String(r.CustomerId) === String(userId)) || null) : null;
-
-            let effectiveSocialData = null;
-            if (socialContainer) {
-                if (actualUserType === 'POPULAR_INVESTOR') {
-                    effectiveSocialData = socialContainer.pi[userId] || {}; 
-                } else if (actualUserType === 'SIGNED_IN_USER') {
-                    effectiveSocialData = socialContainer.signedIn[userId] || {};
-                } else {
-                    effectiveSocialData = socialContainer.generic || {};
-                }
-            }
-
-            const context = ContextFactory.buildPerUserContext({ 
-                todayPortfolio, yesterdayPortfolio, todayHistory, userId, 
-                userType: actualUserType, dateStr, metadata, 
-                mappings: mappingsToUse, 
-                insights, 
-                socialData: effectiveSocialData ? { today: effectiveSocialData } : null,
-                computedDependencies: computedDeps, previousComputedDependencies: prevDeps, 
-                config, deps,
-                verification: userVerification, 
-                
-                rankings: userRanking,
-                yesterdayRankings: userRankingYesterday, 
-                
-                allRankings: rankings,
-                allRankingsYesterday: yesterdayRankings,
-                
-                allVerifications: verifications,
-                
-                ratings: ratings || {},
-                pageViews: pageViews || {},
-                watchlistMembership: watchlistMembership || {},
-                alertHistory: alertHistory || {},
-
-                piMasterList: piMasterListToUse,
-                // [NEW] Pass Series Data
-                seriesData
-            });
-
-            if (metadata.requiresEarliestDataDate && earliestDates) {
-                if (!context.system) context.system = {};
-                context.system.earliestHistoryDate = earliestDates.history;
-            }
-            
-            try { 
-                await calcInstance.process(context); 
-                if (stats) stats.processedUsers++;
-                chunkSuccess++;
-            } catch (e) { 
-                logger.log('WARN', `Calc ${metadata.name} failed for user ${userId}: ${e.message}`); 
-                chunkFailures++;
-            }
-        }
-        return { success: chunkSuccess, failures: chunkFailures };
-    }
-}
-
-module.exports = { StandardExecutor };