bulltrackers-module 1.0.731 → 1.0.733

package/functions/computation-system/executors/StandardExecutor.js

@@ -1,465 +0,0 @@
-/**
- * @fileoverview Executor for "Standard" (User-Level) calculations.
- * REFACTORED: Hoisted data loading, centralized Series/Root logic.
- * UPDATED: Calls getResult() on computations to trigger summary logging.
- * FIXED: Removed redundant shard index increment that caused shard gaps (0, 2, 4...).
- */
-const { normalizeName, getEarliestDataDates } = require('../utils/utils');
-const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs, getHistoryPartRefs } = require('../utils/data_loader');
-const { CachedDataLoader } = require('../data/CachedDataLoader');
-const { ContextFactory } = require('../context/ContextFactory');
-const { commitResults } = require('../persistence/ResultCommitter');
-const { fetchResultSeries } = require('../data/DependencyFetcher');
-const { getManifest } = require('../topology/ManifestLoader');
-const mathLayer = require('../layers/index');
-const { performance } = require('perf_hooks');
-const v8 = require('v8');
-
-// Helper to get calculations - prefer config.calculations, fallback to direct require
-function getCalculations(config) {
-    if (config && config.calculations) return config.calculations;
-    try { return require('aiden-shared-calculations-unified').calculations; } 
-    catch (e) { return {}; }
-}
-
-class StandardExecutor {
-
-    // =========================================================================
-    // ENTRY POINT
-    // =========================================================================
-    static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
-        const dStr = date.toISOString().slice(0, 10);
-        const { logger } = deps;
-
-        // 1. Setup Scope (User Types & Target CID)
-        const requiredUserTypes = new Set(calcs.map(c => (c.userType || 'ALL').toUpperCase()));
-        const userTypeArray = requiredUserTypes.has('ALL') ? null : Array.from(requiredUserTypes);
-        
-        const targetCid = calcs.find(c => c.targetCid)?.targetCid || calcs.find(c => c.manifest?.targetCid)?.manifest?.targetCid;
-        if (targetCid) logger.log('INFO', `[StandardExecutor] 🎯 Targeted Mode: CID ${targetCid}`);
-
-        // 2. Prepare History Context (Refs)
-        const fullRoot = { ...rootData };
-        if (calcs.some(c => c.isHistorical)) {
-            const prevStr = new Date(date.getTime() - 86400000).toISOString().slice(0, 10);
-            let yRefs = await getPortfolioPartRefs(config, deps, prevStr, userTypeArray);
-            
-            if (targetCid && yRefs) {
-                yRefs = yRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
-            }
-            fullRoot.yesterdayPortfolioRefs = yRefs;
-        }
-
-        // 3. Initialize Instances
-        const state = {};
-        for (const c of calcs) {
-            try {
-                const inst = new c.class();
-                inst.manifest = c;
-                inst.results = {};
-                state[normalizeName(c.name)] = inst;
-            } catch (e) { logger.log('WARN', `Failed to init ${c.name}`); }
-        }
-
-        return await StandardExecutor.streamAndProcess(
-            dStr, state, passName, config, deps, fullRoot,
-            rootData.portfolioRefs, rootData.historyRefs,
-            fetchedDeps, previousFetchedDeps, skipStatusWrite,
-            userTypeArray, targetCid
-        );
-    }
-
-    // =========================================================================
-    // STREAMING LOOP
-    // =========================================================================
-    static async streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite, requiredUserTypes = null, targetCid = null) {
-        const { logger } = deps;
-        const calcs = Object.values(state).filter(c => c?.manifest);
-        const streamingCalcs = calcs.filter(c => c.manifest.rootDataDependencies.includes('portfolio') || c.manifest.rootDataDependencies.includes('history'));
-        
-        if (!streamingCalcs.length) return { successUpdates: {}, failureReport: [] };
-
-        const loader = new CachedDataLoader(config, deps);
-        
-        // --- 1. PRE-LOAD GLOBAL DATA (Hoisted) ---
-        const startSetup = performance.now();
-        
-        const [
-            globalRoots, 
-            seriesData, 
-            earliestDates
-        ] = await Promise.all([
-            loadGlobalRoots(loader, dateStr, streamingCalcs, deps),
-            loadSeriesData(loader, dateStr, streamingCalcs, config, deps),
-            streamingCalcs.some(c => c.manifest.requiresEarliestDataDate) ? getEarliestDataDates(config, deps) : null
-        ]);
-
-        const setupDuration = performance.now() - startSetup;
-
-        // --- 2. PREPARE REFS & STATS ---
-        let effPortRefs = portfolioRefs || await getPortfolioPartRefs(config, deps, dateStr, requiredUserTypes);
-        let effHistRefs = historyRefs;
-        
-        if (targetCid) effPortRefs = effPortRefs?.filter(r => !r.cid || String(r.cid) === String(targetCid));
-
-        const needsHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-        if (needsHistory) {
-            effHistRefs = effHistRefs || await getHistoryPartRefs(config, deps, dateStr, requiredUserTypes);
-            if (targetCid) effHistRefs = effHistRefs?.filter(r => !r.cid || String(r.cid) === String(targetCid));
-        }
-
-        const stats = {}; 
-        const shardMap = {};
-        calcs.forEach(c => { 
-            stats[normalizeName(c.manifest.name)] = { 
-                processedUsers: 0, skippedUsers: 0, timings: { setup: setupDuration, stream: 0, processing: 0 } 
-            };
-            shardMap[normalizeName(c.manifest.name)] = 0;
-        });
-
-        // --- 3. STREAM EXECUTION ---
-        const tP_iter = streamPortfolioData(config, deps, dateStr, effPortRefs, requiredUserTypes);
-        const yP_iter = (streamingCalcs.some(c => c.manifest.isHistorical) && rootData.yesterdayPortfolioRefs) 
-                        ? streamPortfolioData(config, deps, new Date(new Date(dateStr).getTime() - 86400000).toISOString().slice(0, 10), rootData.yesterdayPortfolioRefs) 
-                        : null;
-        const tH_iter = needsHistory ? streamHistoryData(config, deps, dateStr, effHistRefs, requiredUserTypes) : null;
-
-        let yP_chunk = {}, tH_chunk = {};
-        let usersSinceFlush = 0;
-        let hasFlushed = false;
-        const aggregatedSuccess = {}; 
-        const aggregatedFailures = [];
-
-        try {
-            for await (const tP_chunk of tP_iter) {
-                const t0 = performance.now();
-                if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
-                if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
-                const tStream = performance.now() - t0;
-
-                const t1 = performance.now();
-                
-                // Parallelize Calc Execution for this Chunk
-                await Promise.all(streamingCalcs.map(calc => 
-                    StandardExecutor.executePerUser(
-                        calc, calc.manifest, dateStr, tP_chunk, yP_chunk, tH_chunk,
-                        fetchedDeps, previousFetchedDeps, config, deps,
-                        stats[normalizeName(calc.manifest.name)],
-                        earliestDates,
-                        seriesData,
-                        globalRoots // <--- PASSED DOWN
-                    )
-                ));
-                
-                const tProc = performance.now() - t1;
-                streamingCalcs.forEach(c => {
-                    const s = stats[normalizeName(c.manifest.name)];
-                    s.timings.stream += tStream;
-                    s.timings.processing += tProc;
-                });
-
-                usersSinceFlush += Object.keys(tP_chunk).length;
-                const heap = v8.getHeapStatistics();
-                if (usersSinceFlush >= 500 || (heap.used_heap_size / heap.heap_size_limit) > 0.7) {
-                    const res = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardMap, stats, 'INTERMEDIATE', true, !hasFlushed);
-                    StandardExecutor.mergeReports(aggregatedSuccess, aggregatedFailures, res);
-                    hasFlushed = true;
-                    usersSinceFlush = 0;
-                }
-            }
-        } finally {
-            if (yP_iter?.return) await yP_iter.return();
-            if (tH_iter?.return) await tH_iter.return();
-        }
-
-        const finalRes = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardMap, stats, 'FINAL', skipStatusWrite, !hasFlushed);
-        StandardExecutor.mergeReports(aggregatedSuccess, aggregatedFailures, finalRes);
-
-        return { successUpdates: aggregatedSuccess, failureReport: aggregatedFailures };
-    }
-
-    // =========================================================================
-    // PER-USER EXECUTION (Pure Logic)
-    // =========================================================================
-    static async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps, config, deps, stats, earliestDates, seriesData, globalRoots) {
-        const { logger } = deps;
-        const targetUserType = metadata.userType;
-        const SCHEMAS = mathLayer.SCHEMAS;
-        
-        let success = 0, failures = 0;
-
-        for (const [userId, todayPortfolio] of Object.entries(portfolioData)) {
-            // 1. Filter User
-            if (metadata.targetCid && String(userId) !== String(metadata.targetCid)) { stats.skippedUsers++; continue; }
-            
-            // 2. Determine Type
-            let actualType = todayPortfolio._userType;
-            if (!actualType) {
-                const isRanked = globalRoots.rankings && globalRoots.rankings.some(r => String(r.CustomerId) === String(userId));
-                actualType = isRanked ? 'POPULAR_INVESTOR' : (todayPortfolio.PublicPositions ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL);
-            }
-            if (targetUserType && targetUserType !== 'all' && targetUserType !== actualType) { stats.skippedUsers++; continue; }
-
-            // 3. Resolve User Specifics from Global Data
-            const userRank = globalRoots.rankings?.find(r => String(r.CustomerId) === String(userId)) || null;
-            const userRankYest = globalRoots.rankingsYesterday?.find(r => String(r.CustomerId) === String(userId)) || null;
-            const userVerify = globalRoots.verifications?.[userId] || null;
-
-            let social = null;
-            if (globalRoots.social) {
-                // Use optional chaining to handle missing sub-objects (pi, signedIn, generic)
-                social = (actualType === 'POPULAR_INVESTOR' ? globalRoots.social.pi?.[userId] : 
-                         (actualType === 'SIGNED_IN_USER' ? globalRoots.social.signedIn?.[userId] : globalRoots.social.generic)) || {};
-            }
-
-            // 4. Build Context
-            const context = ContextFactory.buildPerUserContext({
-                todayPortfolio, 
-                yesterdayPortfolio: yesterdayPortfolioData?.[userId] || null, 
-                todayHistory: historyData?.[userId] || null,
-                userId, userType: actualType, dateStr, metadata,
-                
-                // Injected Global Data
-                mappings: globalRoots.mappings,
-                piMasterList: globalRoots.piMasterList,
-                insights: globalRoots.insights,
-                socialData: social ? { today: social } : null,
-                
-                // Dependency Data
-                computedDependencies: computedDeps, 
-                previousComputedDependencies: prevDeps,
-                config, deps,
-                
-                // Specific Lookups
-                verification: userVerify,
-                rankings: userRank,
-                yesterdayRankings: userRankYest,
-                
-                // Full Access (if needed by calc)
-                allRankings: globalRoots.rankings,
-                allRankingsYesterday: globalRoots.rankingsYesterday,
-                allVerifications: globalRoots.verifications,
-                ratings: globalRoots.ratings || {},
-                pageViews: globalRoots.pageViews || {},
-                watchlistMembership: globalRoots.watchlistMembership || {},
-                alertHistory: globalRoots.alertHistory || {},
-                
-                seriesData
-            });
-
-            if (metadata.requiresEarliestDataDate && earliestDates) {
-                if (!context.system) context.system = {};
-                context.system.earliestHistoryDate = earliestDates.history;
-            }
-
-            try {
-                await calcInstance.process(context);
-                stats.processedUsers++;
-                success++;
-            } catch (e) {
-                logger.log('WARN', `Calc ${metadata.name} failed for user ${userId}: ${e.message}`);
-                failures++;
-            }
-        }
-        return { success, failures };
-    }
-
-    // =========================================================================
-    // HELPERS (Flush & Merge)
-    // =========================================================================
-    static async flushBuffer(state, dateStr, passName, config, deps, shardMap, stats, mode, skipStatus, isInitial) {
-        const transformedState = {};
-        const activeCalcs = [];
-
-        for (const [name, inst] of Object.entries(state)) {
-            // [FIX 1] Allow computations to finalize/log summary stats via getResult()
-            let data;
-            if (typeof inst.getResult === 'function') {
-                data = await inst.getResult(); 
-            } else {
-                data = inst.results || {};
-            }
-
-            // Track active calculations (for debugging/logging only now)
-            if (Object.keys(data).length > 0) {
-                activeCalcs.push(name);
-            }
-
-            // Pivot user-date structure if needed
-            const first = Object.keys(data)[0];
-            if (first && data[first] && typeof data[first] === 'object' && /^\d{4}-\d{2}-\d{2}$/.test(Object.keys(data[first])[0])) {
-                const pivoted = {};
-                for (const [uid, dMap] of Object.entries(data)) {
-                    for (const [dKey, val] of Object.entries(dMap)) {
-                        if (!pivoted[dKey]) pivoted[dKey] = {};
-                        pivoted[dKey][uid] = val;
-                    }
-                }
-                data = pivoted;
-            }
-            transformedState[name] = { manifest: inst.manifest, getResult: async () => data, _executionStats: stats[name] };
-            inst.results = {}; // Clear buffer
-        }
-
-        const res = await commitResults(transformedState, dateStr, passName, config, deps, skipStatus, { flushMode: mode, shardIndexes: shardMap, isInitialWrite: isInitial });
-        
-        // Update shardMap from result. 
-        // ResultCommitter now returns the CORRECT nextShardIndex (e.g. if it wrote shard_0, it returns 1).
-        if (res.shardIndexes) Object.assign(shardMap, res.shardIndexes);
-        
-        return res;
-    }
-
-    static mergeReports(success, failure, result) {
-        if (!result) return;
-        if (result.failureReport) failure.push(...result.failureReport);
-        for (const [name, update] of Object.entries(result.successUpdates)) {
-            if (!success[name]) success[name] = update;
-            else {
-                const m = success[name].metrics;
-                const u = update.metrics;
-                m.storage.sizeBytes += u.storage.sizeBytes;
-                m.storage.keys += u.storage.keys;
-                m.storage.shardCount = Math.max(m.storage.shardCount, u.storage.shardCount);
-                if (u.execution) {
-                    m.execution.timings.stream += u.execution.timings.stream;
-                    m.execution.timings.processing += u.execution.timings.processing;
-                }
-            }
-        }
-    }
-}
-
-// =============================================================================
-// SHARED LOADING HELPERS
-// =============================================================================
-
-/**
- * Pre-loads all shared global datasets required by the active calculations.
- * Returns a consolidated object of { ratings, rankings, insights, ... }
- */
-async function loadGlobalRoots(loader, dateStr, calcs, deps) {
-    const { logger } = deps;
-    const roots = {};
-    
-    // 1. Identify Requirements
-    const reqs = {
-        mappings: true,
-        piMasterList: true,
-        rankings: false,
-        rankingsYesterday: false,
-        verifications: false,
-        insights: false,
-        social: false,
-        ratings: false,
-        pageViews: false,
-        watchlist: false,
-        alerts: false
-    };
-
-    for (const c of calcs) {
-        const deps = c.manifest.rootDataDependencies || [];
-        if (deps.includes('rankings')) reqs.rankings = true;
-        if (deps.includes('rankings') && c.manifest.isHistorical) reqs.rankingsYesterday = true;
-        if (deps.includes('verification')) reqs.verifications = true;
-        if (deps.includes('insights')) reqs.insights = true;
-        if (deps.includes('social')) reqs.social = true;
-        if (deps.includes('ratings')) reqs.ratings = true;
-        if (deps.includes('pageViews')) reqs.pageViews = true;
-        if (deps.includes('watchlist')) reqs.watchlist = true;
-        if (deps.includes('alerts')) reqs.alerts = true;
-    }
-
-    // 2. Fetch Helper
-    const fetch = async (key, method, dateArg, optional = true) => {
-        if (!reqs[key]) return;
-        try {
-            roots[key] = await loader[method](dateArg);
-        } catch (e) {
-            if (!optional) throw e;
-            logger.log('WARN', `[StandardExecutor] Optional root '${key}' failed to load: ${e.message}`);
-            roots[key] = null;
-        }
-    };
-
-    // 3. Execute Loads
-    await Promise.all([
-        fetch('mappings', 'loadMappings', null, false), // Always required
-        fetch('piMasterList', 'loadPIMasterList', null, false),
-        fetch('rankings', 'loadRankings', dateStr),
-        fetch('verifications', 'loadVerifications', dateStr),
-        fetch('insights', 'loadInsights', dateStr),
-        fetch('social', 'loadSocial', dateStr),
-        fetch('ratings', 'loadRatings', dateStr),
-        fetch('pageViews', 'loadPageViews', dateStr),
-        fetch('watchlist', 'loadWatchlistMembership', dateStr),
-        fetch('alerts', 'loadAlertHistory', dateStr)
-    ]);
-
-    if (reqs.rankingsYesterday) {
-        const prev = new Date(new Date(dateStr).getTime() - 86400000).toISOString().slice(0, 10);
-        await fetch('rankingsYesterday', 'loadRankings', prev);
-    }
-    
-    // Map internal names to match loadGlobalRoots structure if needed
-    roots.watchlistMembership = roots.watchlist;
-    roots.alertHistory = roots.alerts;
-    
-    return roots;
-}
-
-async function loadSeriesData(loader, dateStr, calcs, config, deps) {
-    const rootReqs = {};
-    const depReqs = {};  // norm -> { days, originalName, category }
-
-    // [FIX] Calculate User Types here to pass to loadSeries
-    const requiredUserTypes = new Set(calcs.map(c => (c.userType || 'ALL').toUpperCase()));
-    const userTypeArray = requiredUserTypes.has('ALL') ? null : Array.from(requiredUserTypes);
-
-    calcs.forEach(c => {
-        if (c.manifest.rootDataSeries) {
-            Object.entries(c.manifest.rootDataSeries).forEach(([k, v]) => rootReqs[k] = Math.max(rootReqs[k]||0, v.lookback||v));
-        }
-        if (c.manifest.dependencySeries) {
-            Object.entries(c.manifest.dependencySeries).forEach(([k, v]) => {
-                const norm = normalizeName(k);
-                const days = typeof v === 'object' ? v.lookback : v;
-                if (!depReqs[norm] || depReqs[norm].days < days) {
-                    depReqs[norm] = { days, originalName: k };
-                }
-            });
-        }
-    });
-
-    const series = { root: {}, results: {} };
-    const rootMap = {
-        alerts: 'loadAlertHistory', insights: 'loadInsights', ratings: 'loadRatings',
-        watchlist: 'loadWatchlistMembership', rankings: 'loadRankings',
-        // <--- ADDED MAPPING
-        portfolios: 'loadPortfolios' 
-    };
-
-    await Promise.all(Object.entries(rootReqs).map(async ([key, days]) => {
-        if (rootMap[key]) {
-            // [FIX] Pass userTypeArray specifically if loading portfolios
-            const extraArgs = (key === 'portfolios') ? [userTypeArray] : [];
-            series.root[key] = (await loader.loadSeries(rootMap[key], dateStr, days, ...extraArgs)).data;
-        }
-    }));
-
-    // Build lookup from ALL computations using config.calculations
-    const depEntries = Object.values(depReqs);
-    if (depEntries.length) {
-        const depOriginalNames = depEntries.map(e => e.originalName);
-        const maxDays = Math.max(...depEntries.map(e => e.days));
-        
-        const allManifests = getManifest(config.activeProductLines || [], getCalculations(config), deps);
-        const lookup = Object.fromEntries(allManifests.map(m => [normalizeName(m.name), m.category]));
-        
-        series.results = await fetchResultSeries(dateStr, depOriginalNames, lookup, config, deps, maxDays);
-    }
-
-    return series;
-}
-
-module.exports = { StandardExecutor };