bulltrackers-module 1.0.658 → 1.0.660

package/functions/computation-system/legacy/MetaExecutorold.js

@@ -0,0 +1,364 @@
+/**
+ * @fileoverview Executor for "Meta" (global) calculations.
+ * UPDATED: Uses CachedDataLoader for all data access.
+ * UPDATED: Tracks processed shard/item counts.
+ * UPDATED: Sends 'isInitialWrite: true' for robust cleanup.
+ * UPDATED: Support for historical rankings in Meta Context.
+ * UPDATED: Added support for loading Series Data (Root & Results) for lookbacks.
+ * UPDATED: Builds Manifest Lookup for DependencyFetcher.
+ */
+const { normalizeName }     = require('../utils/utils');
+const { CachedDataLoader }  = require('../data/CachedDataLoader');
+const { ContextFactory }    = require('../context/ContextFactory');
+const { commitResults }     = require('../persistence/ResultCommitter');
+const { fetchResultSeries, fetchDependencies } = require('../data/DependencyFetcher');
+const { getManifest }    = require('../topology/ManifestLoader');
+
+class MetaExecutor {
+    static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps) {
+        const dStr = date.toISOString().slice(0, 10);
+        const { logger, db } = deps;
+        const loader = new CachedDataLoader(config, deps);
+
+        // --- [FIXED] Build Global Manifest Lookup using getManifest ---
+        // getManifest is typically cached, so this is cheap.
+        const allManifests = getManifest(config.productLines, config.calculationsDirectory, deps); 
+        const manifestLookup = {};
+        allManifests.forEach(m => { manifestLookup[normalizeName(m.name)] = m.category; });
+
+        // [FIX] Check if any meta calculation needs history
+        const needsHistory = calcs.some(c => c.isHistorical);
+        let rankingsYesterday = null;
+
+        if (needsHistory) {
+             const prevDate = new Date(date);
+             prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+             const prevStr = prevDate.toISOString().slice(0, 10);
+             logger.log('INFO', `[MetaExecutor] Loading historical rankings for ${prevStr}`);
+             rankingsYesterday = await loader.loadRankings(prevStr);
+        }
+
+        // 1. Load Global Dependencies
+        const [mappings, rankings, verifications] = await Promise.all([
+            loader.loadMappings(),
+            loader.loadRankings(dStr),
+            loader.loadVerifications()
+        ]);
+        
+        // [NEW] Load New Root Data Types for Profile Metrics (if any calc needs them)
+        const needsNewRootData = calcs.some(c => {
+            const deps = c.rootDataDependencies || [];
+            return deps.includes('ratings') || deps.includes('pageViews') || 
+                   deps.includes('watchlist') || deps.includes('alerts');
+        });
+        
+        let ratings = null, pageViews = null, watchlistMembership = null, alertHistory = null;
+        if (needsNewRootData) {
+            const loadPromises = [];
+            
+            // Ratings
+            if (calcs.some(c => c.rootDataDependencies?.includes('ratings'))) {
+                loadPromises.push(loader.loadRatings(dStr).then(r => { ratings = r; }).catch(e => {
+                    const allAllowMissing = calcs.every(c => {
+                        const needsRatings = c.rootDataDependencies?.includes('ratings');
+                        return !needsRatings || c.canHaveMissingRoots === true;
+                    });
+                    if (allAllowMissing) { ratings = null; } else { throw e; }
+                }));
+            }
+            
+            // PageViews
+            if (calcs.some(c => c.rootDataDependencies?.includes('pageViews'))) {
+                loadPromises.push(loader.loadPageViews(dStr).then(pv => { pageViews = pv; }).catch(e => {
+                    const allAllowMissing = calcs.every(c => {
+                        const needsPageViews = c.rootDataDependencies?.includes('pageViews');
+                        return !needsPageViews || c.canHaveMissingRoots === true;
+                    });
+                    if (allAllowMissing) { pageViews = null; } else { throw e; }
+                }));
+            }
+            
+            // Watchlist
+            if (calcs.some(c => c.rootDataDependencies?.includes('watchlist'))) {
+                loadPromises.push(loader.loadWatchlistMembership(dStr).then(w => { watchlistMembership = w; }).catch(e => {
+                    const allAllowMissing = calcs.every(c => {
+                        const needsWatchlist = c.rootDataDependencies?.includes('watchlist');
+                        return !needsWatchlist || c.canHaveMissingRoots === true;
+                    });
+                    if (allAllowMissing) { watchlistMembership = null; } else { throw e; }
+                }));
+            }
+            
+            // Alerts
+            if (calcs.some(c => c.rootDataDependencies?.includes('alerts'))) {
+                loadPromises.push(loader.loadAlertHistory(dStr).then(a => { alertHistory = a; }).catch(e => {
+                    const allAllowMissing = calcs.every(c => {
+                        const needsAlerts = c.rootDataDependencies?.includes('alerts');
+                        return !needsAlerts || c.canHaveMissingRoots === true;
+                    });
+                    if (allAllowMissing) { alertHistory = null; } else { throw e; }
+                }));
+            }
+            
+            await Promise.all(loadPromises);
+            
+            // [FIX] Enforce canHaveMissingRoots - validate after loading
+            for (const c of calcs) {
+                const deps = c.rootDataDependencies || [];
+                const canSkip = c.canHaveMissingRoots === true;
+                const isMissing = (key, val) => deps.includes(key) && (val === null || val === undefined);
+                
+                if (!canSkip) {
+                    if (isMissing('ratings', ratings)) throw new Error(`[MetaExecutor] Missing required root 'ratings' for ${c.name}`);
+                    if (isMissing('pageViews', pageViews)) throw new Error(`[MetaExecutor] Missing required root 'pageViews' for ${c.name}`);
+                    if (isMissing('watchlist', watchlistMembership)) throw new Error(`[MetaExecutor] Missing required root 'watchlist' for ${c.name}`);
+                    if (isMissing('alerts', alertHistory)) throw new Error(`[MetaExecutor] Missing required root 'alerts' for ${c.name}`);
+                }
+            }
+        }
+
+        // --- [NEW] Series / Lookback Loading Logic ---
+        const rootSeriesRequests = {};
+        const dependencySeriesRequests = {};
+
+        // 1. Identify requirements from manifests
+        calcs.forEach(c => {
+            if (c.rootDataSeries) {
+                Object.entries(c.rootDataSeries).forEach(([type, conf]) => {
+                    const days = typeof conf === 'object' ? conf.lookback : conf;
+                    if (!rootSeriesRequests[type] || days > rootSeriesRequests[type]) {
+                        rootSeriesRequests[type] = days;
+                    }
+                });
+            }
+            if (c.dependencySeries) {
+                Object.entries(c.dependencySeries).forEach(([depName, conf]) => {
+                    const days = typeof conf === 'object' ? conf.lookback : conf;
+                    const normalized = normalizeName(depName);
+                    if (!dependencySeriesRequests[normalized] || days > dependencySeriesRequests[normalized]) {
+                        dependencySeriesRequests[normalized] = days;
+                    }
+                });
+            }
+        });
+
+        const seriesData = { root: {}, results: {} };
+
+        // 2. Load Root Series
+        for (const [type, days] of Object.entries(rootSeriesRequests)) {
+            let loaderMethod = null;
+            if (type === 'alerts') loaderMethod = 'loadAlertHistory';
+            else if (type === 'insights') loaderMethod = 'loadInsights';
+            else if (type === 'ratings') loaderMethod = 'loadRatings';
+            else if (type === 'watchlist') loaderMethod = 'loadWatchlistMembership';
+            // [CRITICAL UPDATE] Add rankings support for Meta lookbacks
+            else if (type === 'rankings') loaderMethod = 'loadRankings';
+
+            if (loaderMethod) {
+                logger.log('INFO', `[MetaExecutor] Loading ${days}-day series for Root Data '${type}'...`);
+                const series = await loader.loadSeries(loaderMethod, dStr, days);
+                seriesData.root[type] = series.data;
+            }
+        }
+
+        // 3. Load Computation Result Series
+        const calcNamesToFetch = Object.keys(dependencySeriesRequests);
+        if (calcNamesToFetch.length > 0) {
+            const maxDays = Math.max(...Object.values(dependencySeriesRequests));
+            logger.log('INFO', `[MetaExecutor] Loading up to ${maxDays}-day series for Dependencies: ${calcNamesToFetch.join(', ')}`);
+            
+            // Pass manifestLookup to fetcher
+            const resultsSeries = await fetchResultSeries(dStr, calcNamesToFetch, manifestLookup, config, deps, maxDays);
+            seriesData.results = resultsSeries; 
+        }
+
+        const state = {};
+        for (const c of calcs) {
+            const inst = new c.class();
+            inst.manifest = c;
+            
+            // DEBUG: Log what dependencies were fetched
+            const fetchedDepKeys = Object.keys(fetchedDeps || {});
+            logger.log('INFO', `[MetaExecutor] 📦 Fetched dependencies available: ${fetchedDepKeys.length > 0 ? fetchedDepKeys.join(', ') : 'NONE'}`);
+            
+            const context = ContextFactory.buildMetaContext({
+                dateStr: dStr,
+                metadata: c,
+                mappings,
+                insights: { today: rootData.todayInsights },
+                socialData: { today: rootData.todaySocialPostInsights },
+                computedDependencies: fetchedDeps,
+                previousComputedDependencies: previousFetchedDeps,
+                config, deps,
+                allRankings: rankings,
+                allRankingsYesterday: rankingsYesterday,
+                allVerifications: verifications,
+                ratings: ratings || {},
+                pageViews: pageViews || {},
+                watchlistMembership: watchlistMembership || {},
+                alertHistory: alertHistory || {},
+                seriesData
+            });
+
+            // DEBUG: Log dependency availability
+            // CRITICAL: Use class.getDependencies() to get original case-sensitive names
+            const depNames = (c.class && typeof c.class.getDependencies === 'function') 
+                ? c.class.getDependencies() 
+                : (c.getDependencies ? c.getDependencies() : (c.dependencies || []));
+            depNames.forEach(depName => {
+                const depData = context.computed[depName];
+                if (depData) {
+                    const keys = Object.keys(depData);
+                    logger.log('INFO', `[MetaExecutor] ✅ Dependency '${depName}' available for ${c.name}. Keys: ${keys.length} (sample: ${keys.slice(0, 5).join(', ')})`);
+                } else {
+                    logger.log('ERROR', `[MetaExecutor] ❌ Dependency '${depName}' MISSING for ${c.name} in context.computed`);
+                }
+            });
+
+            try {
+                const result = await inst.process(context);
+                
+                // DEBUG: Log result before saving
+                if (result && typeof result === 'object') {
+                    const resultKeys = Object.keys(result);
+                    logger.log('INFO', `[MetaExecutor] ✅ ${c.name} computed result. Keys: ${resultKeys.length} (sample: ${resultKeys.slice(0, 10).join(', ')})`);
+                    if (resultKeys.length === 0) {
+                        logger.log('WARN', `[MetaExecutor] ⚠️ ${c.name} returned EMPTY result object!`);
+                    }
+                } else {
+                    logger.log('WARN', `[MetaExecutor] ⚠️ ${c.name} returned non-object result: ${typeof result} - ${result}`);
+                }
+                
+                // CRITICAL FIX: Do NOT overwrite this.results - the computation already sets it correctly
+                // The computation's process() method sets this.results, and getResult() returns it
+                // We only use the return value for logging/debugging
+                // inst.results should already be set by the computation's process() method
+                if (!inst.results) {
+                    logger.log('WARN', `[MetaExecutor] ⚠️ ${c.name} did not set this.results - using return value as fallback`);
+                    inst.results = result;
+                }
+                
+                // DEBUG: Verify what getResult() will return
+                const finalResult = await inst.getResult();
+                if (finalResult && typeof finalResult === 'object') {
+                    const finalKeys = Object.keys(finalResult);
+                    logger.log('INFO', `[MetaExecutor] ✅ ${c.name} getResult() will return ${finalKeys.length} keys`);
+                } else {
+                    logger.log('WARN', `[MetaExecutor] ⚠️ ${c.name} getResult() returns: ${typeof finalResult}`);
+                }
+                
+                state[c.name] = inst;
+            } catch (e) {
+                logger.log('ERROR', `Meta calc ${c.name} failed: ${e.message}`);
+            }
+        }
+
+        // CRITICAL FIX: Pass 'isInitialWrite: true' to ensure proper cleanup of old meta data
+        return await commitResults(state, dStr, passName, config, deps, false, { isInitialWrite: true });
+    }
+
+    static async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps, config, deps, loader) {
+        const mappings = await loader.loadMappings();
+        const { logger } = deps;
+        const stats = { processedShards: 0, processedItems: 0 };
+        
+        const insights   = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
+        const social     = metadata.rootDataDependencies?.includes('social')   ? { today: await loader.loadSocial(dateStr)   } : null;
+        
+        let rankingsYesterday = null;
+        if (metadata.isHistorical) {
+             const prevDate = new Date(dateStr);
+             prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+             const prevStr = prevDate.toISOString().slice(0, 10);
+             rankingsYesterday = await loader.loadRankings(prevStr);
+        }
+
+        const rankings = await loader.loadRankings(dateStr);
+        const allowMissing = metadata.canHaveMissingRoots === true;
+        
+        let ratings = null;
+        if (metadata.rootDataDependencies?.includes('ratings')) {
+            try { ratings = await loader.loadRatings(dateStr); } 
+            catch (e) { if (!allowMissing) throw e; ratings = null; }
+            if (!ratings && !allowMissing) throw new Error(`Missing ratings`);
+        }
+        
+        let pageViews = null;
+        if (metadata.rootDataDependencies?.includes('pageViews')) {
+            try { pageViews = await loader.loadPageViews(dateStr); } 
+            catch (e) { if (!allowMissing) throw e; pageViews = null; }
+            if (!pageViews && !allowMissing) throw new Error(`Missing pageViews`);
+        }
+        
+        let watchlistMembership = null;
+        if (metadata.rootDataDependencies?.includes('watchlist')) {
+            try { watchlistMembership = await loader.loadWatchlistMembership(dateStr); } 
+            catch (e) { if (!allowMissing) throw e; watchlistMembership = null; }
+            if (!watchlistMembership && !allowMissing) throw new Error(`Missing watchlist`);
+        }
+        
+        let alertHistory = null;
+        if (metadata.rootDataDependencies?.includes('alerts')) {
+            try { alertHistory = await loader.loadAlertHistory(dateStr); } 
+            catch (e) { if (!allowMissing) throw e; alertHistory = null; }
+            if (!alertHistory && !allowMissing) throw new Error(`Missing alerts`);
+        }
+
+        const seriesData = { root: {}, results: {} }; 
+
+        if (metadata.rootDataDependencies?.includes('price')) {
+            logger.log('INFO', `[Executor] Running Batched/Sharded Execution for ${metadata.name}`);
+            const shardRefs = await loader.getPriceShardReferences();
+            if (shardRefs.length === 0) { logger.log('WARN', '[Executor] No price shards found.'); return {}; }
+            
+            let processedCount = 0;
+            for (const ref of shardRefs) {
+                const shardData = await loader.loadPriceShard(ref);
+                const partialContext = ContextFactory.buildMetaContext({ 
+                    dateStr, metadata, mappings, insights, socialData: social, 
+                    prices: { history: shardData }, computedDependencies: computedDeps, 
+                    previousComputedDependencies: prevDeps, config, deps,
+                    allRankings: rankings,
+                    allRankingsYesterday: rankingsYesterday,
+                    ratings: ratings || {},
+                    pageViews: pageViews || {},
+                    watchlistMembership: watchlistMembership || {},
+                    alertHistory: alertHistory || {},
+                    seriesData
+                });
+                
+                await calcInstance.process(partialContext);
+                partialContext.prices = null; 
+                processedCount++;
+                
+                stats.processedShards++;
+                stats.processedItems += Object.keys(shardData).length;
+            }
+            logger.log('INFO', `[Executor] Finished Batched Execution for ${metadata.name} (${processedCount} shards).`);
+            
+            calcInstance._executionStats = stats;
+            return calcInstance.getResult ? await calcInstance.getResult() : {};
+        } else {
+            const context = ContextFactory.buildMetaContext({ 
+                dateStr, metadata, mappings, insights, socialData: social, 
+                prices: {}, computedDependencies: computedDeps, 
+                previousComputedDependencies: prevDeps, config, deps,
+                allRankings: rankings,
+                allRankingsYesterday: rankingsYesterday,
+                ratings: ratings || {},
+                pageViews: pageViews || {},
+                watchlistMembership: watchlistMembership || {},
+                alertHistory: alertHistory || {},
+                seriesData
+            });
+            const res = await calcInstance.process(context);
+            
+            stats.processedItems = 1; 
+            calcInstance._executionStats = stats;
+            
+            return res;
+        }
+    }
+}
+
+module.exports = { MetaExecutor };