bulltrackers-module 1.0.658 → 1.0.659

package/functions/computation-system/executors/MetaExecutor.js

@@ -1,186 +1,52 @@
 /**
  * @fileoverview Executor for "Meta" (global) calculations.
- * UPDATED: Uses CachedDataLoader for all data access.
- * UPDATED: Tracks processed shard/item counts.
- * UPDATED: Sends 'isInitialWrite: true' for robust cleanup.
- * UPDATED: Support for historical rankings in Meta Context.
- * UPDATED: Added support for loading Series Data (Root & Results) for lookbacks.
- * UPDATED: Builds Manifest Lookup for DependencyFetcher.
+ * REFACTORED: Applied DRY principles to Root Data and Series loading.
  */
 const { normalizeName }     = require('../utils/utils');
 const { CachedDataLoader }  = require('../data/CachedDataLoader');
 const { ContextFactory }    = require('../context/ContextFactory');
 const { commitResults }     = require('../persistence/ResultCommitter');
-const { fetchResultSeries, fetchDependencies } = require('../data/DependencyFetcher');
-const { getManifest }    = require('../topology/ManifestLoader');
+const { fetchResultSeries } = require('../data/DependencyFetcher');
+const { getManifest }       = require('../topology/ManifestLoader');
 
 class MetaExecutor {
+
+    // =========================================================================
+    // PRIMARY ENTRY POINT (Batch Execution)
+    // =========================================================================
     static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps) {
+        const { logger } = deps;
         const dStr = date.toISOString().slice(0, 10);
-        const { logger, db } = deps;
         const loader = new CachedDataLoader(config, deps);
 
-        // --- [FIXED] Build Global Manifest Lookup using getManifest ---
-        // getManifest is typically cached, so this is cheap.
+        // 1. Setup Manifest Lookup
         const allManifests = getManifest(config.productLines, config.calculationsDirectory, deps); 
-        const manifestLookup = {};
-        allManifests.forEach(m => { manifestLookup[normalizeName(m.name)] = m.category; });
-
-        // [FIX] Check if any meta calculation needs history
-        const needsHistory = calcs.some(c => c.isHistorical);
-        let rankingsYesterday = null;
-
-        if (needsHistory) {
-             const prevDate = new Date(date);
-             prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-             const prevStr = prevDate.toISOString().slice(0, 10);
-             logger.log('INFO', `[MetaExecutor] Loading historical rankings for ${prevStr}`);
-             rankingsYesterday = await loader.loadRankings(prevStr);
-        }
+        const manifestLookup = Object.fromEntries(allManifests.map(m => [normalizeName(m.name), m.category]));
 
-        // 1. Load Global Dependencies
+        // 2. Load Base Data (Always Required)
         const [mappings, rankings, verifications] = await Promise.all([
             loader.loadMappings(),
             loader.loadRankings(dStr),
             loader.loadVerifications()
         ]);
-        
-        // [NEW] Load New Root Data Types for Profile Metrics (if any calc needs them)
-        const needsNewRootData = calcs.some(c => {
-            const deps = c.rootDataDependencies || [];
-            return deps.includes('ratings') || deps.includes('pageViews') || 
-                   deps.includes('watchlist') || deps.includes('alerts');
-        });
-        
-        let ratings = null, pageViews = null, watchlistMembership = null, alertHistory = null;
-        if (needsNewRootData) {
-            const loadPromises = [];
-            
-            // Ratings
-            if (calcs.some(c => c.rootDataDependencies?.includes('ratings'))) {
-                loadPromises.push(loader.loadRatings(dStr).then(r => { ratings = r; }).catch(e => {
-                    const allAllowMissing = calcs.every(c => {
-                        const needsRatings = c.rootDataDependencies?.includes('ratings');
-                        return !needsRatings || c.canHaveMissingRoots === true;
-                    });
-                    if (allAllowMissing) { ratings = null; } else { throw e; }
-                }));
-            }
-            
-            // PageViews
-            if (calcs.some(c => c.rootDataDependencies?.includes('pageViews'))) {
-                loadPromises.push(loader.loadPageViews(dStr).then(pv => { pageViews = pv; }).catch(e => {
-                    const allAllowMissing = calcs.every(c => {
-                        const needsPageViews = c.rootDataDependencies?.includes('pageViews');
-                        return !needsPageViews || c.canHaveMissingRoots === true;
-                    });
-                    if (allAllowMissing) { pageViews = null; } else { throw e; }
-                }));
-            }
-            
-            // Watchlist
-            if (calcs.some(c => c.rootDataDependencies?.includes('watchlist'))) {
-                loadPromises.push(loader.loadWatchlistMembership(dStr).then(w => { watchlistMembership = w; }).catch(e => {
-                    const allAllowMissing = calcs.every(c => {
-                        const needsWatchlist = c.rootDataDependencies?.includes('watchlist');
-                        return !needsWatchlist || c.canHaveMissingRoots === true;
-                    });
-                    if (allAllowMissing) { watchlistMembership = null; } else { throw e; }
-                }));
-            }
-            
-            // Alerts
-            if (calcs.some(c => c.rootDataDependencies?.includes('alerts'))) {
-                loadPromises.push(loader.loadAlertHistory(dStr).then(a => { alertHistory = a; }).catch(e => {
-                    const allAllowMissing = calcs.every(c => {
-                        const needsAlerts = c.rootDataDependencies?.includes('alerts');
-                        return !needsAlerts || c.canHaveMissingRoots === true;
-                    });
-                    if (allAllowMissing) { alertHistory = null; } else { throw e; }
-                }));
-            }
-            
-            await Promise.all(loadPromises);
-            
-            // [FIX] Enforce canHaveMissingRoots - validate after loading
-            for (const c of calcs) {
-                const deps = c.rootDataDependencies || [];
-                const canSkip = c.canHaveMissingRoots === true;
-                const isMissing = (key, val) => deps.includes(key) && (val === null || val === undefined);
-                
-                if (!canSkip) {
-                    if (isMissing('ratings', ratings)) throw new Error(`[MetaExecutor] Missing required root 'ratings' for ${c.name}`);
-                    if (isMissing('pageViews', pageViews)) throw new Error(`[MetaExecutor] Missing required root 'pageViews' for ${c.name}`);
-                    if (isMissing('watchlist', watchlistMembership)) throw new Error(`[MetaExecutor] Missing required root 'watchlist' for ${c.name}`);
-                    if (isMissing('alerts', alertHistory)) throw new Error(`[MetaExecutor] Missing required root 'alerts' for ${c.name}`);
-                }
-            }
-        }
-
-        // --- [NEW] Series / Lookback Loading Logic ---
-        const rootSeriesRequests = {};
-        const dependencySeriesRequests = {};
 
-        // 1. Identify requirements from manifests
-        calcs.forEach(c => {
-            if (c.rootDataSeries) {
-                Object.entries(c.rootDataSeries).forEach(([type, conf]) => {
-                    const days = typeof conf === 'object' ? conf.lookback : conf;
-                    if (!rootSeriesRequests[type] || days > rootSeriesRequests[type]) {
-                        rootSeriesRequests[type] = days;
-                    }
-                });
-            }
-            if (c.dependencySeries) {
-                Object.entries(c.dependencySeries).forEach(([depName, conf]) => {
-                    const days = typeof conf === 'object' ? conf.lookback : conf;
-                    const normalized = normalizeName(depName);
-                    if (!dependencySeriesRequests[normalized] || days > dependencySeriesRequests[normalized]) {
-                        dependencySeriesRequests[normalized] = days;
-                    }
-                });
-            }
-        });
-
-        const seriesData = { root: {}, results: {} };
-
-        // 2. Load Root Series
-        for (const [type, days] of Object.entries(rootSeriesRequests)) {
-            let loaderMethod = null;
-            if (type === 'alerts') loaderMethod = 'loadAlertHistory';
-            else if (type === 'insights') loaderMethod = 'loadInsights';
-            else if (type === 'ratings') loaderMethod = 'loadRatings';
-            else if (type === 'watchlist') loaderMethod = 'loadWatchlistMembership';
-            // [CRITICAL UPDATE] Add rankings support for Meta lookbacks
-            else if (type === 'rankings') loaderMethod = 'loadRankings';
-
-            if (loaderMethod) {
-                logger.log('INFO', `[MetaExecutor] Loading ${days}-day series for Root Data '${type}'...`);
-                const series = await loader.loadSeries(loaderMethod, dStr, days);
-                seriesData.root[type] = series.data;
-            }
+        // 3. Load Historical Rankings (if needed)
+        let rankingsYesterday = null;
+        if (calcs.some(c => c.isHistorical)) {
+            const prevStr = new Date(date.getTime() - 86400000).toISOString().slice(0, 10);
+            rankingsYesterday = await loader.loadRankings(prevStr);
         }
 
-        // 3. Load Computation Result Series
-        const calcNamesToFetch = Object.keys(dependencySeriesRequests);
-        if (calcNamesToFetch.length > 0) {
-            const maxDays = Math.max(...Object.values(dependencySeriesRequests));
-            logger.log('INFO', `[MetaExecutor] Loading up to ${maxDays}-day series for Dependencies: ${calcNamesToFetch.join(', ')}`);
-            
-            // Pass manifestLookup to fetcher
-            const resultsSeries = await fetchResultSeries(dStr, calcNamesToFetch, manifestLookup, config, deps, maxDays);
-            seriesData.results = resultsSeries; 
-        }
+        // 4. Load Variable Root Data & Series (Refactored Helpers)
+        const variableRoots = await loadVariableRootData(loader, dStr, calcs, logger);
+        const seriesData = await loadSeriesData(loader, dStr, calcs, manifestLookup, config, deps);
 
+        // 5. Execution Loop
         const state = {};
         for (const c of calcs) {
             const inst = new c.class();
             inst.manifest = c;
             
-            // DEBUG: Log what dependencies were fetched
-            const fetchedDepKeys = Object.keys(fetchedDeps || {});
-            logger.log('INFO', `[MetaExecutor] 📦 Fetched dependencies available: ${fetchedDepKeys.length > 0 ? fetchedDepKeys.join(', ') : 'NONE'}`);
-            
             const context = ContextFactory.buildMetaContext({
                 dateStr: dStr,
                 metadata: c,
@@ -193,172 +59,202 @@ class MetaExecutor {
                 allRankings: rankings,
                 allRankingsYesterday: rankingsYesterday,
                 allVerifications: verifications,
-                ratings: ratings || {},
-                pageViews: pageViews || {},
-                watchlistMembership: watchlistMembership || {},
-                alertHistory: alertHistory || {},
+                // Spread variable roots directly (ratings, pageViews, etc.)
+                ...variableRoots,
                 seriesData
             });
 
-            // DEBUG: Log dependency availability
-            // CRITICAL: Use class.getDependencies() to get original case-sensitive names
-            const depNames = (c.class && typeof c.class.getDependencies === 'function') 
-                ? c.class.getDependencies() 
-                : (c.getDependencies ? c.getDependencies() : (c.dependencies || []));
-            depNames.forEach(depName => {
-                const depData = context.computed[depName];
-                if (depData) {
-                    const keys = Object.keys(depData);
-                    logger.log('INFO', `[MetaExecutor] ✅ Dependency '${depName}' available for ${c.name}. Keys: ${keys.length} (sample: ${keys.slice(0, 5).join(', ')})`);
-                } else {
-                    logger.log('ERROR', `[MetaExecutor] ❌ Dependency '${depName}' MISSING for ${c.name} in context.computed`);
-                }
-            });
-
             try {
                 const result = await inst.process(context);
                 
-                // DEBUG: Log result before saving
-                if (result && typeof result === 'object') {
-                    const resultKeys = Object.keys(result);
-                    logger.log('INFO', `[MetaExecutor] ✅ ${c.name} computed result. Keys: ${resultKeys.length} (sample: ${resultKeys.slice(0, 10).join(', ')})`);
-                    if (resultKeys.length === 0) {
-                        logger.log('WARN', `[MetaExecutor] ⚠️ ${c.name} returned EMPTY result object!`);
-                    }
-                } else {
-                    logger.log('WARN', `[MetaExecutor] ⚠️ ${c.name} returned non-object result: ${typeof result} - ${result}`);
-                }
-                
-                // CRITICAL FIX: Do NOT overwrite this.results - the computation already sets it correctly
-                // The computation's process() method sets this.results, and getResult() returns it
-                // We only use the return value for logging/debugging
-                // inst.results should already be set by the computation's process() method
-                if (!inst.results) {
-                    logger.log('WARN', `[MetaExecutor] ⚠️ ${c.name} did not set this.results - using return value as fallback`);
-                    inst.results = result;
-                }
+                // Fallback if the computation didn't set this.results internally
+                if (!inst.results) inst.results = result;
                 
-                // DEBUG: Verify what getResult() will return
-                const finalResult = await inst.getResult();
-                if (finalResult && typeof finalResult === 'object') {
-                    const finalKeys = Object.keys(finalResult);
-                    logger.log('INFO', `[MetaExecutor] ✅ ${c.name} getResult() will return ${finalKeys.length} keys`);
-                } else {
-                    logger.log('WARN', `[MetaExecutor] ⚠️ ${c.name} getResult() returns: ${typeof finalResult}`);
+                // Debug logging condensed
+                if (inst.results && Object.keys(inst.results).length === 0) {
+                    logger.log('WARN', `[MetaExecutor] ⚠️ ${c.name} produced EMPTY results.`);
                 }
                 
                 state[c.name] = inst;
             } catch (e) {
-                logger.log('ERROR', `Meta calc ${c.name} failed: ${e.message}`);
+                logger.log('ERROR', `[MetaExecutor] ❌ ${c.name} failed: ${e.message}`);
             }
         }
 
-        // CRITICAL FIX: Pass 'isInitialWrite: true' to ensure proper cleanup of old meta data
+        // Force 'isInitialWrite: true' for robust cleanup of old keys
         return await commitResults(state, dStr, passName, config, deps, false, { isInitialWrite: true });
     }
 
+    // =========================================================================
+    // SINGLE EXECUTION (Sharded/On-Demand)
+    // =========================================================================
     static async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps, config, deps, loader) {
-        const mappings = await loader.loadMappings();
         const { logger } = deps;
-        const stats = { processedShards: 0, processedItems: 0 };
-        
-        const insights   = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
-        const social     = metadata.rootDataDependencies?.includes('social')   ? { today: await loader.loadSocial(dateStr)   } : null;
-        
+        const calcs = [metadata]; // Treat single as list for helpers
+
+        // 1. Load Data using Shared Helpers
+        const [mappings, rankings, variableRoots, seriesData] = await Promise.all([
+            loader.loadMappings(),
+            loader.loadRankings(dateStr),
+            loadVariableRootData(loader, dateStr, calcs, logger),
+            loadSeriesData(loader, dateStr, calcs, {}, config, deps) // Empty lookup fine for single usage usually
+        ]);
+
         let rankingsYesterday = null;
         if (metadata.isHistorical) {
-             const prevDate = new Date(dateStr);
-             prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-             const prevStr = prevDate.toISOString().slice(0, 10);
-             rankingsYesterday = await loader.loadRankings(prevStr);
+            const prevStr = new Date(new Date(dateStr).getTime() - 86400000).toISOString().slice(0, 10);
+            rankingsYesterday = await loader.loadRankings(prevStr);
         }
 
-        const rankings = await loader.loadRankings(dateStr);
-        const allowMissing = metadata.canHaveMissingRoots === true;
-        
-        let ratings = null;
-        if (metadata.rootDataDependencies?.includes('ratings')) {
-            try { ratings = await loader.loadRatings(dateStr); } 
-            catch (e) { if (!allowMissing) throw e; ratings = null; }
-            if (!ratings && !allowMissing) throw new Error(`Missing ratings`);
-        }
-        
-        let pageViews = null;
-        if (metadata.rootDataDependencies?.includes('pageViews')) {
-            try { pageViews = await loader.loadPageViews(dateStr); } 
-            catch (e) { if (!allowMissing) throw e; pageViews = null; }
-            if (!pageViews && !allowMissing) throw new Error(`Missing pageViews`);
-        }
-        
-        let watchlistMembership = null;
-        if (metadata.rootDataDependencies?.includes('watchlist')) {
-            try { watchlistMembership = await loader.loadWatchlistMembership(dateStr); } 
-            catch (e) { if (!allowMissing) throw e; watchlistMembership = null; }
-            if (!watchlistMembership && !allowMissing) throw new Error(`Missing watchlist`);
-        }
-        
-        let alertHistory = null;
-        if (metadata.rootDataDependencies?.includes('alerts')) {
-            try { alertHistory = await loader.loadAlertHistory(dateStr); } 
-            catch (e) { if (!allowMissing) throw e; alertHistory = null; }
-            if (!alertHistory && !allowMissing) throw new Error(`Missing alerts`);
-        }
+        const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
+        const social   = metadata.rootDataDependencies?.includes('social')   ? { today: await loader.loadSocial(dateStr) }   : null;
 
-        const seriesData = { root: {}, results: {} }; 
+        // 2. Build Context Base
+        const contextBase = {
+            dateStr, metadata, mappings, insights, socialData: social,
+            computedDependencies: computedDeps,
+            previousComputedDependencies: prevDeps, config, deps,
+            allRankings: rankings,
+            allRankingsYesterday: rankingsYesterday,
+            ...variableRoots,
+            seriesData
+        };
 
+        // 3. Sharded Execution (Price) or Standard
         if (metadata.rootDataDependencies?.includes('price')) {
             logger.log('INFO', `[Executor] Running Batched/Sharded Execution for ${metadata.name}`);
             const shardRefs = await loader.getPriceShardReferences();
-            if (shardRefs.length === 0) { logger.log('WARN', '[Executor] No price shards found.'); return {}; }
+            if (!shardRefs.length) { 
+                logger.log('WARN', '[Executor] No price shards found.'); 
+                return {}; 
+            }
             
-            let processedCount = 0;
+            const stats = { processedShards: 0, processedItems: 0 };
             for (const ref of shardRefs) {
                 const shardData = await loader.loadPriceShard(ref);
-                const partialContext = ContextFactory.buildMetaContext({ 
-                    dateStr, metadata, mappings, insights, socialData: social, 
-                    prices: { history: shardData }, computedDependencies: computedDeps, 
-                    previousComputedDependencies: prevDeps, config, deps,
-                    allRankings: rankings,
-                    allRankingsYesterday: rankingsYesterday,
-                    ratings: ratings || {},
-                    pageViews: pageViews || {},
-                    watchlistMembership: watchlistMembership || {},
-                    alertHistory: alertHistory || {},
-                    seriesData
-                });
-                
-                await calcInstance.process(partialContext);
-                partialContext.prices = null; 
-                processedCount++;
+                await calcInstance.process(ContextFactory.buildMetaContext({ 
+                    ...contextBase, 
+                    prices: { history: shardData } 
+                }));
                 
                 stats.processedShards++;
                 stats.processedItems += Object.keys(shardData).length;
             }
-            logger.log('INFO', `[Executor] Finished Batched Execution for ${metadata.name} (${processedCount} shards).`);
-            
             calcInstance._executionStats = stats;
             return calcInstance.getResult ? await calcInstance.getResult() : {};
-        } else {
-            const context = ContextFactory.buildMetaContext({ 
-                dateStr, metadata, mappings, insights, socialData: social, 
-                prices: {}, computedDependencies: computedDeps, 
-                previousComputedDependencies: prevDeps, config, deps,
-                allRankings: rankings,
-                allRankingsYesterday: rankingsYesterday,
-                ratings: ratings || {},
-                pageViews: pageViews || {},
-                watchlistMembership: watchlistMembership || {},
-                alertHistory: alertHistory || {},
-                seriesData
-            });
-            const res = await calcInstance.process(context);
-            
-            stats.processedItems = 1; 
-            calcInstance._executionStats = stats;
-            
+        } 
+        else {
+            const res = await calcInstance.process(ContextFactory.buildMetaContext({ 
+                ...contextBase, 
+                prices: {} 
+            }));
+            calcInstance._executionStats = { processedItems: 1 };
             return res;
         }
     }
 }
 
+// =============================================================================
+// INTERNAL HELPERS
+// =============================================================================
+
+/**
+ * Loads variable root data types (Ratings, PageViews, Watchlist, Alerts) 
+ * based on calculation requirements. Handles strict vs. optional failures.
+ */
+async function loadVariableRootData(loader, dateStr, calcs, logger) {
+    const requirements = {}; 
+    const results = { ratings: null, pageViews: null, watchlistMembership: null, alertHistory: null };
+    
+    // Map internal key to Loader Method
+    const loaderMap = {
+        ratings: { method: 'loadRatings', resultKey: 'ratings' },
+        pageViews: { method: 'loadPageViews', resultKey: 'pageViews' },
+        watchlist: { method: 'loadWatchlistMembership', resultKey: 'watchlistMembership' },
+        alerts: { method: 'loadAlertHistory', resultKey: 'alertHistory' }
+    };
+
+    // 1. Analyze Requirements
+    for (const c of calcs) {
+        const deps = c.rootDataDependencies || [];
+        const strict = c.canHaveMissingRoots !== true;
+        deps.forEach(d => {
+            if (loaderMap[d]) {
+                if (!requirements[d]) requirements[d] = { strict: false };
+                if (strict) requirements[d].strict = true;
+            }
+        });
+    }
+
+    // 2. Fetch Data
+    const promises = Object.entries(requirements).map(async ([key, req]) => {
+        const { method, resultKey } = loaderMap[key];
+        try {
+            results[resultKey] = await loader[method](dateStr);
+        } catch (e) {
+            if (req.strict) throw new Error(`[MetaExecutor] Missing required root '${key}': ${e.message}`);
+            logger.log('WARN', `[MetaExecutor] Missing optional root '${key}'.`);
+            results[resultKey] = null;
+        }
+    });
+
+    await Promise.all(promises);
+    return results;
+}
+
+/**
+ * Loads time-series data for both Root inputs and Computation results.
+ */
+async function loadSeriesData(loader, dateStr, calcs, manifestLookup, config, deps) {
+    const rootRequests = {};
+    const depRequests = {};
+
+    // 1. Aggregate Lookback Depths
+    for (const c of calcs) {
+        if (c.rootDataSeries) {
+            Object.entries(c.rootDataSeries).forEach(([type, val]) => {
+                const days = typeof val === 'object' ? val.lookback : val;
+                rootRequests[type] = Math.max(rootRequests[type] || 0, days);
+            });
+        }
+        if (c.dependencySeries) {
+            Object.entries(c.dependencySeries).forEach(([name, val]) => {
+                const days = typeof val === 'object' ? val.lookback : val;
+                const norm = normalizeName(name);
+                depRequests[norm] = Math.max(depRequests[norm] || 0, days);
+            });
+        }
+    }
+
+    const seriesData = { root: {}, results: {} };
+    
+    // 2. Fetch Root Series
+    const rootLoaders = {
+        alerts: 'loadAlertHistory',
+        insights: 'loadInsights',
+        ratings: 'loadRatings',
+        watchlist: 'loadWatchlistMembership',
+        rankings: 'loadRankings'
+    };
+
+    const rootPromises = Object.entries(rootRequests).map(async ([type, days]) => {
+        if (rootLoaders[type]) {
+            deps.logger.log('INFO', `[MetaExecutor] Loading ${days}-day series for Root '${type}'`);
+            const res = await loader.loadSeries(rootLoaders[type], dateStr, days);
+            seriesData.root[type] = res.data;
+        }
+    });
+
+    // 3. Fetch Dependency Series
+    const depNames = Object.keys(depRequests);
+    if (depNames.length > 0) {
+        const maxDays = Math.max(...Object.values(depRequests));
+        deps.logger.log('INFO', `[MetaExecutor] Loading up to ${maxDays}-day series for Dependencies: ${depNames.join(', ')}`);
+        seriesData.results = await fetchResultSeries(dateStr, depNames, manifestLookup, config, deps, maxDays);
+    }
+
+    await Promise.all(rootPromises);
+    return seriesData;
+}
+
 module.exports = { MetaExecutor };