bulltrackers-module 1.0.732 → 1.0.733

package/functions/computation-system/executors/MetaExecutor.js

@@ -1,279 +0,0 @@
-/**
- * @fileoverview Executor for "Meta" (global) calculations.
- * REFACTORED: Applied DRY principles to Root Data and Series loading.
- */
-const { normalizeName }     = require('../utils/utils');
-const { CachedDataLoader }  = require('../data/CachedDataLoader');
-const { ContextFactory }    = require('../context/ContextFactory');
-const { commitResults }     = require('../persistence/ResultCommitter');
-const { fetchResultSeries } = require('../data/DependencyFetcher');
-const { getManifest }       = require('../topology/ManifestLoader');
-
-// Helper to get calculations - prefer config.calculations, fallback to direct require
-function getCalculations(config) {
-    if (config && config.calculations) return config.calculations;
-    try { return require('aiden-shared-calculations-unified').calculations; } 
-    catch (e) { return {}; }
-}
-
-class MetaExecutor {
-
-    // =========================================================================
-    // PRIMARY ENTRY POINT (Batch Execution)
-    // =========================================================================
-    static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps) {
-        const { logger } = deps;
-        const dStr = date.toISOString().slice(0, 10);
-        const loader = new CachedDataLoader(config, deps);
-
-        // 1. Setup Manifest Lookup (use activeProductLines and calculations from config)
-        const allManifests = getManifest(config.activeProductLines || [], getCalculations(config), deps); 
-        const manifestLookup = Object.fromEntries(allManifests.map(m => [normalizeName(m.name), m.category]));
-
-        // 2. Load Base Data (Always Required)
-        const [mappings, rankings, verifications, piMasterList] = await Promise.all([
-            loader.loadMappings(),
-            loader.loadRankings(dStr),
-            loader.loadVerifications(),
-            loader.loadPIMasterList() // <--- ADDED LOAD
-        ]);
-
-        // 3. Load Historical Rankings (if needed)
-        let rankingsYesterday = null;
-        if (calcs.some(c => c.isHistorical)) {
-            const prevStr = new Date(date.getTime() - 86400000).toISOString().slice(0, 10);
-            rankingsYesterday = await loader.loadRankings(prevStr);
-        }
-
-        // 4. Load Variable Root Data & Series (Refactored Helpers)
-        const variableRoots = await loadVariableRootData(loader, dStr, calcs, logger);
-        const seriesData = await loadSeriesData(loader, dStr, calcs, manifestLookup, config, deps);
-
-        // 5. Execution Loop
-        const state = {};
-        for (const c of calcs) {
-            const inst = new c.class();
-            inst.manifest = c;
-            
-            const context = ContextFactory.buildMetaContext({
-                dateStr: dStr,
-                metadata: c,
-                mappings,
-                insights: { today: rootData.todayInsights },
-                socialData: { today: rootData.todaySocialPostInsights },
-                computedDependencies: fetchedDeps,
-                previousComputedDependencies: previousFetchedDeps,
-                config, deps,
-                allRankings: rankings,
-                allRankingsYesterday: rankingsYesterday,
-                allVerifications: verifications,
-                piMasterList, // <--- INJECTED HERE
-                // Spread variable roots directly (ratings, pageViews, etc.)
-                ...variableRoots,
-                seriesData
-            });
-
-            try {
-                const result = await inst.process(context);
-                
-                // Fallback if the computation didn't set this.results internally
-                if (!inst.results) inst.results = result;
-                
-                // Debug logging condensed
-                if (inst.results && Object.keys(inst.results).length === 0) {
-                    logger.log('WARN', `[MetaExecutor] ⚠️ ${c.name} produced EMPTY results.`);
-                }
-                
-                state[c.name] = inst;
-            } catch (e) {
-                logger.log('ERROR', `[MetaExecutor] ❌ ${c.name} failed: ${e.message}`);
-            }
-        }
-
-        // Force 'isInitialWrite: true' for robust cleanup of old keys
-        return await commitResults(state, dStr, passName, config, deps, false, { isInitialWrite: true });
-    }
-
-    // =========================================================================
-    // SINGLE EXECUTION (Sharded/On-Demand)
-    // =========================================================================
-    static async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps, config, deps, loader) {
-        const { logger } = deps;
-        const calcs = [metadata]; // Treat single as list for helpers
-
-        // Build manifestLookup using calculations from config (set in index.js)
-        const allManifests = getManifest(config.activeProductLines || [], getCalculations(config), deps);
-        const manifestLookup = Object.fromEntries(allManifests.map(m => [normalizeName(m.name), m.category]));
-
-        // 1. Load Data using Shared Helpers
-        const [mappings, rankings, variableRoots, seriesData, piMasterList] = await Promise.all([
-            loader.loadMappings(),
-            loader.loadRankings(dateStr),
-            loadVariableRootData(loader, dateStr, calcs, logger),
-            loadSeriesData(loader, dateStr, calcs, manifestLookup, config, deps),
-            loader.loadPIMasterList()
-        ]);
-
-        let rankingsYesterday = null;
-        if (metadata.isHistorical) {
-            const prevStr = new Date(new Date(dateStr).getTime() - 86400000).toISOString().slice(0, 10);
-            rankingsYesterday = await loader.loadRankings(prevStr);
-        }
-
-        const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
-        const social   = metadata.rootDataDependencies?.includes('social')   ? { today: await loader.loadSocial(dateStr) }   : null;
-
-        // 2. Build Context Base
-        const contextBase = {
-            dateStr, metadata, mappings, insights, socialData: social,
-            computedDependencies: computedDeps,
-            previousComputedDependencies: prevDeps, config, deps,
-            allRankings: rankings,
-            allRankingsYesterday: rankingsYesterday,
-            piMasterList, 
-            ...variableRoots,
-            seriesData
-        };
-
-        // 3. Sharded Execution (Price) or Standard
-        if (metadata.rootDataDependencies?.includes('price')) {
-            logger.log('INFO', `[Executor] Running Batched/Sharded Execution for ${metadata.name}`);
-            const shardRefs = await loader.getPriceShardReferences();
-            if (!shardRefs.length) { 
-                logger.log('WARN', '[Executor] No price shards found.'); 
-                return {}; 
-            }
-            
-            const stats = { processedShards: 0, processedItems: 0 };
-            for (const ref of shardRefs) {
-                const shardData = await loader.loadPriceShard(ref);
-                await calcInstance.process(ContextFactory.buildMetaContext({ 
-                    ...contextBase, 
-                    prices: { history: shardData } 
-                }));
-                
-                stats.processedShards++;
-                stats.processedItems += Object.keys(shardData).length;
-            }
-            calcInstance._executionStats = stats;
-            return calcInstance.getResult ? await calcInstance.getResult() : {};
-        } 
-        else {
-            const res = await calcInstance.process(ContextFactory.buildMetaContext({ 
-                ...contextBase, 
-                prices: {} 
-            }));
-            calcInstance._executionStats = { processedItems: 1 };
-            return res;
-        }
-    }
-}
-
-// =============================================================================
-// INTERNAL HELPERS
-// =============================================================================
-
-/**
- * Loads variable root data types (Ratings, PageViews, Watchlist, Alerts) 
- * based on calculation requirements. Handles strict vs. optional failures.
- */
-async function loadVariableRootData(loader, dateStr, calcs, logger) {
-    const requirements = {}; 
-    const results = { ratings: null, pageViews: null, watchlistMembership: null, alertHistory: null };
-    
-    // Map internal key to Loader Method
-    const loaderMap = {
-        ratings: { method: 'loadRatings', resultKey: 'ratings' },
-        pageViews: { method: 'loadPageViews', resultKey: 'pageViews' },
-        watchlist: { method: 'loadWatchlistMembership', resultKey: 'watchlistMembership' },
-        alerts: { method: 'loadAlertHistory', resultKey: 'alertHistory' }
-    };
-
-    // 1. Analyze Requirements
-    for (const c of calcs) {
-        const deps = c.rootDataDependencies || [];
-        const strict = c.canHaveMissingRoots !== true;
-        deps.forEach(d => {
-            if (loaderMap[d]) {
-                if (!requirements[d]) requirements[d] = { strict: false };
-                if (strict) requirements[d].strict = true;
-            }
-        });
-    }
-
-    // 2. Fetch Data
-    const promises = Object.entries(requirements).map(async ([key, req]) => {
-        const { method, resultKey } = loaderMap[key];
-        try {
-            results[resultKey] = await loader[method](dateStr);
-        } catch (e) {
-            if (req.strict) throw new Error(`[MetaExecutor] Missing required root '${key}': ${e.message}`);
-            logger.log('WARN', `[MetaExecutor] Missing optional root '${key}'.`);
-            results[resultKey] = null;
-        }
-    });
-
-    await Promise.all(promises);
-    return results;
-}
-
-/**
- * Loads time-series data for both Root inputs and Computation results.
- */
-async function loadSeriesData(loader, dateStr, calcs, manifestLookup, config, deps) {
-    const rootRequests = {};
-    const depRequests = {};        // norm -> { days, originalName }
-
-    // 1. Aggregate Lookback Depths from dependencySeries config
-    for (const c of calcs) {
-        if (c.rootDataSeries) {
-            Object.entries(c.rootDataSeries).forEach(([type, val]) => {
-                const days = typeof val === 'object' ? val.lookback : val;
-                rootRequests[type] = Math.max(rootRequests[type] || 0, days);
-            });
-        }
-        if (c.dependencySeries) {
-            Object.entries(c.dependencySeries).forEach(([name, val]) => {
-                const days = typeof val === 'object' ? val.lookback : val;
-                const norm = normalizeName(name);
-                if (!depRequests[norm] || depRequests[norm].days < days) {
-                    depRequests[norm] = { days, originalName: name };
-                }
-            });
-        }
-    }
-
-    const seriesData = { root: {}, results: {} };
-    
-    // 2. Fetch Root Series
-    const rootLoaders = {
-        alerts: 'loadAlertHistory',
-        insights: 'loadInsights',
-        ratings: 'loadRatings',
-        watchlist: 'loadWatchlistMembership',
-        rankings: 'loadRankings'
-    };
-
-    const rootPromises = Object.entries(rootRequests).map(async ([type, days]) => {
-        if (rootLoaders[type]) {
-            deps.logger.log('INFO', `[MetaExecutor] Loading ${days}-day series for Root '${type}'`);
-            const res = await loader.loadSeries(rootLoaders[type], dateStr, days);
-            seriesData.root[type] = res.data;
-        }
-    });
-
-    // 3. Fetch Dependency Series (category comes from manifestLookup)
-    const depEntries = Object.values(depRequests);
-    if (depEntries.length > 0) {
-        const depOriginalNames = depEntries.map(e => e.originalName);
-        const maxDays = Math.max(...depEntries.map(e => e.days));
-        
-        deps.logger.log('INFO', `[MetaExecutor] Loading up to ${maxDays}-day series for Dependencies: ${depOriginalNames.join(', ')}`);
-        seriesData.results = await fetchResultSeries(dateStr, depOriginalNames, manifestLookup, config, deps, maxDays);
-    }
-
-    await Promise.all(rootPromises);
-    return seriesData;
-}
-
-module.exports = { MetaExecutor };

package/functions/computation-system/executors/PriceBatchExecutor.js

@@ -1,108 +0,0 @@
-/**
- * @fileoverview Specialized Executor for Price-Dependent Batch computations.
- */
-const pLimit                                   = require('p-limit');
-const { normalizeName }                        = require('../utils/utils');
-const { getRelevantShardRefs, loadDataByRefs } = require('../utils/data_loader');
-const { CachedDataLoader }                     = require('../data/CachedDataLoader');
-const mathLayer                                = require('../layers/index');
-const { LEGACY_MAPPING }                       = require('../topology/HashManager');
-
-async function runBatchPriceComputation(config, deps, dateStrings, calcs, targetTickers = []) {
-    const { logger, db, calculationUtils } = deps;
-    const cachedLoader = new CachedDataLoader(config, deps);
-    const mappings     = await cachedLoader.loadMappings();
-    
-    let targetInstrumentIds = [];
-    if (targetTickers && targetTickers.length > 0) {
-        const tickerToInst  = mappings.tickerToInstrument || {};
-        targetInstrumentIds = targetTickers.map(t => tickerToInst[t]).filter(id => id);
-        if (targetInstrumentIds.length === 0) { logger.log('WARN', '[BatchPrice] Target tickers provided but no IDs found. Aborting.'); return; } 
-    }
-    
-    const allShardRefs = await getRelevantShardRefs(config, deps, targetInstrumentIds);
-    if (!allShardRefs.length) { logger.log('WARN', '[BatchPrice] No relevant price shards found. Exiting.'); return; }
-    
-    const OUTER_CONCURRENCY_LIMIT = 2, SHARD_BATCH_SIZE = 20, WRITE_BATCH_LIMIT = 50;
-    logger.log('INFO', `[BatchPrice] Execution Plan: ${dateStrings.length} days, ${allShardRefs.length} shards. Concurrency: ${OUTER_CONCURRENCY_LIMIT}.`);
-    
-    const shardChunks = []; 
-    for (let i = 0; i < allShardRefs.length; i += SHARD_BATCH_SIZE) { shardChunks.push(allShardRefs.slice(i, i + SHARD_BATCH_SIZE)); }
-    
-    const outerLimit = pLimit(OUTER_CONCURRENCY_LIMIT);
-    const chunkPromises = [];
-
-    for (let index = 0; index < shardChunks.length; index++) {
-        const shardChunkRefs = shardChunks[index];
-        chunkPromises.push(outerLimit(async () => {
-            try {
-                logger.log('INFO', `[BatchPrice] Processing chunk ${index + 1}/${shardChunks.length} (${shardChunkRefs.length} shards)...`);
-                const pricesData = await loadDataByRefs(config, deps, shardChunkRefs);
-                if (targetInstrumentIds.length > 0) { 
-                    const requestedSet = new Set(targetInstrumentIds); 
-                    for (const loadedInstrumentId in pricesData) { 
-                        if (!requestedSet.has(loadedInstrumentId)) { delete pricesData[loadedInstrumentId]; } 
-                    } 
-                }
-                
-                const writes = [];
-                for (const dateStr of dateStrings) {
-                    const dynamicMathContext = {};
-                    for (const [key, value] of Object.entries(mathLayer)) { 
-                        dynamicMathContext[key] = value; 
-                        if (LEGACY_MAPPING[key]) { dynamicMathContext[LEGACY_MAPPING[key]] = value;} 
-                    }
-                    const context = { mappings, prices: { history: pricesData }, date: { today: dateStr }, math: dynamicMathContext };
-                    
-                    for (const calcManifest of calcs) {
-                        try {
-                            const instance = new calcManifest.class(); 
-                            await instance.process(context); 
-                            const result = await instance.getResult();
-                            if (result && Object.keys(result).length > 0) {
-                                let dataToWrite = result; 
-                                if (result.by_instrument) dataToWrite = result.by_instrument;
-                                if (Object.keys(dataToWrite).length > 0) {
-                                    const docRef = db.collection(config.resultsCollection)
-                                        .doc(dateStr)
-                                        .collection(config.resultsSubcollection)
-                                        .doc(calcManifest.category)
-                                        .collection(config.computationsSubcollection)
-                                        .doc(normalizeName(calcManifest.name));
-                                    
-                                    writes.push({ 
-                                        ref: docRef, 
-                                        data: { 
-                                            ...dataToWrite, 
-                                            _completed: true,
-                                            _lastUpdated: new Date().toISOString() 
-                                        }, 
-                                        options: { merge: true } 
-                                    });
-                                }
-                            }
-                        } catch (err) { logger.log('ERROR', `[BatchPrice] \u2716 Failed ${calcManifest.name} for ${dateStr}: ${err.message}`); }
-                    }
-                }
-                
-                if (writes.length > 0) {
-                    const commitBatches = []; 
-                    for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) { commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT)); }
-                    const commitLimit = pLimit(10);
-                    await Promise.all(commitBatches.map((batchWrites, bIndex) => commitLimit(async () => {
-                        const batch = db.batch(); batchWrites.forEach(w => batch.set(w.ref, w.data, w.options));
-                        try { 
-                            await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`); 
-                        } catch (commitErr) { 
-                            logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message }); 
-                        }
-                    })));
-                }
-            } catch (chunkErr) { logger.log('ERROR', `[BatchPrice] Fatal error processing Chunk ${index}.`, { error: chunkErr.message }); }
-        }));
-    }
-    await Promise.all(chunkPromises);
-    logger.log('INFO', '[BatchPrice] Optimization pass complete.');
-}
-
-module.exports = { runBatchPriceComputation };