bulltrackers-module 1.0.127 → 1.0.129

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -4,797 +4,44 @@
  *
  * This orchestrator is designed to be run by a separate Cloud Function for each "pass".
  * It reads its pass number from the config and executes only those calculations.
- *
- * - Pass 1: Runs "standard" calcs that stream user data.
- * - Pass 2+: Runs "meta" calcs, which it first supplies with dependencies
- * by fetching the results of *previous* passes from Firestore.
- */
-
-const { FieldPath } = require('@google-cloud/firestore');
-const { 
-    getPortfolioPartRefs, 
-    loadFullDayMap, // <-- Ensure loadFullDayMap is imported
-    loadDataByRefs, 
-    loadDailyInsights,
-    loadDailySocialPostInsights,
-    getHistoryPartRefs // <-- IMPORT NEW FUNCTION
-} = require('../utils/data_loader.js');
-
-const {
-    normalizeName,
-    getExpectedDateStrings, 
-    getFirstDateFromSourceData,
-    commitBatchInChunks
-} = require('../utils/utils.js');
-
-
-/**
- * Groups the manifest by pass number.
- * @param {Array<object>} manifest - The computation manifest.
- * @returns {object} { '1': [...], '2': [...] }
- */
-function groupByPass(manifest) {
-    return manifest.reduce((acc, calc) => {
-        (acc[calc.pass] = acc[calc.pass] || []).push(calc);
-        return acc;
-    }, {});
-}
-
-/**
- * Checks if a calculation's root data dependencies are met.
- * @param {object} calcManifest - The manifest entry for the calculation.
- * @param {object} rootDataStatus - The status object from checkRootDataAvailability.
- * @returns {boolean} True if dependencies are met, false otherwise.
- */
-function checkRootDependencies(calcManifest, rootDataStatus) {
-    if (!calcManifest.rootDataDependencies || calcManifest.rootDataDependencies.length === 0) {
-        return true;
-    }
-    for (const dep of calcManifest.rootDataDependencies) {
-        if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) return false;
-        if (dep === 'insights' && !rootDataStatus.hasInsights) return false;
-        if (dep === 'social' && !rootDataStatus.hasSocial) return false;
-        if (dep === 'history' && !rootDataStatus.hasHistory) return false; // <-- ADDED THIS LINE
-    }
-    return true; // All dependencies were met
-}
-
-
-/**
- * Checks if the root data (portfolios, insights, social, history) exists for a given day.
- * @param {string} dateStr - The date string to check (YYYY-MM-DD).
- * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger, calculationUtils.
- * @returns {Promise<object>} { portfolioRefs, insightsData, socialData, historyRefs, hasPortfolio, hasInsights, hasSocial, hasHistory }
+ * This file contains the high-level "manual" of steps. The "how-to" logic
+ * is extracted into 'computation_system_utils.js'.
  */
-async function checkRootDataAvailability(dateStr, config, dependencies) {
-    const { logger } = dependencies;
-    logger.log('INFO', `[PassRunner] Checking root data availability for ${dateStr}...`);
-
-    try {
-        // --- MODIFIED: Add getHistoryPartRefs to the parallel load ---
-        const [portfolioRefs, insightsData, socialData, historyRefs] = await Promise.all([
-            getPortfolioPartRefs(config, dependencies, dateStr),
-            loadDailyInsights(config, dependencies, dateStr),
-            loadDailySocialPostInsights(config, dependencies, dateStr),
-            getHistoryPartRefs(config, dependencies, dateStr) // <-- ADD THIS
-        ]);
+const { groupByPass, checkRootDataAvailability, fetchDependenciesForPass, filterCalculations, runStandardComputationPass, runMetaComputationPass } = require('./orchestration_helpers.js');
+const { getExpectedDateStrings, getFirstDateFromSourceData } = require('../utils/utils.js');
 
-        const hasPortfolio = (portfolioRefs && portfolioRefs.length > 0);
-        const hasInsights = !!insightsData;
-        const hasSocial = !!socialData;
-        const hasHistory = (historyRefs && historyRefs.length > 0); // <-- ADD THIS
-
-        return {
-            portfolioRefs: portfolioRefs || [],
-            insightsData: insightsData || null,
-            socialData: socialData || null,
-            historyRefs: historyRefs || [], // <-- ADD THIS
-            hasPortfolio: hasPortfolio,
-            hasInsights: hasInsights,
-            hasSocial: hasSocial,
-            hasHistory: hasHistory // <-- ADD THIS
-        };
-
-    } catch (err) {
-        logger.log('ERROR', `[PassRunner] Error checking data availability for ${dateStr}`, { errorMessage: err.message });
-        return { 
-            portfolioRefs: [], insightsData: null, socialData: null, historyRefs: [], // <-- ADD historyRefs
-            hasPortfolio: false, hasInsights: false, hasSocial: false, hasHistory: false // <-- ADD hasHistory
-        };
-    }
-}
-
-/**
- * (NEW) Fetches all computed dependencies for a given pass and date from Firestore.
- * @param {string} dateStr - The date string (YYYY-MM-DD).
- * @param {Array<object>} calcsInPass - The manifest entries for this pass.
- * @param {Array<object>} fullManifest - The *entire* computation manifest.
- * @param {object} config - The computation system configuration.
- * @param {object} dependencies - Contains db, logger.
- * @returns {Promise<object>} A map of { 'calc-name': result, ... }
- */
-async function fetchDependenciesForPass(dateStr, calcsInPass, fullManifest, config, dependencies) {
-    const { db, logger } = dependencies;
-    const { resultsCollection, resultsSubcollection, computationsSubcollection } = config;
-    
-    // --- THIS IS THE FIX ---
-    // Build the manifestMap from the *fullManifest* so we can find
-    // dependencies from *all* passes, not just the current one.
-    const manifestMap = new Map();
-    for (const calc of fullManifest) {
-        manifestMap.set(normalizeName(calc.name), calc);
-    }
-    // --- END FIX ---
-    
-    const requiredDeps = new Set();
-    
-    // 1. Get all unique dependencies required by calcs in *this* pass
-    for (const calc of calcsInPass) {
-        if (calc.type === 'meta' && calc.dependencies) {
-            calc.dependencies.forEach(depName => requiredDeps.add(normalizeName(depName)));
-        }
-    }
-
-    if (requiredDeps.size === 0) {
-        logger.log('INFO', `[PassRunner] No Firestore dependencies to fetch for this pass on ${dateStr}.`);
-        return {};
-    }
-
-    logger.log('INFO', `[PassRunner] Fetching ${requiredDeps.size} dependencies from Firestore for ${dateStr}...`);
-    
-    const docRefs = [];
-    const depNames = [];
-
-    // 2. Build the list of Firestore document references
-    for (const calcName of requiredDeps) {
-        const calcManifest = manifestMap.get(calcName); // Now correctly finds deps from other passes
-        if (!calcManifest) {
-            logger.log('ERROR', `[PassRunner] Cannot find manifest entry for dependency "${calcName}". This is a manifest error. Skipping dependency.`);
-            continue;
-        }
-        
-        const category = calcManifest.category || 'unknown';
-        const docRef = db.collection(resultsCollection).doc(dateStr)
-                         .collection(resultsSubcollection).doc(category)
-                         .collection(computationsSubcollection).doc(calcName);
-        
-        docRefs.push(docRef);
-        depNames.push(calcName);
-    }
-
-    // 3. Fetch all dependencies in one batch
-    const fetchedDependencies = {};
-    if (docRefs.length > 0) {
-        const snapshots = await db.getAll(...docRefs);
-        snapshots.forEach((doc, i) => {
-            const calcName = depNames[i];
-            if (doc.exists) {
-                fetchedDependencies[calcName] = doc.data();
-            } else {
-                fetchedDependencies[calcName] = null; // Mark as null if not found
-            }
-        });
-    }
-    
-    logger.log('INFO', `[PassRunner] Successfully fetched ${Object.keys(fetchedDependencies).length} dependencies for ${dateStr}.`);
-    return fetchedDependencies;
-}
-
-
-/**
- * Main pipe: pipe.computationSystem.runComputationPass
- * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger, calculationUtils.
- * @param {Array<object>} computationManifest - The injected computation manifest.
- * @returns {Promise<Object>} Summary of all passes.
- */
 async function runComputationPass(config, dependencies, computationManifest) {
     const { logger } = dependencies;
-    
-    // --- (NEW) Get the pass number this function is responsible for ---
-    const passToRun = String(config.COMPUTATION_PASS_TO_RUN);
-    if (!passToRun) {
-        logger.log('ERROR', '[PassRunner] FATAL: COMPUTATION_PASS_TO_RUN is not defined in config. Aborting.');
-        return;
-    }
-    logger.log('INFO', `🚀 [PassRunner] Starting run for PASS ${passToRun}...`);
-    // --- END NEW ---
+    const passToRun = String(config.COMPUTATION_PASS_TO_RUN); if (!passToRun) return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
+    logger.log('INFO', `🚀 Starting PASS ${passToRun}...`);
 
-    const summary = {};
-    const yesterday = new Date();
-    yesterday.setUTCDate(yesterday.getUTCDate() - 1);
+    const yesterday = new Date(); yesterday.setUTCDate(yesterday.getUTCDate()-1); 
     const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
-
     const firstDate = await getFirstDateFromSourceData(config, dependencies);
-    const startDateUTC = firstDate
-       ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate()))
-       : new Date(config.earliestComputationDate + 'T00:00:00Z');
-
+    const startDateUTC = firstDate ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate())) : new Date(config.earliestComputationDate+'T00:00:00Z');
     const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
 
     const passes = groupByPass(computationManifest);
-    const calcsInThisPass = passes[passToRun] || [];
-    
-    if (calcsInThisPass.length === 0) {
-        logger.log('WARN', `[PassRunner] No calculations found in manifest for Pass ${passToRun}. Exiting.`);
-        return;
-    }
+    const calcsInThisPass = passes[passToRun] || []; if (!calcsInThisPass.length) return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
 
-    const standardCalcs = calcsInThisPass.filter(c => c.type === 'standard');
-    const metaCalcs = calcsInThisPass.filter(c => c.type === 'meta');
-    logger.log('INFO', `[PassRunner] Found ${standardCalcs.length} standard and ${metaCalcs.length} meta calcs for Pass ${passToRun}.`);
+    const standardCalcs = calcsInThisPass.filter(c => c.type==='standard'); 
+    const metaCalcs = calcsInThisPass.filter(c => c.type==='meta');
 
-    
-    // --- Process ONE DAY at a time, in order ---
     for (const dateStr of allExpectedDates) {
-        const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-        
-        // 1. Check for root data (portfolios, insights, social, AND HISTORY)
-        const rootData = await checkRootDataAvailability(dateStr, config, dependencies);
-        const rootDataStatus = {
-            hasPortfolio: rootData.hasPortfolio,
-            hasInsights: rootData.hasInsights,
-            hasSocial: rootData.hasSocial,
-            hasHistory: rootData.hasHistory // <-- ADD THIS
-        };
-
-        if (!rootData.hasPortfolio && !rootData.hasInsights && !rootData.hasSocial && !rootData.hasHistory) {
-            logger.log('WARN', `[PassRunner] Skipping Pass ${passToRun} for ${dateStr} due to missing all root data.`);
-            continue; // Skip to the next day
-        }
-        
-        logger.log('INFO', `[PassRunner] Processing Pass ${passToRun} for ${dateStr}...`);
-        
-        // 2. (NEW) Fetch all dependencies for this pass and date from Firestore
-        // This is skipped for Pass 1, as `requiredDeps.size` will be 0
-        const fetchedDependencies = await fetchDependenciesForPass(
-            dateStr, 
-            calcsInThisPass, 
-            computationManifest, // <-- Pass the *full* manifest here
-            config, 
-            dependencies
-        );
-        
-        const skippedCalculations = new Set();
-        
-        // 3. Filter calculations based on root data
-        const standardCalcsToRun = [];
-        for (const calcManifest of standardCalcs) {
-            if (checkRootDependencies(calcManifest, rootDataStatus)) {
-                standardCalcsToRun.push(calcManifest);
-            } else {
-                logger.log('INFO', `[Pass ${passToRun}] Skipping standard calc "${calcManifest.name}" for ${dateStr} due to missing root data.`);
-                skippedCalculations.add(calcManifest.name);
-            }
-        }
-        
-        const metaCalcsToRun = [];
-        for (const calcManifest of metaCalcs) {
-            const calcName = calcManifest.name;
-            
-            // Check 1: Are root data dependencies met?
-            if (!checkRootDependencies(calcManifest, rootDataStatus)) {
-                logger.log('INFO', `[Pass ${passToRun} (Meta)] Skipping meta calc "${calcName}" for ${dateStr} due to missing root data.`);
-                skippedCalculations.add(calcName);
-                continue;
-            }
-            
-            // Check 2: Are *computed* dependencies (from previous passes) met?
-            let depCheck = true;
-            let missingDepName = '';
-            for (const depName of (calcManifest.dependencies || [])) {
-                const normalizedDepName = normalizeName(depName);
-                if (!fetchedDependencies[normalizedDepName]) { // Check if null or undefined
-                    depCheck = false;
-                    missingDepName = normalizedDepName;
-                    break;
-                }
-            }
-            
-            if (depCheck) {
-                metaCalcsToRun.push(calcManifest);
-            } else {
-                logger.log('WARN', `[Pass ${passToRun} (Meta)] Skipping meta calc "${calcName}" for ${dateStr} due to missing computed dependency "${missingDepName}" from Firestore.`);
-                skippedCalculations.add(calcName);
-            }
-        }
-        
-        // --- 4. Run the filtered calculations ---
+        const dateToProcess = new Date(dateStr+'T00:00:00Z');
         try {
-            // Run standard calcs for this pass (e.g., Pass 1)
-            if (standardCalcsToRun.length > 0) {
-                await runUnifiedComputation(
-                    dateToProcess,
-                    standardCalcsToRun,
-                    `Pass ${passToRun} (Standard)`,
-                    config,
-                    dependencies,
-                    rootData 
-                );
-            }
-
-            // Run meta calcs for this pass (e.g., Pass 2, 3, 4)
-            if (metaCalcsToRun.length > 0) {
-                await runMetaComputation(
-                    dateToProcess,
-                    metaCalcsToRun,
-                    `Pass ${passToRun} (Meta)`,
-                    config,
-                    dependencies,
-                    fetchedDependencies, // <-- This is the NEW object from Firestore
-                    rootData
-                );
-            }
-             logger.log('SUCCESS', `[PassRunner] Completed Pass ${passToRun} for ${dateStr}.`);
+            const rootData = await checkRootDataAvailability(dateStr, config, dependencies); if (!rootData) continue;
+            const fetchedDeps = await fetchDependenciesForPass(dateStr, calcsInThisPass, computationManifest, config, dependencies);
+            const { standardCalcsToRun, metaCalcsToRun } = filterCalculations(standardCalcs, metaCalcs, rootData.status, fetchedDeps, passToRun, dateStr, logger);
+            if (standardCalcsToRun.length) await runStandardComputationPass(dateToProcess, standardCalcsToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData);
+            if (metaCalcsToRun.length) await runMetaComputationPass(dateToProcess, metaCalcsToRun, `Pass ${passToRun} (Meta)`, config, dependencies, fetchedDeps, rootData);
+            logger.log('SUCCESS', `[PassRunner] Completed Pass ${passToRun} for ${dateStr}.`);
         } catch (err) {
             logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
         }
-    } // End dates loop
-
-    logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
-    return summary;
-}
-
-/**
- * Internal sub-pipe: Initializes calculator instances.
- */
-function initializeCalculators(calculationsToRun, logger) {
-    const state = {};
-    for (const calcManifest of calculationsToRun) {
-        const calcName = normalizeName(calcManifest.name);
-        const CalculationClass = calcManifest.class;
-        
-        if (typeof CalculationClass === 'function') {
-            try {
-                const instance = new CalculationClass();
-                instance.manifest = calcManifest; // Attach manifest data
-                state[calcName] = instance;
-            } catch (e) {
-                logger.warn(`[PassRunner] Init failed for ${calcName}`, { errorMessage: e.message });
-                state[calcName] = null;
-            }
-        } else {
-            logger.warn(`[PassRunner] Calculation class not found in manifest for: ${calcName}`);
-            state[calcName] = null;
-        }
-    }
-    return state;
-}
-
-/**
- * Internal sub-pipe: Streams data and calls process() on "standard" calculators.
- */
-async function streamAndProcess(
-    dateStr, todayRefs, state, passName, config, dependencies, 
-    yesterdayPortfolios = {}, 
-    todayInsights = null, 
-    yesterdayInsights = null,
-    todaySocialPostInsights = null,
-    yesterdaySocialPostInsights = null,
-    todayHistoryData = null, // <-- ADD THIS
-    yesterdayHistoryData = null // <-- ADD THIS
-) {
-    const { logger, calculationUtils } = dependencies;
-    logger.log('INFO', `[${passName}] Streaming ${todayRefs.length} 'today' part docs for ${dateStr}...`);
-
-    const yesterdayDate = new Date(dateStr + 'T00:00:00Z');
-    yesterdayDate.setUTCDate(yesterdayDate.getUTCDate() - 1);
-    const yesterdayStr = yesterdayDate.toISOString().slice(0, 10);
-
-    const { instrumentToTicker, instrumentToSector } = await calculationUtils.loadInstrumentMappings();
-
-    const context = { 
-        instrumentMappings: instrumentToTicker, 
-        sectorMapping: instrumentToSector,
-        todayDateStr: dateStr,
-        yesterdayDateStr: yesterdayStr,
-        dependencies: dependencies,
-        config: config
-    };
-
-    const batchSize = config.partRefBatchSize || 10;
-    let isFirstUser = true;
-
-    for (let i = 0; i < todayRefs.length; i += batchSize) {
-        const batchRefs = todayRefs.slice(i, i + batchSize);
-        const todayPortfoliosChunk = await loadDataByRefs(config, dependencies, batchRefs);
-
-        for (const uid in todayPortfoliosChunk) {
-            const p = todayPortfoliosChunk[uid];
-            if (!p) continue;
-            const userType = p.PublicPositions ? 'speculator' : 'normal';
-            context.userType = userType;
-
-            for (const calcName in state) {
-                const calc = state[calcName];
-                if (!calc || typeof calc.process !== 'function') continue;
-                
-                const manifestCalc = calc.manifest;
-                const isSocialOrInsights = manifestCalc.category === 'socialPosts' || manifestCalc.category === 'insights';
-                const isHistoricalCalc = manifestCalc.isHistorical === true;
-                const isSpeculatorCalc = manifestCalc.category === 'speculators';
-                let processArgs;
-                
-                // --- MODIFIED: Add history data to the context args ---
-                const allContextArgs = [
-                    context, 
-                    todayInsights, 
-                    yesterdayInsights, 
-                    todaySocialPostInsights,
-                    yesterdaySocialPostInsights,
-                    todayHistoryData,      // <-- ADD THIS
-                    yesterdayHistoryData   // <-- ADD THIS
-                ];
-
-                if (isSocialOrInsights) {
-                    if (isFirstUser) {
-                         processArgs = [null, null, null, ...allContextArgs];
-                    } else {
-                        continue; // Only run once
-                    }
-                } else if (isHistoricalCalc) {
-                    const pYesterday = yesterdayPortfolios[uid];
-                    if (!pYesterday) continue;
-                    processArgs = [p, pYesterday, uid, ...allContextArgs];
-                } else {
-                    processArgs = [p, null, uid, ...allContextArgs];
-                }
-                
-                if (!isSocialOrInsights) {
-                    if ((userType === 'normal' && isSpeculatorCalc) ||
-                        (userType === 'speculator' && !isSpeculatorCalc && calcName !== 'users-processed')) {
-                        continue; // Skip: wrong user type
-                    }
-                }
-
-                try {
-                    await Promise.resolve(calc.process(...processArgs));
-                } catch (e) {
-                    logger.log('WARN', `Process error in ${calcName} for user ${uid}`, { err: e.message });
-                }
-            }
-             isFirstUser = false;
-        }
     }
-    
-    // Handle case where there are no users but we still need to run insights/social calcs
-    if (todayRefs.length === 0 && isFirstUser) {
-        logger.log('INFO', `[${passName}] No user portfolios found for ${dateStr}. Running insights/social calcs once.`);
-        
-        // --- MODIFIED: Add null history data ---
-        const allContextArgs = [ 
-            context, 
-            todayInsights, 
-            yesterdayInsights, 
-            todaySocialPostInsights, 
-            yesterdaySocialPostInsights,
-            todayHistoryData,      // <-- ADD THIS
-            yesterdayHistoryData   // <-- ADD THIS
-        ];
-        
-        for (const calcName in state) {
-            const calc = state[calcName];
-            if (!calc || typeof calc.process !== 'function') continue;
-            const manifestCalc = calc.manifest;
-            const isSocialOrInsights = manifestCalc.category === 'socialPosts' || manifestCalc.category === 'insights';
-            if (isSocialOrInsights) {
-                try {
-                    await Promise.resolve(calc.process(null, null, null, ...allContextArgs));
-                } catch (e) {
-                    logger.log('WARN', `Process error in ${calcName} for no-user run`, { err: e.message });
-                }
-            }
-        }
-    }
-}
 
-
-/**
- * Internal sub-pipe: Runs "standard" computations (Pass 1) for a single date.
- */
-async function runUnifiedComputation(dateToProcess, calculationsToRun, passName, config, dependencies, rootData) {
-    const { db, logger } = dependencies;
-    const dateStr = dateToProcess.toISOString().slice(0, 10);
-    logger.log('INFO', `[${passName}] Starting run for ${dateStr} with ${calculationsToRun.length} calcs.`);
-
-    try {
-        // --- NEW: Get history root data from the orchestrator ---
-        const {
-            portfolioRefs: todayRefs,
-            insightsData: todayInsightsData,
-            socialData: todaySocialPostInsightsData,
-            historyRefs: todayHistoryRefs // <-- ADD THIS
-        } = rootData;
-        
-        let yesterdayPortfolios = {};
-        let yesterdayInsightsData = null; 
-        let yesterdaySocialPostInsightsData = null; 
-        let todayHistoryData = null; // <-- ADD THIS
-        let yesterdayHistoryData = null; // <-- ADD THIS
-        
-        // --- MODIFIED: Add checks for history data ---
-        const requiresYesterdayPortfolio = calculationsToRun.some(c => c.isHistorical === true);
-        const requiresYesterdayInsights = calculationsToRun.some(c => c.class.prototype.process.toString().includes('yesterdayInsights'));
-        const requiresYesterdaySocialPosts = calculationsToRun.some(c => c.class.prototype.process.toString().includes('yesterdaySocialPostInsights'));
-        const requiresHistory = calculationsToRun.some(c => c.rootDataDependencies.includes('history'));
-        const requiresYesterdayHistory = calculationsToRun.some(c => c.isHistorical === true && c.class.prototype.process.toString().includes('yesterdayHistoryData'));
-        // --- END MODIFICATION ---
-
-
-        // --- (This block is now much larger, handling all "yesterday" and "history" data) ---
-        if (requiresYesterdayPortfolio || requiresYesterdayInsights || requiresYesterdaySocialPosts || requiresHistory || requiresYesterdayHistory) { 
-            
-             if(requiresYesterdayInsights) {
-                let daysAgo = 1; const maxLookback = 30;
-                while (!yesterdayInsightsData && daysAgo <= maxLookback) {
-                    const prev = new Date(dateToProcess); prev.setUTCDate(prev.getUTCDate() - daysAgo);
-                    const prevStr = prev.toISOString().slice(0, 10);
-                    yesterdayInsightsData = await loadDailyInsights(config, dependencies, prevStr);
-                    if (yesterdayInsightsData) logger.log('INFO', `[${passName}] Found 'yesterday' instrument insights data from ${daysAgo} day(s) ago (${prevStr}).`);
-                    else daysAgo++;
-                }
-                if (!yesterdayInsightsData) logger.log('WARN', `[${passName}] Could not find any 'yesterday' instrument insights data within a ${maxLookback} day lookback.`);
-            }
-            if(requiresYesterdaySocialPosts) {
-                let daysAgo = 1; const maxLookback = 30; 
-                while (!yesterdaySocialPostInsightsData && daysAgo <= maxLookback) {
-                    const prev = new Date(dateToProcess); prev.setUTCDate(prev.getUTCDate() - daysAgo);
-                    const prevStr = prev.toISOString().slice(0, 10);
-                    yesterdaySocialPostInsightsData = await loadDailySocialPostInsights(config, dependencies, prevStr);
-                    if (yesterdaySocialPostInsightsData) logger.log('INFO', `[${passName}] Found 'yesterday' social post insights data from ${daysAgo} day(s) ago (${prevStr}).`);
-                    else daysAgo++;
-                }
-                if (!yesterdaySocialPostInsightsData) logger.log('WARN', `[${passName}] Could not find any 'yesterday' social post insights data within a ${maxLookback} day lookback.`);
-            }
-            if (requiresYesterdayPortfolio) {
-                const prev = new Date(dateToProcess); prev.setUTCDate(prev.getUTCDate() - 1);
-                const prevStr = prev.toISOString().slice(0, 10);
-                const yesterdayRefs = await getPortfolioPartRefs(config, dependencies, prevStr);
-                if (yesterdayRefs.length > 0) {
-                    yesterdayPortfolios = await loadFullDayMap(config, dependencies, yesterdayRefs);
-                     logger.log('INFO', `[${passName}] Loaded yesterday's (${prevStr}) portfolio map for historical calcs.`);
-                } else {
-                    logger.log('WARN', `[${passName}] Yesterday's (${prevStr}) portfolio data not found. Historical calcs requiring it will be skipped.`);
-                }
-            }
-
-            // --- NEW: Add logic to load Today's History Data (if needed) ---
-            if (requiresHistory && todayHistoryRefs.length > 0) {
-                 logger.log('INFO', `[${passName}] Loading today's (${dateStr}) history map...`);
-                 todayHistoryData = await loadFullDayMap(config, dependencies, todayHistoryRefs);
-                 logger.log('INFO', `[${passName}] Loaded today's history map with ${Object.keys(todayHistoryData).length} users.`);
-            }
-            
-            // --- NEW: Add logic to load Yesterday's History Data (if needed) ---
-            if (requiresYesterdayHistory) {
-                const prev = new Date(dateToProcess);
-                prev.setUTCDate(prev.getUTCDate() - 1);
-                const prevStr = prev.toISOString().slice(0, 10);
-                const yesterdayHistoryRefs = await getHistoryPartRefs(config, dependencies, prevStr);
-                if (yesterdayHistoryRefs.length > 0) {
-                    yesterdayHistoryData = await loadFullDayMap(config, dependencies, yesterdayHistoryRefs);
-                     logger.log('INFO', `[${passName}] Loaded yesterday's (${prevStr}) history map for historical calcs.`);
-                } else {
-                    logger.log('WARN', `[${passName}] Yesterday's (${prevStr}) history data not found. Historical calcs requiring it will be skipped.`);
-                }
-            }
-        }
-        
-        const state = initializeCalculators(calculationsToRun, logger);
-        
-        // --- MODIFIED: Pass new history data ---
-        await streamAndProcess(
-            dateStr, 
-            todayRefs, 
-            state, 
-            passName, 
-            config, 
-            dependencies, 
-            yesterdayPortfolios, 
-            todayInsightsData, 
-            yesterdayInsightsData,
-            todaySocialPostInsightsData,
-            yesterdaySocialPostInsightsData,
-            todayHistoryData,         // <-- ADD THIS
-            yesterdayHistoryData      // <-- ADD THIS
-        );
-
-        let successCount = 0;
-        const resultsCollectionRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection);
-
-        for (const calcName in state) {
-            const calc = state[calcName];
-            if (!calc || typeof calc.getResult !== 'function') {
-                 if (!calc) logger.log('WARN', `[${passName}] Skipping ${calcName} for ${dateStr} because it failed to initialize.`);
-                 continue;
-            }
-
-            const category = calc.manifest.category || 'unknown';
-            let result = null;
-            try {
-                result = await Promise.resolve(calc.getResult());
-                
-                // --- Database write logic (Identical to original file) ---
-                const pendingWrites = [];
-                const summaryData = {};
-                if (result && Object.keys(result).length > 0) {
-                    let isSharded = false;
-                    const shardedCollections = {
-                        'sharded_user_profile': config.shardedUserProfileCollection,
-                        'sharded_user_profitability': config.shardedProfitabilityCollection
-                    };
-                    for (const resultKey in shardedCollections) {
-                        if (result[resultKey]) {
-                            isSharded = true;
-                            const shardCollectionName = shardedCollections[resultKey];
-                            const shardedData = result[resultKey];
-                            for (const shardId in shardedData) {
-                                const shardDocData = shardedData[shardId];
-                                if (shardDocData && Object.keys(shardDocData).length > 0) {
-                                    const shardRef = db.collection(shardCollectionName).doc(shardId);
-                                    pendingWrites.push({ ref: shardRef, data: shardedData[shardId] });
-                                }
-                            }
-                            const { [resultKey]: _, ...otherResults } = result;
-                            if (Object.keys(otherResults).length > 0) {
-                                const computationDocRef = resultsCollectionRef.doc(category)
-                                    .collection(config.computationsSubcollection)
-                                    .doc(calcName);
-                                pendingWrites.push({ ref: computationDocRef, data: otherResults });
-                            }
-                        }
-                    }
-                    if (!isSharded) {
-                        const computationDocRef = resultsCollectionRef.doc(category)
-                                                          .collection(config.computationsSubcollection)
-                                                          .doc(calcName);
-                        pendingWrites.push({ ref: computationDocRef, data: result });
-                    }
-                    if (!summaryData[category]) summaryData[category] = {};
-                    summaryData[category][calcName] = true;
-                     if (Object.keys(summaryData).length > 0) {
-                        const topLevelDocRef = db.collection(config.resultsCollection).doc(dateStr);
-                        pendingWrites.push({ ref: topLevelDocRef, data: summaryData });
-                     }
-                    if (pendingWrites.length > 0) {
-                        await commitBatchInChunks(config, dependencies, pendingWrites, `Commit ${passName} ${dateStr} [${calcName}]`);
-                        successCount++;
-                    }
-                } else {
-                     if (result === null) logger.log('INFO', `[${passName}] Calculation ${calcName} returned null for ${dateStr}. This is expected if no data was processed.`);
-                     else logger.log('WARN', `[${passName}] Calculation ${calcName} produced empty results {} for ${dateStr}. Skipping write.`);
-                }
-            } catch (e) {
-                logger.log('ERROR', `[${passName}] getResult/Commit failed for ${calcName} on ${dateStr}`, { err: e.message, stack: e.stack });
-            }
-        }
-        const completionStatus = successCount === calculationsToRun.length ? 'SUCCESS' : 'WARN';
-        logger.log(completionStatus, `[${passName}] Completed ${dateStr}. Success: ${successCount}/${calculationsToRun.length}.`);
-
-    } catch (err) {
-        logger.log('ERROR', `[${passName}] Fatal error for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
-        throw err; // Re-throw to stop the orchestrator for this day
-    }
-}
-
-
-/**
- * Internal sub-pipe: Runs "meta" computations (Pass 2, 3, 4) for a single date.
- */
-async function runMetaComputation(
-    dateToProcess, 
-    calculationsToRun, // This is the *filtered* list
-    passName, 
-    config, 
-    dependencies, 
-    fetchedDependencies, // <-- This is the NEW object from Firestore
-    rootData
-) {
-    const { db, logger } = dependencies;
-    const dateStr = dateToProcess.toISOString().slice(0, 10);
-    logger.log('INFO', `[${passName}] Starting run for ${dateStr} with ${calculationsToRun.length} calcs.`);
-    
-    try {
-        const resultsCollectionRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection);
-        const compsSub = config.computationsSubcollection || 'computations';
-        let successCount = 0;
-        
-        // Add rootData to dependencies for meta-calcs that need to stream users
-        const dependenciesForMetaCalc = {
-            ...dependencies,
-            rootData: rootData
-        };
-
-        for (const manifestCalc of calculationsToRun) {
-            const calcName = normalizeName(manifestCalc.name);
-            const category = manifestCalc.category || 'unknown';
-            const CalcClass = manifestCalc.class;
-            
-            if (typeof CalcClass !== 'function') {
-                 logger.log('ERROR', `[${passName}] Invalid class in manifest for ${calcName}. Skipping.`);
-                 continue;
-            }
-            
-            const instance = new CalcClass();
-            let result = null;
-            try {
-                // --- Call process with the fetched dependencies ---
-                result = await Promise.resolve(instance.process(
-                    dateStr, 
-                    dependenciesForMetaCalc,
-                    config, 
-                    fetchedDependencies // <-- Pass the pre-fetched results
-                ));
-                
-                // --- Database write logic (Identical to original file) ---
-                const pendingWrites = [];
-                const summaryData = {};
-                if (result && Object.keys(result).length > 0) {
-                    let isSharded = false;
-                    const shardedCollections = {
-                        'sharded_user_profile': config.shardedUserProfileCollection,
-                        'sharded_user_profitability': config.shardedProfitabilityCollection 
-                    };
-                    for (const resultKey in shardedCollections) {
-                        if (result[resultKey]) {
-                            isSharded = true;
-                            const shardCollectionName = shardedCollections[resultKey];
-                            if (!shardCollectionName) {
-                                logger.log('ERROR', `[${passName}] Missing config key for sharded collection: ${resultKey}`);
-                                continue;
-                            }
-                            const shardedData = result[resultKey];
-                            for (const shardId in shardedData) {
-                                const shardDocData = shardedData[shardId];
-                                if (shardDocData && (Object.keys(shardDocData).length > 0)) {
-                                    const shardRef = db.collection(shardCollectionName).doc(shardId);
-                                    pendingWrites.push({ ref: shardRef, data: shardDocData, merge: true });
-                                }
-                            }
-                            const { [resultKey]: _, ...otherResults } = result;
-                            result = otherResults;
-                        }
-                    }
-                    if (result && Object.keys(result).length > 0) {
-                        const computationDocRef = resultsCollectionRef.doc(category)
-                                                          .collection(compsSub)
-                                                          .doc(calcName);
-                        pendingWrites.push({ ref: computationDocRef, data: result });
-                    }
-                    if (!summaryData[category]) summaryData[category] = {};
-                    summaryData[category][calcName] = true;
-                     if (Object.keys(summaryData).length > 0) {
-                        const topLevelDocRef = db.collection(config.resultsCollection).doc(dateStr);
-                        pendingWrites.push({ ref: topLevelDocRef, data: summaryData });
-                     }
-                    if (pendingWrites.length > 0) {
-                        await commitBatchInChunks(
-                            config,
-                            dependencies,
-                            pendingWrites,
-                            `Commit ${passName} ${dateStr} [${calcName}]`
-                        );
-                        successCount++;
-                    }
-                } else {
-                     logger.log('WARN', `[${passName}] Meta-calculation ${calcName} produced no results for ${dateStr}. Skipping write.`);
-                }
-            } catch (e) {
-                logger.log('ERROR', `[${passName}] Meta-calc process/commit failed for ${calcName} on ${dateStr}`, { err: e.message, stack: e.stack });
-            }
-        }
-
-        const completionStatus = successCount === calculationsToRun.length ? 'SUCCESS' : 'WARN';
-        logger.log(completionStatus, `[${passName}] Completed ${dateStr}. Success: ${successCount}/${calculationsToRun.length}.`);
-        
-    } catch (err) {
-        logger.log('ERROR', `[${passName}] Fatal error for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
-        throw err; 
-    }
+    logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 
-
-module.exports = {
-    runComputationPass,
-};
+module.exports = { runComputationPass };