bulltrackers-module 1.0.105 → 1.0.106

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,114 +1,648 @@
-/**
- * @fileoverview Main pipe: pipe.computationSystem.runOrchestration
- *
- * This function is the "Scheduler" for the event-driven DAG.
- * It is triggered by a Cloud Scheduler (e.g., at 01:00 UTC).
- *
- * Its ONLY responsibility is to:
- * 1. Find all dates that are new or incomplete (using the StateManager).
- * 2. For each of those dates, initialize a new state document.
- * 3. Publish a "node task" for each "root node" (nodes with no dependencies).
- */
-
-const { FieldPath } = require('@google-cloud/firestore');
-// Import the new graph and state manager
-const { ComputationGraph } = require('../utils/dependency_graph.js');
-const stateManager = require('../utils/computation_state_manager.js');
-// Import existing utils for finding dates
-const { getExpectedDateStrings, getFirstDateFromSourceData } = require('../utils/utils.js');
-
-/**
- * Main pipe: pipe.computationSystem.runOrchestration
- * @param {object} config - The computation system configuration object.
- * @param {object} dependencies - Contains db, logger, calculationUtils, pubsubUtils.
- * @param {object} calculations - The injected calculations object from 'aiden-shared-calculations-unified'.
- * @param {object} dependencyConfig - The loaded computation_dependencies.json object.
- * @returns {Promise<Object>} Summary of the orchestration kickoff.
- */
-async function runComputationOrchestrator(config, dependencies, calculations, dependencyConfig) {
-    const { logger, db, calculationUtils, pubsubUtils } = dependencies;
-
-    logger.log('INFO', '🚀 [DAG Orchestrator] Starting scheduled run...');
-
-    // 1. Build the computation dependency graph
-    // The graph is now passed in dependencies, initialized in index.js
-    const graph = dependencies.computationGraph;
-    if (!graph || !graph.isBuilt) {
-        logger.log('ERROR', '[DAG Orchestrator] ComputationGraph not built or provided. Halting.');
-        return { success: false, error: "ComputationGraph not available." };
-    }
-    
-    const allNodes = graph.getAllNodes();
-    const rootNodes = graph.getRootNodes();
-
-    if (rootNodes.length === 0) {
-        logger.log('ERROR', '[DAG Orchestrator] No root nodes found in graph. Halting.');
-        return { success: false, error: "No root nodes (Batch 1) found." };
-    }
-
-    // 2. Find date range to check
-    const yesterday = new Date();
-    yesterday.setUTCDate(yesterday.getUTCDate() - 1);
-    const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
-
-    const firstDate = await getFirstDateFromSourceData(config, dependencies);
-    const startDateUTC = firstDate
-       ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate()))
-       : new Date(config.earliestComputationDate + 'T00:00:00Z');
-
-    const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
-    if (allExpectedDates.length === 0) {
-         logger.log('WARN', '[DAG Orchestrator] No dates found in range to process.');
-         return { success: true, message: "No dates in range." };
-    }
-
-    // 3. Find all new or incomplete dates using the StateManager
-    const datesToProcess = await stateManager.getIncompleteDates(dependencies, allExpectedDates);
-
-    if (datesToProcess.length === 0) {
-        logger.log('SUCCESS', '[DAG Orchestrator] All dates are up-to-date. Nothing to do.');
-        return { success: true, datesKickedOff: 0, message: "All dates up-to-date." };
-    }
-
-    logger.log('INFO', `[DAG Orchestrator] Found ${datesToProcess.length} dates needing processing.`);
-
-    // 4. Create and publish root node tasks for each date
-    const tasks = [];
-    for (const date of datesToProcess) {
-        // 4a. Initialize the state document for this date
-        await stateManager.initializeState(dependencies, date, allNodes);
-
-        // 4b. Create tasks for all root nodes for this date
-        for (const rootNode of rootNodes) {
-            tasks.push({
-                date: date,
-                nodeName: rootNode.name,
-                isRootNode: true // Add flag for clarity
-            });
-        }
-    }
-
-    const topicName = config.computeNodeTopicName; // Use a new topic name
-    if (!topicName) {
-       logger.log('ERROR', '[DAG Orchestrator] `computeNodeTopicName` is not defined in config. Cannot publish tasks.');
-       return { success: false, error: "Config missing 'computeNodeTopicName'." };
-    }
-
-    await pubsubUtils.batchPublishTasks(dependencies, {
-        topicName: topicName,
-        tasks: tasks,
-        taskType: 'Compute Root Node Kickoff'
-    });
-
-    logger.log('SUCCESS', `[DAG Orchestrator] Published ${tasks.length} root node tasks to ${topicName} (for ${datesToProcess.length} dates).`);
-    
-    return {
-        orchestrator: "Event-Driven DAG",
-        datesKickedOff: datesToProcess.length,
-        rootNodeTasks: tasks.length
-    };
-}
-
-module.exports = {
-    runComputationOrchestrator,
+/**
+ * @fileoverview Main pipe: pipe.computationSystem.runOrchestration
+ * REFACTORED: Now stateless and receives dependencies.
+ * All internal helpers now receive (config, dependencies) as well.
+ * UPDATED: Reads a static manifest to determine execution order and
+ * passes computed dependencies in-memory.
+ * NEW: Checks for root data (portfolios, insights, social)
+ * before processing any calculations for a given day.
+ */
+
+const { FieldPath } = require('@google-cloud/firestore');
+// Import sub-pipes/utils from their new locations
+const { 
+    getPortfolioPartRefs, 
+    loadFullDayMap, 
+    loadDataByRefs, 
+    loadDailyInsights,
+    loadDailySocialPostInsights
+} = require('../utils/data_loader.js');
+
+const {
+    normalizeName,
+    getExpectedDateStrings, 
+    getFirstDateFromSourceData,
+    commitBatchInChunks
+} = require('../utils/utils.js');
+
+
+/**
+ * Groups the manifest by pass number.
+ * @param {Array<object>} manifest - The computation manifest.
+ * @returns {object} { '1': [...], '2': [...] }
+ */
+function groupByPass(manifest) {
+    return manifest.reduce((acc, calc) => {
+        (acc[calc.pass] = acc[calc.pass] || []).push(calc);
+        return acc;
+    }, {});
+}
+
+/**
+ * --- NEW HELPER ---
+ * Checks if the root data (portfolios, insights, social) exists for a given day.
+ * @param {string} dateStr - The date string to check (YYYY-MM-DD).
+ * @param {object} config - The computation system configuration object.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
+ * @returns {Promise<object>} { portfolioRefs, insightsData, socialData, isAvailable }
+ */
+async function checkRootDataAvailability(dateStr, config, dependencies) {
+    const { logger } = dependencies;
+    logger.log('INFO', `[Orchestrator] Checking root data availability for ${dateStr}...`);
+
+    try {
+        const [portfolioRefs, insightsData, socialData] = await Promise.all([
+            getPortfolioPartRefs(config, dependencies, dateStr),
+            loadDailyInsights(config, dependencies, dateStr),
+            loadDailySocialPostInsights(config, dependencies, dateStr)
+        ]);
+
+        const isAvailable = (portfolioRefs && portfolioRefs.length > 0) || !!insightsData || !!socialData;
+
+        if (isAvailable) {
+            logger.log('INFO', `[Orchestrator] Root data found for ${dateStr}. (Portfolio parts: ${portfolioRefs.length}, Insights: ${!!insightsData}, Social: ${!!socialData})`);
+        }
+
+        return {
+            portfolioRefs: portfolioRefs || [],
+            insightsData: insightsData || null,
+            socialData: socialData || null,
+            isAvailable: isAvailable
+        };
+
+    } catch (err) {
+        logger.log('ERROR', `[Orchestrator] Error checking data availability for ${dateStr}`, { errorMessage: err.message });
+        return { portfolioRefs: [], insightsData: null, socialData: null, isAvailable: false };
+    }
+}
+
+
+/**
+ * Main pipe: pipe.computationSystem.runOrchestration
+ * @param {object} config - The computation system configuration object.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
+ * @param {Array<object>} computationManifest - The injected computation manifest.
+ * @returns {Promise<Object>} Summary of all passes.
+ */
+async function runComputationOrchestrator(config, dependencies, computationManifest) {
+    const { logger } = dependencies;
+    const summary = {};
+
+    const yesterday = new Date();
+    yesterday.setUTCDate(yesterday.getUTCDate() - 1);
+    const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
+
+    // Pass dependencies to sub-pipe
+    const firstDate = await getFirstDateFromSourceData(config, dependencies);
+    const startDateUTC = firstDate
+       ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate()))
+       : new Date(config.earliestComputationDate + 'T00:00:00Z');
+
+    const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
+
+    // --- Group the manifest by pass number ---
+    const passes = groupByPass(computationManifest);
+    const passNumbers = Object.keys(passes).sort((a, b) => a - b);
+    
+    // --- Process ONE DAY at a time, in order ---
+    for (const dateStr of allExpectedDates) {
+        const dateToProcess = new Date(dateStr + 'T00:00:00Z');
+        
+        // --- NEW: Check for root data *before* processing the day ---
+        const rootData = await checkRootDataAvailability(dateStr, config, dependencies);
+        if (!rootData.isAvailable) {
+            logger.log('WARN', `[Orchestrator] Skipping all computations for ${dateStr} due to missing root data (no portfolios, insights, or social data found).`);
+            continue; // Skip to the next day
+        }
+        // --- END NEW CHECK ---
+
+        logger.log('INFO', `[Orchestrator] Processing all passes for ${dateStr}...`);
+        
+        // This cache will hold results *for this day only*
+        const dailyResultsCache = new Map();
+        let passSuccess = true;
+
+        for (const passNum of passNumbers) {
+            if (!passSuccess) {
+                logger.log('WARN', `[Orchestrator] Skipping Pass ${passNum} for ${dateStr} due to previous pass failure.`);
+                break; // Skip subsequent passes if a previous one failed
+            }
+            
+            const calcsInPass = passes[passNum] || [];
+            
+            const standardCalcs = calcsInPass.filter(c => c.type === 'standard');
+            const metaCalcs = calcsInPass.filter(c => c.type === 'meta');
+            
+            logger.log('INFO', `[Orchestrator] Starting Pass ${passNum} for ${dateStr} (${standardCalcs.length} standard, ${metaCalcs.length} meta).`);
+
+            try {
+                // 1. Run standard calcs for this pass
+                if (standardCalcs.length > 0) {
+                    const standardResults = await runUnifiedComputation(
+                        dateToProcess,
+                        standardCalcs, // Pass the manifest objects
+                        `Pass ${passNum} (Standard)`,
+                        config,
+                        dependencies,
+                        rootData // <-- Pass pre-fetched root data
+                    );
+                    
+                    // Add results to cache
+                    for (const [calcName, result] of Object.entries(standardResults)) {
+                        dailyResultsCache.set(calcName, result);
+                    }
+                }
+
+                // 2. Run meta calcs for this pass
+                if (metaCalcs.length > 0) {
+                    const metaResults = await runMetaComputation(
+                        dateToProcess,
+                        metaCalcs, // Pass the manifest objects
+                        `Pass ${passNum} (Meta)`,
+                        config,
+                        dependencies,
+                        dailyResultsCache // <-- PASS THE CACHE
+                    );
+
+                    // Add results to cache
+                    for (const [calcName, result] of Object.entries(metaResults)) {
+                        dailyResultsCache.set(calcName, result);
+                    }
+                }
+                 logger.log('SUCCESS', `[Orchestrator] Completed Pass ${passNum} for ${dateStr}.`);
+            } catch (err) {
+                logger.log('ERROR', `[Orchestrator] FAILED Pass ${passNum} for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
+                passSuccess = false;
+            }
+        } // End passes loop
+        logger.log('INFO', `[Orchestrator] Finished processing for ${dateStr}.`);
+    } // End dates loop
+
+    logger.log('INFO', '[Orchestrator] Computation orchestration finished.');
+    return summary;
+}
+
+/**
+ * Internal sub-pipe: Initializes calculator instances.
+ * --- MODIFIED: Attaches the manifest entry to the instance. ---
+ */
+function initializeCalculators(calculationsToRun, logger) {
+    const state = {};
+    for (const calcManifest of calculationsToRun) {
+        const calcName = normalizeName(calcManifest.name);
+        const CalculationClass = calcManifest.class;
+        
+        if (typeof CalculationClass === 'function') {
+            try {
+                const instance = new CalculationClass();
+                instance.manifest = calcManifest; // <-- Attach manifest data
+                state[calcName] = instance;
+            } catch (e) {
+                logger.warn(`[Orchestrator] Init failed for ${calcName}`, { errorMessage: e.message });
+                state[calcName] = null;
+            }
+        } else {
+            logger.warn(`[Orchestrator] Calculation class not found in manifest for: ${calcName}`);
+            state[calcName] = null;
+        }
+    }
+    return state;
+}
+
+/**
+ * Internal sub-pipe: Streams data and calls process() on calculators.
+ * --- MODIFIED: Uses manifest flags for logic. ---
+ */
+async function streamAndProcess(
+    dateStr, todayRefs, state, passName, config, dependencies, 
+    yesterdayPortfolios = {}, 
+    todayInsights = null, 
+    yesterdayInsights = null,
+    todaySocialPostInsights = null,
+    yesterdaySocialPostInsights = null
+) {
+    const { logger, calculationUtils } = dependencies;
+    logger.log('INFO', `[${passName}] Streaming ${todayRefs.length} 'today' part docs for ${dateStr}...`);
+
+    const yesterdayDate = new Date(dateStr + 'T00:00:00Z');
+    yesterdayDate.setUTCDate(yesterdayDate.getUTCDate() - 1);
+    const yesterdayStr = yesterdayDate.toISOString().slice(0, 10);
+
+    const { instrumentToTicker, instrumentToSector } = await calculationUtils.loadInstrumentMappings();
+
+    const context = { 
+        instrumentMappings: instrumentToTicker, 
+        sectorMapping: instrumentToSector,
+        todayDateStr: dateStr,
+        yesterdayDateStr: yesterdayStr,
+        dependencies: dependencies,
+        config: config
+    };
+
+    const batchSize = config.partRefBatchSize || 10;
+    let isFirstUser = true;
+
+    for (let i = 0; i < todayRefs.length; i += batchSize) {
+        const batchRefs = todayRefs.slice(i, i + batchSize);
+        const todayPortfoliosChunk = await loadDataByRefs(config, dependencies, batchRefs);
+
+        for (const uid in todayPortfoliosChunk) {
+            const p = todayPortfoliosChunk[uid];
+            if (!p) continue;
+
+            const userType = p.PublicPositions ? 'speculator' : 'normal';
+
+            for (const calcName in state) { // calcName is already normalized
+                const calc = state[calcName];
+                if (!calc || typeof calc.process !== 'function') continue;
+                
+                // --- NEW ROBUST LOGIC ---
+                const manifestCalc = calc.manifest;
+                const isSocialOrInsights = manifestCalc.category === 'socialPosts' || manifestCalc.category === 'insights';
+                const isHistoricalCalc = manifestCalc.isHistorical === true;
+                const isSpeculatorCalc = manifestCalc.category === 'speculators';
+                // --- END NEW LOGIC ---
+
+                let processArgs;
+                const allContextArgs = [
+                    context, 
+                    todayInsights, 
+                    yesterdayInsights, 
+                    todaySocialPostInsights,
+                    yesterdaySocialPostInsights
+                ];
+
+                if (isSocialOrInsights) {
+                    if (isFirstUser) {
+                         processArgs = [null, null, null, ...allContextArgs];
+                    } else {
+                        continue; // Only run once for the first "user"
+                    }
+                } else if (isHistoricalCalc) { // Assumes historical
+                    const pYesterday = yesterdayPortfolios[uid];
+                    if (!pYesterday) {
+                         continue; // Skip if no yesterday data
+                    }
+                    processArgs = [p, pYesterday, uid, ...allContextArgs];
+                } else {
+                    // Standard daily calculation
+                    processArgs = [p, null, uid, ...allContextArgs];
+                }
+                
+                // --- NEW ROBUST CHECK for user type ---
+                if (!isSocialOrInsights) {
+                    if ((userType === 'normal' && isSpeculatorCalc) ||
+                        (userType === 'speculator' && !isSpeculatorCalc && calcName !== 'users-processed')) {
+                        continue; // Skip: wrong user type for this calc
+                    }
+                }
+                // --- END NEW CHECK ---
+
+                try {
+                    await Promise.resolve(calc.process(...processArgs));
+                } catch (e) {
+                    logger.log('WARN', `Process error in ${calcName} for user ${uid}`, { err: e.message });
+                }
+            }
+             isFirstUser = false;
+        }
+    }
+    
+    // Handle case where there are no users but we still need to run insights/social calcs
+    if (todayRefs.length === 0 && isFirstUser) {
+        logger.log('INFO', `[${passName}] No user portfolios found for ${dateStr}. Running insights/social calcs once.`);
+        const allContextArgs = [
+            context, 
+            todayInsights, 
+            yesterdayInsights, 
+            todaySocialPostInsights,
+            yesterdaySocialPostInsights
+        ];
+
+        for (const calcName in state) {
+            const calc = state[calcName];
+            if (!calc || typeof calc.process !== 'function') continue;
+            
+            const manifestCalc = calc.manifest;
+            const isSocialOrInsights = manifestCalc.category === 'socialPosts' || manifestCalc.category === 'insights';
+
+            if (isSocialOrInsights) {
+                try {
+                    await Promise.resolve(calc.process(null, null, null, ...allContextArgs));
+                } catch (e) {
+                    logger.log('WARN', `Process error in ${calcName} for no-user run`, { err: e.message });
+                }
+            }
+        }
+    }
+}
+
+
+/**
+ * Internal sub-pipe: Runs "standard" computations for a single date.
+ * MODIFIED: Accepts pre-fetched rootData.
+ * MODIFIED: Returns a map of results for the in-memory cache.
+ */
+async function runUnifiedComputation(dateToProcess, calculationsToRun, passName, config, dependencies, rootData) {
+    const { db, logger } = dependencies;
+    const dateStr = dateToProcess.toISOString().slice(0, 10);
+    logger.log('INFO', `[${passName}] Starting run for ${dateStr} with ${calculationsToRun.length} calcs.`);
+
+    // This map will store the final results to be returned
+    const passResults = {};
+
+    try {
+        // --- NEW: Get root data from the orchestrator ---
+        const {
+            portfolioRefs: todayRefs,
+            insightsData: todayInsightsData,
+            socialData: todaySocialPostInsightsData
+        } = rootData;
+        // --- END NEW ---
+        
+        // (The check for data availability is now done in the orchestrator)
+
+        let yesterdayPortfolios = {};
+        let yesterdayInsightsData = null; 
+        let yesterdaySocialPostInsightsData = null; 
+        
+        // Check if any calc needs yesterday's data
+        // --- MODIFIED: Use the manifest entry ---
+        const requiresYesterdayPortfolio = calculationsToRun.some(c => c.isHistorical === true);
+        const requiresYesterdayInsights = calculationsToRun.some(c => c.class.prototype.process.toString().includes('yesterdayInsights'));
+        const requiresYesterdaySocialPosts = calculationsToRun.some(c => c.class.prototype.process.toString().includes('yesterdaySocialPostInsights'));
+
+        // --- FULL "YESTERDAY" LOGIC ---
+        if (requiresYesterdayPortfolio || requiresYesterdayInsights || requiresYesterdaySocialPosts) { 
+            
+            if(requiresYesterdayInsights) {
+                let daysAgo = 1;
+                const maxLookback = 30;
+                while (!yesterdayInsightsData && daysAgo <= maxLookback) {
+                    const prev = new Date(dateToProcess);
+                    prev.setUTCDate(prev.getUTCDate() - daysAgo);
+                    const prevStr = prev.toISOString().slice(0, 10);
+                    yesterdayInsightsData = await loadDailyInsights(config, dependencies, prevStr);
+                    if (yesterdayInsightsData) {
+                        logger.log('INFO', `[${passName}] Found 'yesterday' instrument insights data from ${daysAgo} day(s) ago (${prevStr}).`);
+                    } else {
+                        daysAgo++;
+                    }
+                }
+                if (!yesterdayInsightsData) logger.log('WARN', `[${passName}] Could not find any 'yesterday' instrument insights data within a ${maxLookback} day lookback.`);
+            }
+            
+            if(requiresYesterdaySocialPosts) {
+                let daysAgo = 1;
+                const maxLookback = 30; 
+                while (!yesterdaySocialPostInsightsData && daysAgo <= maxLookback) {
+                    const prev = new Date(dateToProcess);
+                    prev.setUTCDate(prev.getUTCDate() - daysAgo);
+                    const prevStr = prev.toISOString().slice(0, 10);
+                    yesterdaySocialPostInsightsData = await loadDailySocialPostInsights(config, dependencies, prevStr);
+                    if (yesterdaySocialPostInsightsData) {
+                         logger.log('INFO', `[${passName}] Found 'yesterday' social post insights data from ${daysAgo} day(s) ago (${prevStr}).`);
+                    } else {
+                        daysAgo++;
+                    }
+                }
+                if (!yesterdaySocialPostInsightsData) logger.log('WARN', `[${passName}] Could not find any 'yesterday' social post insights data within a ${maxLookback} day lookback.`);
+            }
+
+            if (requiresYesterdayPortfolio) {
+                const prev = new Date(dateToProcess);
+                prev.setUTCDate(prev.getUTCDate() - 1);
+                const prevStr = prev.toISOString().slice(0, 10);
+                const yesterdayRefs = await getPortfolioPartRefs(config, dependencies, prevStr);
+                if (yesterdayRefs.length > 0) {
+                    yesterdayPortfolios = await loadFullDayMap(config, dependencies, yesterdayRefs);
+                     logger.log('INFO', `[${passName}] Loaded yesterday's (${prevStr}) portfolio map for historical calcs.`);
+                } else {
+                    logger.log('WARN', `[${passName}] Yesterday's (${prevStr}) portfolio data not found. Historical calcs requiring it will be skipped.`);
+                }
+            }
+        }
+        // --- END FULL "YESTERDAY" LOGIC ---
+        
+        const state = initializeCalculators(calculationsToRun, logger);
+        await streamAndProcess(
+            dateStr, 
+            todayRefs, 
+            state, 
+            passName, 
+            config, 
+            dependencies, 
+            yesterdayPortfolios, 
+            todayInsightsData, 
+            yesterdayInsightsData,
+            todaySocialPostInsightsData,
+            yesterdaySocialPostInsightsData
+        );
+
+        let successCount = 0;
+        const resultsCollectionRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection);
+
+        for (const calcName in state) { // calcName is already normalized
+            const calc = state[calcName];
+            if (!calc || typeof calc.getResult !== 'function') continue;
+
+            // --- MODIFIED: Get category from the attached manifest ---
+            const category = calc.manifest.category || 'unknown';
+
+            try {
+                const result = await Promise.resolve(calc.getResult());
+                
+                // Add to results map for in-memory cache
+                if (result) {
+                    passResults[calcName] = result;
+                }
+
+                const pendingWrites = [];
+                const summaryData = {};
+
+                if (result && Object.keys(result).length > 0) {
+                    // (Special handling for sharded calcs remains the same)
+                    let isSharded = false;
+                    const shardedCollections = {
+                        'sharded_user_profile': config.shardedUserProfileCollection,
+                        'sharded_user_profitability': config.shardedProfitabilityCollection
+                    };
+
+                    for (const resultKey in shardedCollections) {
+                        if (result[resultKey]) {
+                            isSharded = true;
+                            const shardCollectionName = shardedCollections[resultKey];
+                            const shardedData = result[resultKey];
+                            
+                            for (const shardId in shardedData) {
+                                const shardDocData = shardedData[shardId];
+                                if (shardDocData && Object.keys(shardDocData).length > 0) {
+                                    const shardRef = db.collection(shardCollectionName).doc(shardId);
+                                    pendingWrites.push({ ref: shardRef, data: shardedData[shardId] });
+                                }
+                            }
+                            // Get results *not* in the shard key
+                            const { [resultKey]: _, ...otherResults } = result;
+                            if (Object.keys(otherResults).length > 0) {
+                                const computationDocRef = resultsCollectionRef.doc(category)
+                                    .collection(config.computationsSubcollection)
+                                    .doc(calcName);
+                                pendingWrites.push({ ref: computationDocRef, data: otherResults });
+                            }
+                        }
+                    }
+                    
+                    if (!isSharded) {
+                        const computationDocRef = resultsCollectionRef.doc(category)
+                                                          .collection(config.computationsSubcollection)
+                                                          .doc(calcName);
+                        pendingWrites.push({ ref: computationDocRef, data: result });
+                    }
+                    // --- END SHARDED HANDLING ---
+
+                    if (!summaryData[category]) summaryData[category] = {};
+                    summaryData[category][calcName] = true;
+
+                     if (Object.keys(summaryData).length > 0) {
+                        const topLevelDocRef = db.collection(config.resultsCollection).doc(dateStr);
+                        pendingWrites.push({ ref: topLevelDocRef, data: summaryData });
+                     }
+
+                    if (pendingWrites.length > 0) {
+                        await commitBatchInChunks(
+                            config,
+                            dependencies,
+                            pendingWrites,
+                            `Commit ${passName} ${dateStr} [${calcName}]`
+                        );
+                        successCount++;
+                    }
+                } else {
+                     logger.log('WARN', `[${passName}] Calculation ${calcName} produced no results for ${dateStr}. Skipping write to allow backfill.`);
+                }
+            } catch (e) {
+                logger.log('ERROR', `[${passName}] getResult/Commit failed for ${calcName} on ${dateStr}`, { err: e.message });
+            }
+        }
+
+        const completionStatus = successCount === calculationsToRun.length ? 'SUCCESS' : 'WARN';
+        logger.log(completionStatus, `[${passName}] Completed ${dateStr}. Success: ${successCount}/${calculationsToRun.length}.`);
+        
+        return passResults; // --- RETURN THE RESULTS ---
+
+    } catch (err) {
+        logger.log('ERROR', `[${passName}] Fatal error for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
+        throw err; // Re-throw to stop the orchestrator for this day
+    }
+}
+
+
+/**
+ * Internal sub-pipe: Runs "meta" or "backtest" computations for a single date.
+ * MODIFIED: Accepts in-memory cache and passes dependencies to calcs.
+ * Returns a map of results.
+ */
+async function runMetaComputation(dateToProcess, calculationsToRun, passName, config, dependencies, dailyResultsCache) {
+    const { db, logger } = dependencies;
+    const dateStr = dateToProcess.toISOString().slice(0, 10);
+    logger.log('INFO', `[${passName}] Starting run for ${dateStr} with ${calculationsToRun.length} calcs.`);
+    
+    const passResults = {}; // Map to store and return results
+
+    try {
+        const resultsCollectionRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection);
+        let successCount = 0;
+
+        for (const manifestCalc of calculationsToRun) {
+            const calcName = normalizeName(manifestCalc.name);
+            // --- MODIFIED: Get category from manifest ---
+            const category = manifestCalc.category || 'unknown';
+            const CalcClass = manifestCalc.class;
+            
+            if (typeof CalcClass !== 'function') {
+                 logger.log('ERROR', `[${passName}] Invalid class in manifest for ${calcName}. Skipping.`);
+                 continue;
+            }
+            
+            const instance = new CalcClass();
+
+            try {
+                // --- Gather dependencies from the cache ---
+                const computedDependencies = {};
+                let missingDep = false;
+                if (manifestCalc.dependencies) {
+                    for (const depName of manifestCalc.dependencies) {
+                        const normalizedDepName = normalizeName(depName);
+                        if (!dailyResultsCache.has(normalizedDepName)) {
+                            logger.log('ERROR', `[${passName}] Missing required dependency "${normalizedDepName}" for calculation "${calcName}". This should not happen. Skipping calc.`);
+                            missingDep = true;
+                            break;
+                        }
+                        computedDependencies[normalizedDepName] = dailyResultsCache.get(normalizedDepName);
+                    }
+                }
+                if (missingDep) continue;
+
+                // --- Call process with the dependencies ---
+                const result = await Promise.resolve(instance.process(
+                    dateStr, 
+                    dependencies, 
+                    config, 
+                    computedDependencies // <-- PASS IN-MEMORY DEPS
+                ));
+                
+                // Add to results map
+                if (result) {
+                    passResults[calcName] = result;
+                }
+                
+                const pendingWrites = [];
+                const summaryData = {};
+
+                if (result && Object.keys(result).length > 0) {
+                    const computationDocRef = resultsCollectionRef.doc(category)
+                                                      .collection(config.computationsSubcollection)
+                                                      .doc(calcName);
+                    pendingWrites.push({ ref: computationDocRef, data: result });
+                    
+                    if (!summaryData[category]) summaryData[category] = {};
+                    summaryData[category][calcName] = true;
+
+                     if (Object.keys(summaryData).length > 0) {
+                        const topLevelDocRef = db.collection(config.resultsCollection).doc(dateStr);
+                        pendingWrites.push({ ref: topLevelDocRef, data: summaryData });
+                     }
+
+                    if (pendingWrites.length > 0) {
+                        await commitBatchInChunks(
+                            config,
+                            dependencies,
+                            pendingWrites,
+                            `Commit ${passName} ${dateStr} [${calcName}]`
+                        );
+                        successCount++;
+                    }
+                } else {
+                     logger.log('WARN', `[${passName}] Meta-calculation ${calcName} produced no results for ${dateStr}. Skipping write.`);
+                }
+            } catch (e) {
+                logger.log('ERROR', `[${passName}] Meta-calc process/commit failed for ${calcName} on ${dateStr}`, { err: e.message, stack: e.stack });
+                // Don't re-throw, allow other meta-calcs in the pass to run
+            }
+        }
+
+        const completionStatus = successCount === calculationsToRun.length ? 'SUCCESS' : 'WARN';
+        logger.log(completionStatus, `[${passName}] Completed ${dateStr}. Success: ${successCount}/${calculationsToRun.length}.`);
+        
+        return passResults; // --- RETURN THE RESULTS ---
+
+    } catch (err) {
+        logger.log('ERROR', `[${passName}] Fatal error for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
+        throw err; // Re-throw to stop the orchestrator for this day
+    }
+}
+
+
+module.exports = {
+    runComputationOrchestrator,
 };