bulltrackers-module 1.0.117 → 1.0.119

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -0,0 +1,729 @@
+/**
+ * @fileoverview
+ * Main "Pass Runner" for the V2 Computation System.
+ *
+ * This orchestrator is designed to be run by a separate Cloud Function for each "pass".
+ * It reads its pass number from the config and executes only those calculations.
+ *
+ * - Pass 1: Runs "standard" calcs that stream user data.
+ * - Pass 2+: Runs "meta" calcs, which it first supplies with dependencies
+ * by fetching the results of *previous* passes from Firestore.
+ */
+
+const { FieldPath } = require('@google-cloud/firestore');
+const { 
+    getPortfolioPartRefs, 
+    loadFullDayMap, 
+    loadDataByRefs, 
+    loadDailyInsights,
+    loadDailySocialPostInsights
+} = require('../utils/data_loader.js');
+
+const {
+    normalizeName,
+    getExpectedDateStrings, 
+    getFirstDateFromSourceData,
+    commitBatchInChunks
+} = require('../utils/utils.js');
+
+
+/**
+ * Groups the manifest by pass number.
+ * @param {Array<object>} manifest - The computation manifest.
+ * @returns {object} { '1': [...], '2': [...] }
+ */
+function groupByPass(manifest) {
+    return manifest.reduce((acc, calc) => {
+        (acc[calc.pass] = acc[calc.pass] || []).push(calc);
+        return acc;
+    }, {});
+}
+
+/**
+ * Checks if a calculation's root data dependencies are met.
+ * @param {object} calcManifest - The manifest entry for the calculation.
+ * @param {object} rootDataStatus - The status object from checkRootDataAvailability.
+ * @returns {boolean} True if dependencies are met, false otherwise.
+ */
+function checkRootDependencies(calcManifest, rootDataStatus) {
+    if (!calcManifest.rootDataDependencies || calcManifest.rootDataDependencies.length === 0) {
+        return true;
+    }
+    for (const dep of calcManifest.rootDataDependencies) {
+        if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) return false;
+        if (dep === 'insights' && !rootDataStatus.hasInsights) return false;
+        if (dep === 'social' && !rootDataStatus.hasSocial) return false;
+    }
+    return true; // All dependencies were met
+}
+
+
+/**
+ * Checks if the root data (portfolios, insights, social) exists for a given day.
+ * @param {string} dateStr - The date string to check (YYYY-MM-DD).
+ * @param {object} config - The computation system configuration object.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
+ * @returns {Promise<object>} { portfolioRefs, insightsData, socialData, hasPortfolio, hasInsights, hasSocial }
+ */
+async function checkRootDataAvailability(dateStr, config, dependencies) {
+    const { logger } = dependencies;
+    logger.log('INFO', `[PassRunner] Checking root data availability for ${dateStr}...`);
+
+    try {
+        const [portfolioRefs, insightsData, socialData] = await Promise.all([
+            getPortfolioPartRefs(config, dependencies, dateStr),
+            loadDailyInsights(config, dependencies, dateStr),
+            loadDailySocialPostInsights(config, dependencies, dateStr)
+        ]);
+
+        const hasPortfolio = (portfolioRefs && portfolioRefs.length > 0);
+        const hasInsights = !!insightsData;
+        const hasSocial = !!socialData;
+
+        return {
+            portfolioRefs: portfolioRefs || [],
+            insightsData: insightsData || null,
+            socialData: socialData || null,
+            hasPortfolio: hasPortfolio,
+            hasInsights: hasInsights,
+            hasSocial: hasSocial
+        };
+
+    } catch (err) {
+        logger.log('ERROR', `[PassRunner] Error checking data availability for ${dateStr}`, { errorMessage: err.message });
+        return { 
+            portfolioRefs: [], insightsData: null, socialData: null,
+            hasPortfolio: false, hasInsights: false, hasSocial: false 
+        };
+    }
+}
+
+/**
+ * (NEW) Fetches all computed dependencies for a given pass and date from Firestore.
+ * @param {string} dateStr - The date string (YYYY-MM-DD).
+ * @param {Array<object>} calcsInPass - The manifest entries for this pass.
+ * @param {object} config - The computation system configuration.
+ * @param {object} dependencies - Contains db, logger.
+ * @returns {Promise<object>} A map of { 'calc-name': result, ... }
+ */
+async function fetchDependenciesForPass(dateStr, calcsInPass, config, dependencies) {
+    const { db, logger } = dependencies;
+    const { resultsCollection, resultsSubcollection, computationsSubcollection } = config;
+    
+    const requiredDeps = new Set();
+    const manifestMap = new Map(); // Store manifest entry by name
+    
+    // 1. Get all unique dependencies required by calcs in *this* pass
+    for (const calc of calcsInPass) {
+        manifestMap.set(calc.name, calc);
+        if (calc.type === 'meta' && calc.dependencies) {
+            calc.dependencies.forEach(depName => requiredDeps.add(normalizeName(depName)));
+        }
+    }
+
+    if (requiredDeps.size === 0) {
+        logger.log('INFO', `[PassRunner] No Firestore dependencies to fetch for this pass on ${dateStr}.`);
+        return {};
+    }
+
+    logger.log('INFO', `[PassRunner] Fetching ${requiredDeps.size} dependencies from Firestore for ${dateStr}...`);
+    
+    const docRefs = [];
+    const depNames = [];
+
+    // 2. Build the list of Firestore document references
+    // This assumes all dependencies are in the *main manifest* (which they should be)
+    // and we can find their category.
+    for (const calcName of requiredDeps) {
+        const calcManifest = manifestMap.get(calcName);
+        if (!calcManifest) {
+            logger.log('ERROR', `[PassRunner] Cannot find manifest entry for dependency "${calcName}". This is a manifest error. Skipping dependency.`);
+            continue;
+        }
+        
+        const category = calcManifest.category || 'unknown';
+        const docRef = db.collection(resultsCollection).doc(dateStr)
+                         .collection(resultsSubcollection).doc(category)
+                         .collection(computationsSubcollection).doc(calcName);
+        
+        docRefs.push(docRef);
+        depNames.push(calcName);
+    }
+
+    // 3. Fetch all dependencies in one batch
+    const fetchedDependencies = {};
+    if (docRefs.length > 0) {
+        const snapshots = await db.getAll(...docRefs);
+        snapshots.forEach((doc, i) => {
+            const calcName = depNames[i];
+            if (doc.exists) {
+                fetchedDependencies[calcName] = doc.data();
+            } else {
+                fetchedDependencies[calcName] = null; // Mark as null if not found
+            }
+        });
+    }
+    
+    logger.log('INFO', `[PassRunner] Successfully fetched ${Object.keys(fetchedDependencies).length} dependencies for ${dateStr}.`);
+    return fetchedDependencies;
+}
+
+
+/**
+ * Main pipe: pipe.computationSystem.runComputationPass
+ * @param {object} config - The computation system configuration object.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
+ * @param {Array<object>} computationManifest - The injected computation manifest.
+ * @returns {Promise<Object>} Summary of all passes.
+ */
+async function runComputationPass(config, dependencies, computationManifest) {
+    const { logger } = dependencies;
+    
+    // --- (NEW) Get the pass number this function is responsible for ---
+    const passToRun = String(config.COMPUTATION_PASS_TO_RUN);
+    if (!passToRun) {
+        logger.log('ERROR', '[PassRunner] FATAL: COMPUTATION_PASS_TO_RUN is not defined in config. Aborting.');
+        return;
+    }
+    logger.log('INFO', `🚀 [PassRunner] Starting run for PASS ${passToRun}...`);
+    // --- END NEW ---
+
+    const summary = {};
+    const yesterday = new Date();
+    yesterday.setUTCDate(yesterday.getUTCDate() - 1);
+    const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
+
+    const firstDate = await getFirstDateFromSourceData(config, dependencies);
+    const startDateUTC = firstDate
+       ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate()))
+       : new Date(config.earliestComputationDate + 'T00:00:00Z');
+
+    const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
+
+    const passes = groupByPass(computationManifest);
+    const calcsInThisPass = passes[passToRun] || [];
+    
+    if (calcsInThisPass.length === 0) {
+        logger.log('WARN', `[PassRunner] No calculations found in manifest for Pass ${passToRun}. Exiting.`);
+        return;
+    }
+
+    const standardCalcs = calcsInThisPass.filter(c => c.type === 'standard');
+    const metaCalcs = calcsInThisPass.filter(c => c.type === 'meta');
+    logger.log('INFO', `[PassRunner] Found ${standardCalcs.length} standard and ${metaCalcs.length} meta calcs for Pass ${passToRun}.`);
+
+    
+    // --- Process ONE DAY at a time, in order ---
+    for (const dateStr of allExpectedDates) {
+        const dateToProcess = new Date(dateStr + 'T00:00:00Z');
+        
+        // 1. Check for root data (portfolios, insights, social)
+        const rootData = await checkRootDataAvailability(dateStr, config, dependencies);
+        const rootDataStatus = {
+            hasPortfolio: rootData.hasPortfolio,
+            hasInsights: rootData.hasInsights,
+            hasSocial: rootData.hasSocial
+        };
+
+        if (!rootData.hasPortfolio && !rootData.hasInsights && !rootData.hasSocial) {
+            logger.log('WARN', `[PassRunner] Skipping Pass ${passToRun} for ${dateStr} due to missing all root data.`);
+            continue; // Skip to the next day
+        }
+        
+        logger.log('INFO', `[PassRunner] Processing Pass ${passToRun} for ${dateStr}...`);
+        
+        // 2. (NEW) Fetch all dependencies for this pass and date from Firestore
+        // This is skipped for Pass 1, as `requiredDeps.size` will be 0
+        const fetchedDependencies = await fetchDependenciesForPass(dateStr, calcsInThisPass, config, dependencies);
+        
+        const skippedCalculations = new Set();
+        
+        // 3. Filter calculations based on root data
+        const standardCalcsToRun = [];
+        for (const calcManifest of standardCalcs) {
+            if (checkRootDependencies(calcManifest, rootDataStatus)) {
+                standardCalcsToRun.push(calcManifest);
+            } else {
+                logger.log('INFO', `[Pass ${passToRun}] Skipping standard calc "${calcManifest.name}" for ${dateStr} due to missing root data.`);
+                skippedCalculations.add(calcManifest.name);
+            }
+        }
+        
+        const metaCalcsToRun = [];
+        for (const calcManifest of metaCalcs) {
+            const calcName = calcManifest.name;
+            
+            // Check 1: Are root data dependencies met?
+            if (!checkRootDependencies(calcManifest, rootDataStatus)) {
+                logger.log('INFO', `[Pass ${passToRun} (Meta)] Skipping meta calc "${calcName}" for ${dateStr} due to missing root data.`);
+                skippedCalculations.add(calcName);
+                continue;
+            }
+            
+            // Check 2: Are *computed* dependencies (from previous passes) met?
+            let depCheck = true;
+            let missingDepName = '';
+            for (const depName of (calcManifest.dependencies || [])) {
+                const normalizedDepName = normalizeName(depName);
+                if (!fetchedDependencies[normalizedDepName]) { // Check if null or undefined
+                    depCheck = false;
+                    missingDepName = normalizedDepName;
+                    break;
+                }
+            }
+            
+            if (depCheck) {
+                metaCalcsToRun.push(calcManifest);
+            } else {
+                logger.log('WARN', `[Pass ${passToRun} (Meta)] Skipping meta calc "${calcName}" for ${dateStr} due to missing computed dependency "${missingDepName}" from Firestore.`);
+                skippedCalculations.add(calcName);
+            }
+        }
+        
+        // --- 4. Run the filtered calculations ---
+        try {
+            // Run standard calcs for this pass (e.g., Pass 1)
+            if (standardCalcsToRun.length > 0) {
+                await runUnifiedComputation(
+                    dateToProcess,
+                    standardCalcsToRun,
+                    `Pass ${passToRun} (Standard)`,
+                    config,
+                    dependencies,
+                    rootData 
+                );
+            }
+
+            // Run meta calcs for this pass (e.g., Pass 2, 3, 4)
+            if (metaCalcsToRun.length > 0) {
+                await runMetaComputation(
+                    dateToProcess,
+                    metaCalcsToRun,
+                    `Pass ${passToRun} (Meta)`,
+                    config,
+                    dependencies,
+                    fetchedDependencies, // <-- This is the NEW object from Firestore
+                    rootData
+                );
+            }
+             logger.log('SUCCESS', `[PassRunner] Completed Pass ${passToRun} for ${dateStr}.`);
+        } catch (err) {
+            logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
+        }
+    } // End dates loop
+
+    logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
+    return summary;
+}
+
+/**
+ * Internal sub-pipe: Initializes calculator instances.
+ */
+function initializeCalculators(calculationsToRun, logger) {
+    const state = {};
+    for (const calcManifest of calculationsToRun) {
+        const calcName = normalizeName(calcManifest.name);
+        const CalculationClass = calcManifest.class;
+        
+        if (typeof CalculationClass === 'function') {
+            try {
+                const instance = new CalculationClass();
+                instance.manifest = calcManifest; // Attach manifest data
+                state[calcName] = instance;
+            } catch (e) {
+                logger.warn(`[PassRunner] Init failed for ${calcName}`, { errorMessage: e.message });
+                state[calcName] = null;
+            }
+        } else {
+            logger.warn(`[PassRunner] Calculation class not found in manifest for: ${calcName}`);
+            state[calcName] = null;
+        }
+    }
+    return state;
+}
+
+/**
+ * Internal sub-pipe: Streams data and calls process() on "standard" calculators.
+ */
+async function streamAndProcess(
+    dateStr, todayRefs, state, passName, config, dependencies, 
+    yesterdayPortfolios = {}, 
+    todayInsights = null, 
+    yesterdayInsights = null,
+    todaySocialPostInsights = null,
+    yesterdaySocialPostInsights = null
+) {
+    const { logger, calculationUtils } = dependencies;
+    logger.log('INFO', `[${passName}] Streaming ${todayRefs.length} 'today' part docs for ${dateStr}...`);
+
+    const yesterdayDate = new Date(dateStr + 'T00:00:00Z');
+    yesterdayDate.setUTCDate(yesterdayDate.getUTCDate() - 1);
+    const yesterdayStr = yesterdayDate.toISOString().slice(0, 10);
+
+    const { instrumentToTicker, instrumentToSector } = await calculationUtils.loadInstrumentMappings();
+
+    const context = { 
+        instrumentMappings: instrumentToTicker, 
+        sectorMapping: instrumentToSector,
+        todayDateStr: dateStr,
+        yesterdayDateStr: yesterdayStr,
+        dependencies: dependencies,
+        config: config
+    };
+
+    const batchSize = config.partRefBatchSize || 10;
+    let isFirstUser = true;
+
+    for (let i = 0; i < todayRefs.length; i += batchSize) {
+        const batchRefs = todayRefs.slice(i, i + batchSize);
+        const todayPortfoliosChunk = await loadDataByRefs(config, dependencies, batchRefs);
+
+        for (const uid in todayPortfoliosChunk) {
+            const p = todayPortfoliosChunk[uid];
+            if (!p) continue;
+            const userType = p.PublicPositions ? 'speculator' : 'normal';
+            context.userType = userType;
+
+            for (const calcName in state) {
+                const calc = state[calcName];
+                if (!calc || typeof calc.process !== 'function') continue;
+                
+                const manifestCalc = calc.manifest;
+                const isSocialOrInsights = manifestCalc.category === 'socialPosts' || manifestCalc.category === 'insights';
+                const isHistoricalCalc = manifestCalc.isHistorical === true;
+                const isSpeculatorCalc = manifestCalc.category === 'speculators';
+                let processArgs;
+                const allContextArgs = [
+                    context, 
+                    todayInsights, 
+                    yesterdayInsights, 
+                    todaySocialPostInsights,
+                    yesterdaySocialPostInsights
+                ];
+
+                if (isSocialOrInsights) {
+                    if (isFirstUser) {
+                         processArgs = [null, null, null, ...allContextArgs];
+                    } else {
+                        continue; // Only run once
+                    }
+                } else if (isHistoricalCalc) {
+                    const pYesterday = yesterdayPortfolios[uid];
+                    if (!pYesterday) continue;
+                    processArgs = [p, pYesterday, uid, ...allContextArgs];
+                } else {
+                    processArgs = [p, null, uid, ...allContextArgs];
+                }
+                
+                if (!isSocialOrInsights) {
+                    if ((userType === 'normal' && isSpeculatorCalc) ||
+                        (userType === 'speculator' && !isSpeculatorCalc && calcName !== 'users-processed')) {
+                        continue; // Skip: wrong user type
+                    }
+                }
+
+                try {
+                    await Promise.resolve(calc.process(...processArgs));
+                } catch (e) {
+                    logger.log('WARN', `Process error in ${calcName} for user ${uid}`, { err: e.message });
+                }
+            }
+             isFirstUser = false;
+        }
+    }
+    
+    // Handle case where there are no users but we still need to run insights/social calcs
+    if (todayRefs.length === 0 && isFirstUser) {
+        logger.log('INFO', `[${passName}] No user portfolios found for ${dateStr}. Running insights/social calcs once.`);
+        const allContextArgs = [ context, todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights ];
+        for (const calcName in state) {
+            const calc = state[calcName];
+            if (!calc || typeof calc.process !== 'function') continue;
+            const manifestCalc = calc.manifest;
+            const isSocialOrInsights = manifestCalc.category === 'socialPosts' || manifestCalc.category === 'insights';
+            if (isSocialOrInsights) {
+                try {
+                    await Promise.resolve(calc.process(null, null, null, ...allContextArgs));
+                } catch (e) {
+                    logger.log('WARN', `Process error in ${calcName} for no-user run`, { err: e.message });
+                }
+            }
+        }
+    }
+}
+
+
+/**
+ * Internal sub-pipe: Runs "standard" computations (Pass 1) for a single date.
+ */
+async function runUnifiedComputation(dateToProcess, calculationsToRun, passName, config, dependencies, rootData) {
+    const { db, logger } = dependencies;
+    const dateStr = dateToProcess.toISOString().slice(0, 10);
+    logger.log('INFO', `[${passName}] Starting run for ${dateStr} with ${calculationsToRun.length} calcs.`);
+
+    try {
+        const {
+            portfolioRefs: todayRefs,
+            insightsData: todayInsightsData,
+            socialData: todaySocialPostInsightsData
+        } = rootData;
+        
+        let yesterdayPortfolios = {};
+        let yesterdayInsightsData = null; 
+        let yesterdaySocialPostInsightsData = null; 
+        
+        const requiresYesterdayPortfolio = calculationsToRun.some(c => c.isHistorical === true);
+        const requiresYesterdayInsights = calculationsToRun.some(c => c.class.prototype.process.toString().includes('yesterdayInsights'));
+        const requiresYesterdaySocialPosts = calculationsToRun.some(c => c.class.prototype.process.toString().includes('yesterdaySocialPostInsights'));
+
+        if (requiresYesterdayPortfolio || requiresYesterdayInsights || requiresYesterdaySocialPosts) {
+            // ... [Identical logic for fetching yesterday's data as in original file] ...
+            // This logic is complex and correct, so it is preserved.
+             if(requiresYesterdayInsights) {
+                let daysAgo = 1; const maxLookback = 30;
+                while (!yesterdayInsightsData && daysAgo <= maxLookback) {
+                    const prev = new Date(dateToProcess); prev.setUTCDate(prev.getUTCDate() - daysAgo);
+                    const prevStr = prev.toISOString().slice(0, 10);
+                    yesterdayInsightsData = await loadDailyInsights(config, dependencies, prevStr);
+                    if (yesterdayInsightsData) logger.log('INFO', `[${passName}] Found 'yesterday' instrument insights data from ${daysAgo} day(s) ago (${prevStr}).`);
+                    else daysAgo++;
+                }
+                if (!yesterdayInsightsData) logger.log('WARN', `[${passName}] Could not find any 'yesterday' instrument insights data within a ${maxLookback} day lookback.`);
+            }
+            if(requiresYesterdaySocialPosts) {
+                let daysAgo = 1; const maxLookback = 30; 
+                while (!yesterdaySocialPostInsightsData && daysAgo <= maxLookback) {
+                    const prev = new Date(dateToProcess); prev.setUTCDate(prev.getUTCDate() - daysAgo);
+                    const prevStr = prev.toISOString().slice(0, 10);
+                    yesterdaySocialPostInsightsData = await loadDailySocialPostInsights(config, dependencies, prevStr);
+                    if (yesterdaySocialPostInsightsData) logger.log('INFO', `[${passName}] Found 'yesterday' social post insights data from ${daysAgo} day(s) ago (${prevStr}).`);
+                    else daysAgo++;
+                }
+                if (!yesterdaySocialPostInsightsData) logger.log('WARN', `[${passName}] Could not find any 'yesterday' social post insights data within a ${maxLookback} day lookback.`);
+            }
+            if (requiresYesterdayPortfolio) {
+                const prev = new Date(dateToProcess); prev.setUTCDate(prev.getUTCDate() - 1);
+                const prevStr = prev.toISOString().slice(0, 10);
+                const yesterdayRefs = await getPortfolioPartRefs(config, dependencies, prevStr);
+                if (yesterdayRefs.length > 0) {
+                    yesterdayPortfolios = await loadFullDayMap(config, dependencies, yesterdayRefs);
+                     logger.log('INFO', `[${passName}] Loaded yesterday's (${prevStr}) portfolio map for historical calcs.`);
+                } else {
+                    logger.log('WARN', `[${passName}] Yesterday's (${prevStr}) portfolio data not found. Historical calcs requiring it will be skipped.`);
+                }
+            }
+        }
+        
+        const state = initializeCalculators(calculationsToRun, logger);
+        await streamAndProcess(
+            dateStr, 
+            todayRefs, 
+            state, 
+            passName, 
+            config, 
+            dependencies, 
+            yesterdayPortfolios, 
+            todayInsightsData, 
+            yesterdayInsightsData,
+            todaySocialPostInsightsData,
+            yesterdaySocialPostInsightsData
+        );
+
+        let successCount = 0;
+        const resultsCollectionRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection);
+
+        for (const calcName in state) {
+            const calc = state[calcName];
+            if (!calc || typeof calc.getResult !== 'function') {
+                 if (!calc) logger.log('WARN', `[${passName}] Skipping ${calcName} for ${dateStr} because it failed to initialize.`);
+                 continue;
+            }
+
+            const category = calc.manifest.category || 'unknown';
+            let result = null;
+            try {
+                result = await Promise.resolve(calc.getResult());
+                
+                // --- Database write logic (Identical to original file) ---
+                const pendingWrites = [];
+                const summaryData = {};
+                if (result && Object.keys(result).length > 0) {
+                    let isSharded = false;
+                    const shardedCollections = {
+                        'sharded_user_profile': config.shardedUserProfileCollection,
+                        'sharded_user_profitability': config.shardedProfitabilityCollection
+                    };
+                    for (const resultKey in shardedCollections) {
+                        if (result[resultKey]) {
+                            isSharded = true;
+                            const shardCollectionName = shardedCollections[resultKey];
+                            const shardedData = result[resultKey];
+                            for (const shardId in shardedData) {
+                                const shardDocData = shardedData[shardId];
+                                if (shardDocData && Object.keys(shardDocData).length > 0) {
+                                    const shardRef = db.collection(shardCollectionName).doc(shardId);
+                                    pendingWrites.push({ ref: shardRef, data: shardedData[shardId] });
+                                }
+                            }
+                            const { [resultKey]: _, ...otherResults } = result;
+                            if (Object.keys(otherResults).length > 0) {
+                                const computationDocRef = resultsCollectionRef.doc(category)
+                                    .collection(config.computationsSubcollection)
+                                    .doc(calcName);
+                                pendingWrites.push({ ref: computationDocRef, data: otherResults });
+                            }
+                        }
+                    }
+                    if (!isSharded) {
+                        const computationDocRef = resultsCollectionRef.doc(category)
+                                                          .collection(config.computationsSubcollection)
+                                                          .doc(calcName);
+                        pendingWrites.push({ ref: computationDocRef, data: result });
+                    }
+                    if (!summaryData[category]) summaryData[category] = {};
+                    summaryData[category][calcName] = true;
+                     if (Object.keys(summaryData).length > 0) {
+                        const topLevelDocRef = db.collection(config.resultsCollection).doc(dateStr);
+                        pendingWrites.push({ ref: topLevelDocRef, data: summaryData });
+                     }
+                    if (pendingWrites.length > 0) {
+                        await commitBatchInChunks(config, dependencies, pendingWrites, `Commit ${passName} ${dateStr} [${calcName}]`);
+                        successCount++;
+                    }
+                } else {
+                     if (result === null) logger.log('INFO', `[${passName}] Calculation ${calcName} returned null for ${dateStr}. This is expected if no data was processed.`);
+                     else logger.log('WARN', `[${passName}] Calculation ${calcName} produced empty results {} for ${dateStr}. Skipping write.`);
+                }
+            } catch (e) {
+                logger.log('ERROR', `[${passName}] getResult/Commit failed for ${calcName} on ${dateStr}`, { err: e.message, stack: e.stack });
+            }
+        }
+        const completionStatus = successCount === calculationsToRun.length ? 'SUCCESS' : 'WARN';
+        logger.log(completionStatus, `[${passName}] Completed ${dateStr}. Success: ${successCount}/${calculationsToRun.length}.`);
+
+    } catch (err) {
+        logger.log('ERROR', `[${passName}] Fatal error for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
+        throw err; // Re-throw to stop the orchestrator for this day
+    }
+}
+
+
+/**
+ * Internal sub-pipe: Runs "meta" computations (Pass 2, 3, 4) for a single date.
+ */
+async function runMetaComputation(
+    dateToProcess, 
+    calculationsToRun, // This is the *filtered* list
+    passName, 
+    config, 
+    dependencies, 
+    fetchedDependencies, // <-- This is the NEW object from Firestore
+    rootData
+) {
+    const { db, logger } = dependencies;
+    const dateStr = dateToProcess.toISOString().slice(0, 10);
+    logger.log('INFO', `[${passName}] Starting run for ${dateStr} with ${calculationsToRun.length} calcs.`);
+    
+    try {
+        const resultsCollectionRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection);
+        const compsSub = config.computationsSubcollection || 'computations';
+        let successCount = 0;
+        
+        // Add rootData to dependencies for meta-calcs that need to stream users
+        const dependenciesForMetaCalc = {
+            ...dependencies,
+            rootData: rootData
+        };
+
+        for (const manifestCalc of calculationsToRun) {
+            const calcName = normalizeName(manifestCalc.name);
+            const category = manifestCalc.category || 'unknown';
+            const CalcClass = manifestCalc.class;
+            
+            if (typeof CalcClass !== 'function') {
+                 logger.log('ERROR', `[${passName}] Invalid class in manifest for ${calcName}. Skipping.`);
+                 continue;
+            }
+            
+            const instance = new CalcClass();
+            let result = null;
+            try {
+                // --- Call process with the fetched dependencies ---
+                result = await Promise.resolve(instance.process(
+                    dateStr, 
+                    dependenciesForMetaCalc,
+                    config, 
+                    fetchedDependencies // <-- Pass the pre-fetched results
+                ));
+                
+                // --- Database write logic (Identical to original file) ---
+                const pendingWrites = [];
+                const summaryData = {};
+                if (result && Object.keys(result).length > 0) {
+                    let isSharded = false;
+                    const shardedCollections = {
+                        'sharded_user_profile': config.shardedUserProfileCollection,
+                        'sharded_user_profitability': config.shardedProfitabilityCollection 
+                    };
+                    for (const resultKey in shardedCollections) {
+                        if (result[resultKey]) {
+                            isSharded = true;
+                            const shardCollectionName = shardedCollections[resultKey];
+                            if (!shardCollectionName) {
+                                logger.log('ERROR', `[${passName}] Missing config key for sharded collection: ${resultKey}`);
+                                continue;
+                            }
+                            const shardedData = result[resultKey];
+                            for (const shardId in shardedData) {
+                                const shardDocData = shardedData[shardId];
+                                if (shardDocData && (Object.keys(shardDocData).length > 0)) {
+                                    const shardRef = db.collection(shardCollectionName).doc(shardId);
+                                    pendingWrites.push({ ref: shardRef, data: shardDocData, merge: true });
+                                }
+                            }
+                            const { [resultKey]: _, ...otherResults } = result;
+                            result = otherResults;
+                        }
+                    }
+                    if (result && Object.keys(result).length > 0) {
+                        const computationDocRef = resultsCollectionRef.doc(category)
+                                                          .collection(compsSub)
+                                                          .doc(calcName);
+                        pendingWrites.push({ ref: computationDocRef, data: result });
+                    }
+                    if (!summaryData[category]) summaryData[category] = {};
+                    summaryData[category][calcName] = true;
+                     if (Object.keys(summaryData).length > 0) {
+                        const topLevelDocRef = db.collection(config.resultsCollection).doc(dateStr);
+                        pendingWrites.push({ ref: topLevelDocRef, data: summaryData });
+                     }
+                    if (pendingWrites.length > 0) {
+                        await commitBatchInChunks(
+                            config,
+                            dependencies,
+                            pendingWrites,
+                            `Commit ${passName} ${dateStr} [${calcName}]`
+                        );
+                        successCount++;
+                    }
+                } else {
+                     logger.log('WARN', `[${passName}] Meta-calculation ${calcName} produced no results for ${dateStr}. Skipping write.`);
+                }
+            } catch (e) {
+                logger.log('ERROR', `[${passName}] Meta-calc process/commit failed for ${calcName} on ${dateStr}`, { err: e.message, stack: e.stack });
+            }
+        }
+
+        const completionStatus = successCount === calculationsToRun.length ? 'SUCCESS' : 'WARN';
+        logger.log(completionStatus, `[${passName}] Completed ${dateStr}. Success: ${successCount}/${calculationsToRun.length}.`);
+        
+    } catch (err) {
+        logger.log('ERROR', `[${passName}] Fatal error for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
+        throw err; 
+    }
+}
+
+
+module.exports = {
+    runComputationPass,
+};

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -688,6 +688,7 @@ async function runMetaComputation(
 
     try {
         const resultsCollectionRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection);
+        const compsSub = config.computationsSubcollection || 'computations';
         let successCount = 0;
         
         const dependenciesForMetaCalc = {

package/index.js

@@ -32,22 +32,16 @@ const orchestrator = {
 };
 
 // --- Pipe 2: Dispatcher ---
-
+// ... (identical to original file)
 const dispatcher = {
-    // Main Pipe
     handleRequest: require('./functions/dispatcher/index').handleRequest,
-    
-    // Sub-Pipe
     dispatchTasksInBatches: require('./functions/dispatcher/helpers/dispatch_helpers').dispatchTasksInBatches,
 };
 
 // --- Pipe 3: Task Engine ---
-
+// ... (identical to original file)
 const taskEngine = {
-    // Main Pipe
     handleRequest: require('./functions/task-engine/handler_creator').handleRequest,
-    
-    // Sub-Pipes
     handleDiscover: require('./functions/task-engine/helpers/discover_helpers').handleDiscover,
     handleVerify: require('./functions/task-engine/helpers/verify_helpers').handleVerify,
     handleUpdate: require('./functions/task-engine/helpers/update_helpers').handleUpdate,
@@ -55,8 +49,10 @@ const taskEngine = {
 
 // --- Pipe 4: Computation System ---
 const computationSystem = {
-    // Main Pipe
-    runOrchestration: require('./functions/computation-system/helpers/orchestration_helpers').runComputationOrchestrator,
+    // --- (MODIFICATION) ---
+    // The main pipe is now the new pass runner
+    runComputationPass: require('./functions/computation-system/helpers/computation_pass_runner').runComputationPass,
+    // --- (END MODIFICATION) ---
     
     // Sub-Pipes (Exposing utils for potential external use)
     dataLoader: require('./functions/computation-system/utils/data_loader'),
@@ -64,17 +60,14 @@ const computationSystem = {
 };
 
 // --- Pipe 5: API ---
-
+// ... (identical to original file)
 const api = {
-    // Main Pipe
     createApiApp: require('./functions/generic-api/index').createApiApp,
-    
-    // Sub-Pipes (Helpers exposed for documentation/testing)
     helpers: require('./functions/generic-api/helpers/api_helpers'),
 };
 
 // --- Pipe 6: Maintenance ---
-// Standalone cleanup and utility functions TODO -- Some of these are not really maintenance pipes, socials could do with its own pipe..?
+// ... (identical to original file)
 const maintenance = {
     runSpeculatorCleanup: require('./functions/speculator-cleanup-orchestrator/helpers/cleanup_helpers').runCleanup,
     handleInvalidSpeculator: require('./functions/invalid-speculator-handler/helpers/handler_helpers').handleInvalidSpeculator,
@@ -86,7 +79,7 @@ const maintenance = {
 };
 
 // --- Pipe 7: Proxy ---
-
+// ... (identical to original file)
 const proxy = {
     handlePost: require('./functions/appscript-api/index').handlePost,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.117",
+  "version": "1.0.119",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [