bulltrackers-module 1.0.171 → 1.0.173

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,800 +1,268 @@
-const { FieldPath } = require('@google-cloud/firestore');
-const { 
-    getPortfolioPartRefs, 
-    loadFullDayMap, 
-    loadDataByRefs, 
-    loadDailyInsights, 
-    loadDailySocialPostInsights, 
-    getHistoryPartRefs, 
-    streamPortfolioData, 
-    streamHistoryData 
-} = require('../utils/data_loader.js');
-const { normalizeName, commitBatchInChunks } = require('../utils/utils.js');
-const { batchStoreSchemas } = require('../utils/schema_capture.js');
-
 /**
- * Stage 1: Group manifest by pass number
- * @param {Array} manifest - The full manifest array.
- * @returns {object} An object with pass numbers as keys and calc arrays as values.
+ * FIXED: orchestration_helpers.js
+ * V3.2: Enabled streaming of Trading History data (tH_iter) for computations
+ * that require it, alongside Portfolio data.
  */
-function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
 
-/**
- * Stage 2: Check root data dependencies for a calc
- * @param {object} calcManifest - A single calculation's manifest entry.
- * @param {object} rootDataStatus - The status object from checkRootDataAvailability.
- * @returns {{canRun: boolean, missing: string[]}}
- */
+const { ComputationController } = require('../controllers/computation_controller');
+const { 
+    getPortfolioPartRefs, loadDailyInsights, loadDailySocialPostInsights, 
+    getHistoryPartRefs, streamPortfolioData, streamHistoryData 
+} = require('../utils/data_loader');
+const { batchStoreSchemas } = require('../utils/schema_capture');
+const { normalizeName, commitBatchInChunks } = require('../utils/utils');
+
+// --- Helpers ---
+
+function groupByPass(manifest) { 
+    return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); 
+}
+
 function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
-    if (!calcManifest.rootDataDependencies || !calcManifest.rootDataDependencies.length) { return { canRun: true, missing }; }
+    if (!calcManifest.rootDataDependencies) return { canRun: true, missing };
     for (const dep of calcManifest.rootDataDependencies) {
-        if (dep === 'portfolio'     && !rootDataStatus.hasPortfolio) missing.push('portfolio');
-        else if (dep === 'insights' && !rootDataStatus.hasInsights)  missing.push('insights');
-        else if (dep === 'social'   && !rootDataStatus.hasSocial)    missing.push('social');
-        else if (dep === 'history'  && !rootDataStatus.hasHistory)   missing.push('history');
+        if      (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
+        else if (dep === 'insights'  && !rootDataStatus.hasInsights)  missing.push('insights');
+        else if (dep === 'social'    && !rootDataStatus.hasSocial)    missing.push('social');
+        else if (dep === 'history'   && !rootDataStatus.hasHistory)   missing.push('history');
     }
     return { canRun: missing.length === 0, missing };
 }
 
-/**
- * Stage 3: Check root data availability for a date
- * @param {string} dateStr - The date to check (YYYY-MM-DD).
- * @param {object} config - The computation system config.
- * @param {object} dependencies - All shared dependencies (db, logger, etc.).
- * @param {object} earliestDates - The map of earliest data dates.
- * @returns {Promise<object|null>} The root data object or null.
- */
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
     const { logger } = dependencies;
-    logger.log('INFO', `[PassRunner] Checking root data for ${dateStr}...`);
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
     
-    let portfolioRefs = [], insightsData = null, socialData = null, historyRefs = [];
+    let portfolioRefs = [], historyRefs = [];
     let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false;
+    let insightsData = null, socialData = null;
 
     try {
         const tasks = [];
-        // Only query for data if the processing date is on or after the earliest possible date
-        if (dateToProcess >= earliestDates.portfolio) {
-            tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(res => { portfolioRefs = res; hasPortfolio = !!(res?.length); }));
-        }
-        if (dateToProcess >= earliestDates.insights) {
-            tasks.push(loadDailyInsights(config, dependencies, dateStr).then(res => { insightsData = res; hasInsights = !!res; }));
-        }
-        if (dateToProcess >= earliestDates.social) {
-            tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(res => { socialData = res; hasSocial = !!res; }));
-        }
-        if (dateToProcess >= earliestDates.history) {
-            tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(res => { historyRefs = res; hasHistory = !!(res?.length); }));
-        }
+        if (dateToProcess >= earliestDates.portfolio) 
+            tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio = !!r.length; }));
+        if (dateToProcess >= earliestDates.insights)
+            tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
+        if (dateToProcess >= earliestDates.social)
+            tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
+        if (dateToProcess >= earliestDates.history)
+            tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
         
         await Promise.all(tasks);
         
-        logger.log('INFO', `[PassRunner] Data availability for ${dateStr}: P:${hasPortfolio}, I:${hasInsights}, S:${hasSocial}, H:${hasHistory}`);
+        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) return null;
         
-        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) { logger.log('WARN', `[PassRunner] No root data at all for ${dateStr}.`); return null; }
-        
-        return {
-            portfolioRefs,
-            todayInsights: insightsData,
-            todaySocialPostInsights: socialData,
-            historyRefs,
-            status: { hasPortfolio, hasInsights, hasSocial, hasHistory }
+        return { 
+            portfolioRefs, historyRefs, 
+            todayInsights: insightsData, todaySocialPostInsights: socialData, 
+            status: { hasPortfolio, hasInsights, hasSocial, hasHistory } 
         };
     } catch (err) {
-        logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message });
+        logger.log('ERROR', `Error checking data: ${err.message}`);
         return null;
     }
 }
 
-/**
- * Stage 4: Fetch ALL existing computed results for the pass AND their dependencies.
- * @param {string} dateStr - The date to fetch (YYYY-MM-DD).
- * @param {Array} calcsInPass - The calculations in the *current* pass.
- * @param {Array} fullManifest - The *entire* computation manifest.
- * @param {object} config - The computation system config.
- * @param {object} dependencies - Shared dependencies.
- * @returns {Promise<object>} A map of { [calcName]: result } for all found dependencies.
- */
-async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db, logger }) {
+// --- OPTIMIZED FETCH ---
+async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
     const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
     const calcsToFetch = new Set();
     
-    // Add all calcs in this pass (to check for completion)
-    for (const calc of calcsInPass) { calcsToFetch.add(normalizeName(calc.name));
-        
-        // Add all dependencies of those calcs (for meta-calcs)
-        if (calc.dependencies && calc.dependencies.length > 0) {
-            for (const depName of calc.dependencies) {
-                calcsToFetch.add(normalizeName(depName));
-            }
+    for (const calc of calcsInPass) {
+        if (calc.dependencies) {
+            calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d)));
+        }
+        if (includeSelf && calc.isHistorical) {
+            calcsToFetch.add(normalizeName(calc.name));
         }
     }
     
-    if (!calcsToFetch.size) { return {}; }
-    
-    logger.log('INFO', `[PassRunner] Checking for ${calcsToFetch.size} existing results and dependencies for ${dateStr}...`);
+    if (!calcsToFetch.size) return {};
     
+    const fetched = {};
     const docRefs = [];
-    const depNames = [];
+    const names = [];
     
-    for (const calcName of calcsToFetch) {
-        const calcManifest = manifestMap.get(calcName);
-        if (!calcManifest) { logger.log('ERROR', `[PassRunner] Missing manifest for dependency: ${calcName} on ${dateStr}`); continue; }
-        
-        docRefs.push(
-            db.collection(config.resultsCollection)
-              .doc(dateStr)
-              .collection(config.resultsSubcollection)
-              .doc(calcManifest.category || 'unknown')
-              .collection(config.computationsSubcollection)
-              .doc(calcName)
-        );
-        depNames.push(calcName);
+    for (const name of calcsToFetch) {
+        const m = manifestMap.get(name);
+        if (m) {
+            docRefs.push(db.collection(config.resultsCollection).doc(dateStr)
+                .collection(config.resultsSubcollection).doc(m.category || 'unknown')
+                .collection(config.computationsSubcollection).doc(name));
+            names.push(name);
+        }
     }
-
-    const fetched = {};
+    
     if (docRefs.length) {
-        const snapshots = await db.getAll(...docRefs);
-        snapshots.forEach((doc, i) => {
-            const data = doc.exists ? doc.data() : null;
-            // A result only "exists" if it was marked as completed
-            if (data && data._completed === true) { fetched[depNames[i]] = data;
-            } else { fetched[depNames[i]] = null; // Treat as not existing
+        const snaps = await db.getAll(...docRefs);
+        snaps.forEach((doc, i) => { 
+            if(doc.exists && doc.data()._completed) {
+                fetched[names[i]] = doc.data(); 
             }
         });
     }
-    
-    // Verbose logging for what was found/missing
-    const foundDeps = Object.entries(fetched).filter(([, data]) => data !== null).map(([key]) => key);
-    const missingDeps = Object.entries(fetched).filter(([, data]) => data === null).map(([key]) => key);
-    if (foundDeps.length > 0) { logger.log('TRACE', `[PassRunner] Found ${foundDeps.length} existing results for ${dateStr}: [${foundDeps.join(', ')}]`); }
-    if (missingDeps.length > 0) { logger.log('TRACE', `[PassRunner] Did not find ${missingDeps.length} results for ${dateStr}: [${missingDeps.join(', ')}]`); }
-
     return fetched;
 }
 
-/**
- * Stage 5: Filter calculations based on data availability and completion status.
- */
-function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, earliestDates, logger) {
-    const skipped = new Set();
-    const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-
-    // Helper to find the true earliest date a calc can run
-    const getTrueEarliestRunDate = (calc) => {
-        let earliestRunDate = new Date('1970-01-01T00:00:00Z');
-        const dependencies = calc.rootDataDependencies || [];
-        
-        for (const dep of dependencies) {
-            if (dep === 'portfolio' && earliestDates.portfolio > earliestRunDate) earliestRunDate = earliestDates.portfolio;
-            if (dep === 'history'   && earliestDates.history   > earliestRunDate) earliestRunDate = earliestDates.history;
-            if (dep === 'social'    && earliestDates.social    > earliestRunDate) earliestRunDate = earliestDates.social;
-            if (dep === 'insights'  && earliestDates.insights  > earliestRunDate) earliestRunDate = earliestDates.insights;
-        }
-        
-        // If it's historical, it needs T-1 data, so add one day
-        if (calc.isHistorical && earliestRunDate.getTime() > 0) { earliestRunDate.setUTCDate(earliestRunDate.getUTCDate() + 1); }
-        return earliestRunDate;
-    };
-
-    const filterCalc = (calc) => {
-        // 1. Skip if already completed
-        if (existingResults[calc.name]) { logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Result already exists (and is complete).`);
-            skipped.add(calc.name);
-            return false;
-        }
-        
-        // 2. Skip if date is before this calc's earliest possible run date
-        const earliestRunDate = getTrueEarliestRunDate(calc);
-        if (dateToProcess < earliestRunDate) { logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Date is before true earliest run date (${earliestRunDate.toISOString().slice(0, 10)}).`);
-            skipped.add(calc.name);
-            return false;
-        }
-        
-        // 3. Skip if missing root data
-        const { canRun, missing: missingRoot } = checkRootDependencies(calc, rootDataStatus);
-        if (!canRun) { logger.log('INFO', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Data missing for this date: [${missingRoot.join(', ')}]`);
-            skipped.add(calc.name);
-            return false;
-        }
-        
-        // 4. (Meta Calcs) Skip if missing computed dependencies
-        if (calc.type === 'meta') {
-            const missingDeps = (calc.dependencies || [])
-                .map(normalizeName)
-                .filter(d => !existingResults[d]); // This check is now robust
-                
-            if (missingDeps.length > 0) { logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${calc.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`);
-                skipped.add(calc.name);
-                return false;
-            }
-        }
+function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, earliestDates) { // TODO passtorun is unused, why? 
+    const filter = (c) => {                                                                                                 // Since we are using toplogical sorting for deps, surely the pass being run is implicit
+        if (existingResults[c.name]) return false;                                                                          // Am a bit confused here. Feel this may be a bug.
+        let earliest = new Date('1970-01-01');
+        (c.rootDataDependencies || []).forEach(d => { if(earliestDates[d] > earliest) earliest = earliestDates[d]; });
+        if (c.isHistorical) earliest.setUTCDate(earliest.getUTCDate() + 1);
+        if (new Date(dateStr) < earliest) return false;
+        if (!checkRootDependencies(c, rootDataStatus).canRun) return false;
+        if (c.type === 'meta' && c.dependencies && c.dependencies.some(d => !existingResults[normalizeName(d)])) return false;
         return true;
     };
-
-    const standardCalcsToRun = standardCalcs.filter(filterCalc);
-    const metaCalcsToRun = metaCalcs.filter(filterCalc);
-    
-    return { standardCalcsToRun, metaCalcsToRun };
+    return { standardCalcsToRun: standardCalcs.filter(filter), metaCalcsToRun: metaCalcs.filter(filter) };
 }
 
-/**
- * Stage 6: Initialize calculator instances
- * @param {Array} calcs - The calculations to initialize.
- * @param {object} logger - The logger instance.
- * @returns {object} A map of { [calcName]: instance }.
- */
-function initializeCalculators(calcs, logger) {
-    const state = {};
-    for (const c of calcs) {
-        const name = normalizeName(c.name);
-        const Cl = c.class;
-        if (typeof Cl === 'function') {
-            try {
-                const inst = new Cl();
-                inst.manifest = c; // Attach manifest for context
-                state[name] = inst;
-            } catch (e) { logger.warn(`Initialization failed for ${name}`, { errorMessage: e.message });
-                state[name] = null;
-            }
-        } else { logger.warn(`Class is missing for ${name}`);
-            state[name] = null;
-        }
-    }
-    return state;
-}
+// --- EXECUTION DELEGATES ---
 
-/**
- * Stage 7: Load T-1 (yesterday) data needed by historical calculations.
- * --- THIS IS THE FULLY CORRECTED FUNCTION (WITH FRIEND'S BUG FIX) ---
- */
-async function loadHistoricalData(date, calcs, config, deps, rootData) {
+async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps) {
     const { logger } = deps;
-    const updated = { ...rootData };
-    const tasks = [];
-
-    // Check what T-1 data is needed by any calc in this pass
-    const needsYesterdayInsights = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('insights'));
-    const needsYesterdaySocial = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('social'));
-    const needsYesterdayPortfolio = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('portfolio'));
-    const needsYesterdayHistory = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('history')); // <-- The check
-    const needsYesterdayDependencies = calcs.some(c => c.isHistorical && c.dependencies && c.dependencies.length > 0);
-
-    const prev = new Date(date);
-    prev.setUTCDate(prev.getUTCDate() - 1);
-    const prevStr = prev.toISOString().slice(0, 10);
+    const controller = new ComputationController(config, deps);
 
-    if (needsYesterdayInsights) {
-        tasks.push((async () => { logger.log('INFO', `[PassRunner] Loading YESTERDAY insights data for ${prevStr}`);
-            updated.yesterdayInsights = await loadDailyInsights(config, deps, prevStr);
-        })());
-    }
-    if (needsYesterdaySocial) {
-        tasks.push((async () => { logger.log('INFO', `[PassRunner] Loading YESTERDAY social data for ${prevStr}`);
-            updated.yesterdaySocialPostInsights = await loadDailySocialPostInsights(config, deps, prevStr);
-        })());
-    }
-    if (needsYesterdayPortfolio) {
-        tasks.push((async () => { logger.log('INFO', `[PassRunner] Getting YESTERDAY portfolio refs for ${prevStr}`);
-            updated.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
-        })());
-    }
-    
-    // --- THIS IS THE MISSING LOGIC BLOCK (FIXED) ---
-    if (needsYesterdayHistory) {
-        tasks.push((async () => { logger.log('INFO', `[PassRunner] Getting YESTERDAY history refs for ${prevStr}`);
-            updated.yesterdayHistoryRefs = await getHistoryPartRefs(config, deps, prevStr);
-        })());
-    }
-    // --- END MISSING LOGIC BLOCK ---
-    
-    if (needsYesterdayDependencies) {
-        tasks.push((async () => { logger.log('INFO', `[PassRunner] Loading YESTERDAY computed dependencies for ${prevStr}`);
-            // This fetches T-1 results for *all* calcs in the current pass,
-            // which is robust and covers all historical dependency needs.
-            updated.yesterdayDependencyData = await fetchExistingResults(prevStr, calcs, calcs.map(c => c.manifest), config, deps);
-        })());
-    }
+    const calcs = Object.values(state).filter(c => c && c.manifest);
+    const streamingCalcs = calcs.filter(c => 
+        c.manifest.rootDataDependencies.includes('portfolio') || 
+        c.manifest.rootDataDependencies.includes('history')
+    );
 
-    await Promise.all(tasks);
-    return updated;
-}
+    if (streamingCalcs.length === 0) return;
 
-/**
- * Stage 8: Stream and process data for standard calculations.
- * --- THIS IS THE FULLY CORRECTED FUNCTION (WITH ALL FIXES) ---
- * --- REPLICATES THE 7-ARGUMENT "HACK" SIGNATURE FROM TEST HARNESS ---
- */
-async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps) {
-    const { logger, calculationUtils } = deps;
-    const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, yesterdayDependencyData } = rootData;
-    
-    // Create the shared context object
-    const mappings = await calculationUtils.loadInstrumentMappings();
-    const context = {
-        instrumentMappings: mappings.instrumentToTicker,
-        sectorMapping: mappings.instrumentToSector,
-        todayDateStr: dateStr,
-        dependencies: deps,
-        config,
-        yesterdaysDependencyData: yesterdayDependencyData
-    };
+    logger.log('INFO', `[${passName}] Streaming for ${streamingCalcs.length} computations...`);
 
-    // --- Run non-streaming (meta) calcs once ---
-    // This logic is 1:1 with the test harness
-    let firstUser = true;
-    for (const name in state) {
-        const calc = state[name];
-        if (!calc || typeof calc.process !== 'function') continue;
-        
-        const cat = calc.manifest.category;
-        if (cat === 'socialPosts' || cat === 'insights') {
-            if (firstUser) {
-                logger.log('INFO', `[${passName}] Running non-streaming calc: ${name} for ${dateStr}`);
-                
-                // --- CALLING 7-ARGUMENT "HACK" SIGNATURE (for non-streaming) ---
-                // We emulate the test harness's `process` call for social/insights
-                const userContext = { ...context, userType: 'n/a' };
-                const todayPayload = null; // No user data
-                const yesterdayPayload = null; // No user data
+    await controller.loader.loadMappings();
+    const prevDate = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+    const prevDateStr = prevDate.toISOString().slice(0, 10);
 
-                try {
-                    await Promise.resolve(calc.process(
-                        todayPayload,           // Arg 1: The data object
-                        yesterdayPayload,       // Arg 2: Yesterday's data
-                        null,                   // Arg 3: User ID
-                        userContext,            // Arg 4: Context
-                        todayInsights,          // Arg 5: Today Insights
-                        yesterdayInsights,      // Arg 6: Yesterday Insights
-                        fetchedDeps             // Arg 7: Fetched Dependencies
-                    ));
-                } catch (e) { logger.log('WARN', `Process error on ${name} (non-stream) for ${dateStr}`, { err: e.message }); }
-            }
-        }
-    }
+    // 1. Today's Portfolio Stream
+    const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
     
-    // --- FIX 1: THE FAULTY GUARD CLAUSE (From friend) ---
-    // This now correctly checks for calcs that need 'portfolio' OR 'history',
-    // matching the test harness behavior of running history-only passes.
-    const calcsThatStream = Object.values(state).filter(calc => 
-        calc && calc.manifest && (
-            calc.manifest.rootDataDependencies.includes('portfolio') ||
-            calc.manifest.rootDataDependencies.includes('history')
-        )
-    );
-
-    if (calcsThatStream.length === 0) {
-        logger.log('INFO', `[${passName}] No portfolio or history streaming calcs to run for ${dateStr}. Skipping stream.`);
-        return;
+    // 2. Yesterday's Portfolio Stream (for 'isHistorical' calcs)
+    const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
+    const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) 
+        ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) 
+        : null;
+
+    // 3. Today's History Stream (NEW)
+    const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
+    const tH_iter = (needsTradingHistory && historyRefs)
+        ? streamHistoryData(config, deps, dateStr, historyRefs)
+        : null;
+
+    let yP_chunk = {};
+    let tH_chunk = {};
+
+    for await (const tP_chunk of tP_iter) {
+        if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
+        if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
+
+        const promises = streamingCalcs.map(calc => 
+            controller.executor.executePerUser(
+                calc,
+                calc.manifest,
+                dateStr,
+                tP_chunk,
+                yP_chunk, // Yesterday Portfolio
+                tH_chunk, // Today History (NEW ARGUMENT)
+                fetchedDeps,
+                previousFetchedDeps
+            )
+        );
+        await Promise.all(promises);
     }
-
-    // --- FIX 2: THE TYPO (From friend) ---
-    // This log message now correctly references 'calcsThatStream'.
-    logger.log('INFO', `[${passName}] Streaming portfolio & historical data for ${calcsThatStream.length} calcs for ${dateStr}...`);
-    
-    const prevDate = new Date(dateStr + 'T00:00:00Z');
-    prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-    const prevDateStr = prevDate.toISOString().slice(0, 10);
-
-    // Check which iterators we need
-    const needsYesterdayPortfolio = Object.values(state).some(c => c && c.manifest.isHistorical && c.manifest.rootDataDependencies.includes('portfolio'));
-    const needsTodayHistory = Object.values(state).some(c => c && c.manifest.rootDataDependencies.includes('history'));
-    const needsYesterdayHistory = Object.values(state).some(c => c && c.manifest.isHistorical && c.manifest.rootDataDependencies.includes('history'));
-
-    // --- Create all necessary iterators ---
-    // (This code is now correct because rootData.yesterdayHistoryRefs will be populated by Fix 1)
-    const tP_iterator = streamPortfolioData(config, deps, dateStr, portfolioRefs);
-    const yP_iterator = needsYesterdayPortfolio ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
-    const hT_iterator = needsTodayHistory ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
-    const hY_iterator = needsYesterdayHistory ? streamHistoryData(config, deps, prevDateStr, rootData.yesterdayHistoryRefs) : null;
-
-    let yesterdayPortfolios = {};
-    let todayHistoryData = {};
-    let yesterdayHistoryData = {};
-
-    // Pre-load the first chunk of historical data
-    if (yP_iterator) { Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {}); }
-    if (hT_iterator) { Object.assign(todayHistoryData, (await hT_iterator.next()).value || {}); }
-    if (hY_iterator) { Object.assign(yesterdayHistoryData, (await hY_iterator.next()).value || {}); }
-
-    for await (const chunk of tP_iterator) {
-        // Load the *next* chunk of historical data to stay in sync
-        if (yP_iterator) { Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {}); }
-        if (hT_iterator) { Object.assign(todayHistoryData, (await hT_iterator.next()).value || {}); }
-        if (hY_iterator) { Object.assign(yesterdayHistoryData, (await hY_iterator.next()).value || {}); }
-
-        for (const uid in chunk) {
-            const p = chunk[uid]; 
-            if (!p) continue;
-
-            const userType = p.PublicPositions ? 'speculator' : 'normal';
-            const userContext = { ...context, userType };
-            
-            // Get corresponding T-1 data
-            const pY = yesterdayPortfolios[uid] || null;
-            const hT = todayHistoryData[uid] || null;
-            const hY = yesterdayHistoryData[uid] || null; // <-- This will now have data
-
-            for (const name in state) {
-                const calc = state[name];
-                if (!calc || typeof calc.process !== 'function') continue;
-
-                const manifest = calc.manifest;
-                const cat = manifest.category;
-                const isSocialOrInsights = cat === 'socialPosts' || cat === 'insights';
-                if (isSocialOrInsights) continue; // Handled above
-
-                const isSpeculatorCalc = cat === 'speculators';
-                const isUserProcessed = name === 'users-processed';
-
-                if (userType === 'normal' && isSpeculatorCalc) continue;
-                if (userType === 'speculator' && !isSpeculatorCalc && !isUserProcessed) continue;
-
-                if (manifest.isHistorical && !pY) {
-                    if (cat !== 'behavioural' && name !== 'historical-performance-aggregator') {
-                        continue;
-                    }
-                }
-
-                // --- FIX 3: REPLICATE 7-ARGUMENT "HACK" SIGNATURE ---
-                // This logic block replicates the test harness's 'todayPayload'
-                // and 'yesterdayPayload' construction.
-                const rootDataDeps = manifest.rootDataDependencies || ['portfolio'];
-                const needsHistoricalData = manifest.isHistorical || false;
-                
-                let todayPayload = null;
-                let yesterdayPayload = null;
-
-                if (rootDataDeps.includes('portfolio')) {
-                    todayPayload = p || {}; // Start with portfolio
-                    yesterdayPayload = needsHistoricalData ? (pY || {}) : null;
-                    // Nest history if also requested
-                    if (rootDataDeps.includes('history')) {
-                        todayPayload.history = hT;
-                        if (yesterdayPayload) yesterdayPayload.history = hY;
-                    }
-                } else if (rootDataDeps.includes('history')) {
-                    // If *only* history is requested, it becomes Arg 1
-                    todayPayload = hT;
-                    yesterdayPayload = needsHistoricalData ? hY : null;
-                } else {
-                    // Fallback for calcs like price-metrics
-                    todayPayload = p || {};
-                    yesterdayPayload = needsHistoricalData ? (pY || {}) : null;
-                }
-                // --- END PAYLOAD CONSTRUCTION ---
-
-                try {
-                    // Call with the 7-argument signature
-                    await Promise.resolve(calc.process(
-                        todayPayload,           // Arg 1: The data object (built above)
-                        yesterdayPayload,       // Arg 2: Yesterday's data
-                        uid,                    // Arg 3: User ID
-                        userContext,            // Arg 4: Context
-                        todayInsights,          // Arg 5: Today Insights
-                        yesterdayInsights,      // Arg 6: Yesterday Insights
-                        fetchedDeps             // Arg 7: Fetched Dependencies
-                    ));
-                } catch (e) {
-                    logger.log('WARN', `Process error on ${name} for ${uid} on ${dateStr}`, { err: e.message });
-                }
-            } // end for(calc)
-            
-            firstUser = false;
-            
-            // Clear processed users from memory
-            if (pY) { delete yesterdayPortfolios[uid]; }
-            if (hT) { delete todayHistoryData[uid]; }
-            if (hY) { delete yesterdayHistoryData[uid]; }
-        } // end for(uid in chunk)
-    } // end for await(chunk)
-
-    // Clear stale data to prevent memory leaks
-    yesterdayPortfolios = {};
-    todayHistoryData = {};
-    yesterdayHistoryData = {};
-
-    logger.log('INFO', `[${passName}] Finished streaming data for ${dateStr}.`);
+    logger.log('INFO', `[${passName}] Streaming complete.`);
 }
 
+// --- RUNNERS ---
 
-/**
- * Stage 9: Run standard computations
- * --- MODIFIED: Now accepts 'fetchedDeps' and passes it to 'streamAndProcess' ---
- */
-async function runStandardComputationPass(date, calcs, passName, config, deps, rootData, fetchedDeps) {
-    const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
-    if (calcs.length === 0) {
-        logger.log('INFO', `[${passName}] No standard calcs to run for ${dStr} after filtering.`);
-        return;
-    }
-    
-    logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} standard calcs: [${calcs.map(c => c.name).join(', ')}]`);
-    
-    // Load T-1 data (portfolio, insights, social, history, computed)
-    const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
+async function runStandardComputationPass(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps) {
+    const dStr = date.toISOString().slice(0, 10);
+    const logger = deps.logger;
     
-    // Initialize calcs
-    const state = initializeCalculators(calcs, logger);
-    
-    // Stream T and T-1 data and process
-    // --- THIS IS THE CHANGE ---
-    await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps);
-    // --- END CHANGE ---
-
-    // --- Verbose Logging Setup ---
-    const successCalcs = [];
-    const failedCalcs = [];
-    
-    const standardWrites = [];
-    const shardedWrites = {};
-    const schemasToStore = [];
-
-    // --- Get Results ---
-    for (const name in state) {
-        const calc = state[name];
-        if (!calc || typeof calc.getResult !== 'function') continue;
+    const fullRoot = { ...rootData };
+    if (calcs.some(c => c.isHistorical)) {
+        const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
+        const prevStr = prev.toISOString().slice(0, 10);
+        fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
+    }
 
+    const state = {};
+    for (const c of calcs) {
         try {
-            // --- THIS IS THE CHANGE ---
-            // Pass 'fetchedDeps' to getResult, just like the test harness
-            const result = await Promise.resolve(calc.getResult(fetchedDeps));
-            // --- END CHANGE ---
-            
-            if (result && Object.keys(result).length > 0) {
-                const standardResult = {};
-                
-                // --- Handle Sharded Writes ---
-                for (const key in result) {
-                    if (key.startsWith('sharded_')) {
-                        const shardedData = result[key];
-                        for (const collectionName in shardedData) {
-                            if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};
-                            Object.assign(shardedWrites[collectionName], shardedData[collectionName]);
-                        }
-                    } else {
-                        standardResult[key] = result[key];
-                    }
-                }
-                
-                // --- Handle Standard Writes ---
-                if (Object.keys(standardResult).length > 0) {
-                    const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
-                        .collection(config.resultsSubcollection).doc(calc.manifest.category)
-                        .collection(config.computationsSubcollection).doc(name);
-                    
-                    standardResult._completed = true; // Mark as complete
-                    standardWrites.push({ ref: docRef, data: standardResult });
-                }
-                
-                // --- Capture Schema ---
-                const calcClass = calc.manifest.class;
-                let staticSchema = null;
-                if (calcClass && typeof calcClass.getSchema === 'function') {
-                    try {
-                        staticSchema = calcClass.getSchema();
-                    } catch (e) { logger.log('WARN', `[SchemaCapture] Failed to get static schema for ${name} on ${dStr}`, { err: e.message }); }
-                } else { logger.log('TRACE', `[SchemaCapture] No static schema found for ${name}. Skipping manifest entry.`); }
-                
-                if (staticSchema) {
-                    schemasToStore.push({
-                        name,
-                        category: calc.manifest.category,
-                        schema: staticSchema,
-                        metadata: {
-                            isHistorical: calc.manifest.isHistorical || false,
-                            dependencies: calc.manifest.dependencies || [],
-                            rootDataDependencies: calc.manifest.rootDataDependencies || [],
-                            pass: calc.manifest.pass,
-                            type: calc.manifest.type || 'standard'
-                        }
-                    });
-                }
-                successCalcs.push(name);
-            } else {
-                // Calc ran but returned no data
-                successCalcs.push(name);
-            }
-        } catch (e) { logger.log('ERROR', `getResult failed for ${name} on ${dStr}`, { err: e.message, stack: e.stack });
-            failedCalcs.push({ name, error: e.message });
-        }
-    } // --- End Get Results Loop ---
-
-    // --- Commit Writes ---
-    if (schemasToStore.length > 0) {
-        batchStoreSchemas(deps, config, schemasToStore).catch(err => { logger.log('WARN', `[SchemaCapture] Non-blocking schema storage failed for ${dStr}`, { errorMessage: err.message }); }); }
-    
-    if (standardWrites.length > 0) { await commitBatchInChunks(config, deps, standardWrites, `${passName} Standard ${dStr}`); }
-    
-    for (const docPath in shardedWrites) {
-        const docData = shardedWrites[docPath];
-        const shardedDocWrites = [];
-        let docRef;
-        if (docPath.includes('/')) { docRef = deps.db.doc(docPath);
-        } else { const collection = (docPath.startsWith('user_profile_history')) ? config.shardedUserProfileCollection : config.shardedProfitabilityCollection; docRef = deps.db.collection(collection).doc(docPath); }
-        
-        if (docData && typeof docData === 'object' && !Array.isArray(docData)) {
-            docData._completed = true;
-            shardedDocWrites.push({ ref: docRef, data: docData });
-        } else { logger.log('ERROR', `[${passName}] Invalid sharded document data for ${docPath} on ${dStr}. Not an object.`, { data: docData }); }
-        
-        if (shardedDocWrites.length > 0) { await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${docPath} ${dStr}`); }
+            const inst = new c.class();
+            inst.manifest = c;
+            state[normalizeName(c.name)] = inst;
+        } catch(e) { logger.log('WARN', `Failed to init ${c.name}`); }
     }
 
-    // --- Final Verbose Log ---
-    const logMetadata = {
-        total_expected: calcs.length,
-        success_count: successCalcs.length,
-        failed_count: failedCalcs.length,
-        successful_calcs: successCalcs,
-        failed_calcs: failedCalcs
-    };
-    logger.log( failedCalcs.length === 0 ? 'SUCCESS' : 'WARN', `[${passName}] Completed ${dStr}.`, logMetadata );
+    await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps);
+    await commitResults(state, dStr, passName, config, deps);
 }
 
-/**
- * Stage 10: Run meta computations
- * @param {Date} date - The date to run for.
- * @param {Array} calcs - The meta calculations to run.
- * @param {string} passName - The name of the pass (for logging).
- * @param {object} config - Computation system config.
- * @param {object} deps - Shared dependencies.
- * @param {object} fetchedDeps - In-memory results from *previous* passes.
- * @param {object} rootData - The loaded root data for today.
- */
-async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) {
-    const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
-    if (calcs.length === 0) { logger.log('INFO', `[${passName}] No meta calcs to run for ${dStr} after filtering.`); return; }
-    
-    logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} meta calcs: [${calcs.map(c => c.name).join(', ')}]`);
-    
-    // Load T-1 data (needed for stateful meta calcs)
-    const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
-
-    // --- Verbose Logging Setup ---
-    const successCalcs = [];
-    const failedCalcs = [];
-    
-    const standardWrites = [];
-    const shardedWrites = {};
-    const schemasToStore = [];
+async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, previousFetchedDeps, rootData) {
+    const controller = new ComputationController(config, deps);
+    const dStr = date.toISOString().slice(0, 10);
+    const state = {};
 
     for (const mCalc of calcs) {
-        const name = normalizeName(mCalc.name);
-        const Cl = mCalc.class;
-        
-        if (typeof Cl !== 'function') {
-            logger.log('ERROR', `Invalid class ${name} on ${dStr}`);
-            failedCalcs.push({ name, error: "Invalid class" });
-            continue;
-        }
-        
-        const inst = new Cl();
-         
         try {
-            if (typeof inst.process !== 'function') {
-                logger.log('ERROR', `Meta-calc ${name} is missing a 'process' method.`);
-                failedCalcs.push({ name, error: "Missing process method" });
-                continue;
-            }
-            
-            // Meta-calc `process` is different: it receives the date, full dependencies, config, and fetched dependencies.
-            // It *also* gets the T-1 root data via the `dependencies.rootData` object.
-            
-            // --- REPLICATE 5-ARGUMENT "HACK" SIGNATURE (from test harness) ---
-            // This is the "hack" fix for meta calcs from worker.js.
-            const metaPayload = {
-                social: rootData.todaySocialPostInsights,
-                insights: rootData.todayInsights,
-                priceData: null, // You don't load this yet, but test harness has it
-                yesterdayInsights: fullRoot.yesterdayInsights,
-                yesterdayPriceData: null, // You don't load this yet
-                date: dStr
-            };
+            const inst = new mCalc.class();
+            inst.manifest = mCalc;
+            await controller.executor.executeOncePerDay(inst, mCalc, dStr, fetchedDeps, previousFetchedDeps);
+            state[normalizeName(mCalc.name)] = inst;
+        } catch (e) { deps.logger.log('ERROR', `Meta calc failed ${mCalc.name}: ${e.message}`); }
+    }
 
-            const result = await Promise.resolve(inst.process(
-                metaPayload,    // Arg 1: The data object (for your hacks)
-                fullRoot,       // Arg 2: (rootData)
-                deps,           // Arg 3: (dependencies)
-                config,         // Arg 4: (config)
-                fetchedDeps     // Arg 5: (fetchedDependencies)
-            ));
-            // --- END SIGNATURE REPLICATION ---
+    await commitResults(state, dStr, passName, config, deps);
+}
 
-            if (result && Object.keys(result).length > 0) {
-                const standardResult = {};
-                
-                // --- Handle Sharded Writes ---
-                for (const key in result) {
-                    if (key.startsWith('sharded_')) {
-                        const shardedData = result[key];
-                        for (const collectionName in shardedData) {
-                            if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};
-                            Object.assign(shardedWrites[collectionName], shardedData[collectionName]);
-                        }
-                    } else {
-                        standardResult[key] = result[key];
-                    }
-                }
-                
-                // --- Handle Standard Writes ---
-                if (Object.keys(standardResult).length > 0) {
-                    const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
-                        .collection(config.resultsSubcollection).doc(mCalc.category)
-                        .collection(config.computationsSubcollection).doc(name);
-                    
-                    standardResult._completed = true; // Mark as complete
-                    standardWrites.push({ ref: docRef, data: standardResult });
-                }
-                
-                // --- Capture Schema ---
-                const calcClass = mCalc.class;
-                let staticSchema = null;
-                if (calcClass && typeof calcClass.getSchema === 'function') {
-                    try {
-                        staticSchema = calcClass.getSchema();
-                    } catch (e) { logger.log('WARN', `[SchemaCapture] Failed to get static schema for ${name} on ${dStr}`, { err: e.message }); }
-                } else { logger.log('TRACE', `[SchemaCapture] No static schema found for ${name}. Skipping manifest entry.`); }
-                
-                if (staticSchema) {
-                    schemasToStore.push({
-                        name,
-                        category: mCalc.category,
-                        schema: staticSchema,
-                        metadata: {
-                            isHistorical: mCalc.isHistorical || false,
-                            dependencies: mCalc.dependencies || [],
-                            rootDataDependencies: mCalc.rootDataDependencies || [],
-                            pass: mCalc.pass,
-                            type: 'meta'
-                        }
-                    });
-                }
-                successCalcs.push(name);
-            } else {
-                // Calc ran but returned no data
-                successCalcs.push(name);
+async function commitResults(stateObj, dStr, passName, config, deps) {
+    const writes = [], schemas = [], sharded = {};
+    for (const name in stateObj) {
+        const calc = stateObj[name];
+        try {
+            const result = await calc.getResult(); 
+            if (!result) continue;
+            const standardRes = {};
+            for (const key in result) {
+                if (key.startsWith('sharded_')) { // TODO - This should iedally become redundant, computations themselves should NEVER return an object so large it requires sharding...
+                    const sData = result[key];
+                    for (const c in sData) { sharded[c] = sharded[c] || {}; Object.assign(sharded[c], sData[c]); }
+                } else standardRes[key] = result[key];
             }
-        } catch (e) { logger.log('ERROR', `Meta-calc failed ${name} for ${dStr}`, { err: e.message, stack: e.stack });
-            failedCalcs.push({ name, error: e.message });
-        }
-    } // --- End Meta-Calc Loop ---
-
-    // --- Commit Writes ---
-    if (schemasToStore.length > 0) {
-        batchStoreSchemas(deps, config, schemasToStore).catch(err => {
-            logger.log('WARN', `[SchemaCapture] Non-blocking schema storage failed for ${dStr}`, { errorMessage: err.message });
-        });
+            if (Object.keys(standardRes).length) {
+                standardRes._completed = true;
+                writes.push({
+                    ref: deps.db.collection(config.resultsCollection).doc(dStr)
+                        .collection(config.resultsSubcollection).doc(calc.manifest.category)
+                        .collection(config.computationsSubcollection).doc(name),
+                    data: standardRes
+                });
+            }
+            if (calc.manifest.class.getSchema) {
+                schemas.push({ name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: calc.manifest });
+            }
+        } catch (e) { deps.logger.log('ERROR', `Commit failed ${name}: ${e.message}`); }
     }
+
+    if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(()=>{});
+    if (writes.length) await commitBatchInChunks(config, deps, writes, `${passName} Results`);
     
-    if (standardWrites.length > 0) { await commitBatchInChunks(config, deps, standardWrites, `${passName} Meta ${dStr}`); }
-    
-    for (const collectionName in shardedWrites) {
-        const docs = shardedWrites[collectionName];
-        const shardedDocWrites = [];
-        for (const docId in docs) {
-            const docRef = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(collectionName).doc(docId);
-            const docData = docs[docId];
-            docData._completed = true; // Mark as complete
-            shardedDocWrites.push({ ref: docRef, data: docData });
+    for (const col in sharded) {
+        const sWrites = [];
+        for (const id in sharded[col]) {
+            const ref = id.includes('/') ? deps.db.doc(id) : deps.db.collection(col).doc(id);
+            sWrites.push({ ref, data: { ...sharded[col][id], _completed: true } });
         }
-        if (shardedDocWrites.length > 0) { await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${collectionName} ${dStr}`); }
+        if (sWrites.length) await commitBatchInChunks(config, deps, sWrites, `${passName} Sharded ${col}`);
     }
-    
-    // --- Final Verbose Log ---
-    const logMetadata = {
-        total_expected: calcs.length,
-        success_count: successCalcs.length,
-        failed_count: failedCalcs.length,
-        successful_calcs: successCalcs,
-        failed_calcs: failedCalcs
-    };
-    logger.log( failedCalcs.length === 0 ? 'SUCCESS' : 'WARN', `[${passName}] Completed ${dStr}.`,logMetadata );
 }
 
-
 module.exports = {
     groupByPass,
     checkRootDataAvailability,