bulltrackers-module 1.0.144 → 1.0.146

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,6 +1,6 @@
 const { FieldPath } = require('@google-cloud/firestore');
 // --- MODIFIED: Import streamPortfolioData ---
-const { getPortfolioPartRefs, loadFullDayMap, loadDataByRefs, loadDailyInsights, loadDailySocialPostInsights, getHistoryPartRefs, streamPortfolioData } = require('../utils/data_loader.js');
+const { getPortfolioPartRefs, loadFullDayMap, loadDataByRefs, loadDailyInsights, loadDailySocialPostInsights, getHistoryPartRefs, streamPortfolioData, streamHistoryData } = require('../utils/data_loader.js');
 const { normalizeName, commitBatchInChunks } = require('../utils/utils.js');
 
 /** Stage 1: Group manifest by pass number */
@@ -12,8 +12,7 @@ function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[ca
  */
 function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
-    if (!calcManifest.rootDataDependencies || !calcManifest.rootDataDependencies.length) {
-        return { canRun: true, missing };}
+    if (!calcManifest.rootDataDependencies || !calcManifest.rootDataDependencies.length) { return { canRun: true, missing };}
     for (const dep of calcManifest.rootDataDependencies) {
         if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
         else if (dep === 'insights' && !rootDataStatus.hasInsights) missing.push('insights');
@@ -34,8 +33,6 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
     let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false;
     try {
         const tasks = [];
-        // This logic is correct. It *avoids* calling get...Refs
-        // if the dateToProcess is before the earliest known data.
         if (dateToProcess >= earliestDates.portfolio)
             {tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(res => {portfolioRefs = res;hasPortfolio = !!(res?.length);}));}
         if (dateToProcess >= earliestDates.insights) {
@@ -44,32 +41,11 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
             tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(res => {socialData = res;hasSocial = !!res;}));}
         if (dateToProcess >= earliestDates.history) {
             tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(res => {historyRefs = res;hasHistory = !!(res?.length);}));}
-
         await Promise.all(tasks);
-
-        // --- NEW: Log what was *actually* found ---
         logger.log('INFO', `[PassRunner] Data availability for ${dateStr}: P:${hasPortfolio}, I:${hasInsights}, S:${hasSocial}, H:${hasHistory}`);
 
-        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) {
-            logger.log('WARN', `[PassRunner] No root data at all for ${dateStr}.`);
-            // We return null to skip the entire day
-            return null;
-        }
-        
-        /**
-         * --- THIS IS THE FIX ---
-         * Rename keys to match what streamAndProcess (Stage 8) expects.
-         * 'insightsData' is renamed to 'todayInsights'.
-         * 'socialData' is renamed to 'todaySocialPostInsights'.
-         */
-        return {
-            portfolioRefs, 
-            todayInsights: insightsData,      // <-- FIX
-            todaySocialPostInsights: socialData, // <-- FIX
-            historyRefs,
-            status: { hasPortfolio, hasInsights, hasSocial, hasHistory }
-        };
-
+        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) { logger.log('WARN', `[PassRunner] No root data at all for ${dateStr}.`); return null; }
+        return { portfolioRefs, todayInsights: insightsData, todaySocialPostInsights: socialData, historyRefs, status: { hasPortfolio, hasInsights, hasSocial, hasHistory } };
     } catch (err) { logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message }); return null; }
 }
 
@@ -105,76 +81,32 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
 function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, earliestDates, logger) {
     const skipped = new Set();
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-
-    /**
-     * --- CORRECTED LOGIC ---
-     * Helper to get the true earliest date a calc can run.
-     */
     const getTrueEarliestRunDate = (calc) => {
-        let earliestRunDate = new Date('1970-01-01T00:00:00Z'); // Start at zero
+        let earliestRunDate = new Date('1970-01-01T00:00:00Z'); 
         const dependencies = calc.rootDataDependencies || [];
-
-        // 1. Find the LATEST "today" dependency
-        // This is the date where all *today* data is first available
         for (const dep of dependencies) {
             if (dep === 'portfolio' && earliestDates.portfolio > earliestRunDate) earliestRunDate = earliestDates.portfolio;
             if (dep === 'history' && earliestDates.history > earliestRunDate) earliestRunDate = earliestDates.history;
             if (dep === 'social' && earliestDates.social > earliestRunDate) earliestRunDate = earliestDates.social;
             if (dep === 'insights' && earliestDates.insights > earliestRunDate) earliestRunDate = earliestDates.insights;
         }
-
-        // 2. If the calc is historical, shift the *final* date by +1
-        // This universally applies the "+1" rule if *any* yesterday data is needed,
-        // (as long as we found a dependency in step 1).
-        if (calc.isHistorical && earliestRunDate.getTime() > 0) {
-            earliestRunDate.setUTCDate(earliestRunDate.getUTCDate() + 1);
-        }
-        
+        if (calc.isHistorical && earliestRunDate.getTime() > 0) { earliestRunDate.setUTCDate(earliestRunDate.getUTCDate() + 1); }
         return earliestRunDate;
     };
-
     const filterCalc = (calc) => {
-        // 1. Skip if result already exists
-        if (existingResults[calc.name]) {
-            logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Result already exists.`);
-            skipped.add(calc.name);
-            return false;
-        }
+        if (existingResults[calc.name]) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Result already exists.`); skipped.add(calc.name); return false;}
 
-        // 2. Check *true* earliest run date
         const earliestRunDate = getTrueEarliestRunDate(calc);
-        if (dateToProcess < earliestRunDate) {
-            logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Date is before true earliest run date (${earliestRunDate.toISOString().slice(0, 10)}).`);
-            skipped.add(calc.name);
-            return false;
-        }
-
-        // 3. Check if *today's* root data was *actually* found
-        // This handles gaps *after* the earliest date
+        if (dateToProcess < earliestRunDate) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Date is before true earliest run date (${earliestRunDate.toISOString().slice(0, 10)}).`); skipped.add(calc.name); return false; }
+        
         const { canRun, missing: missingRoot } = checkRootDependencies(calc, rootDataStatus);
-        if (!canRun) {
-            logger.log('INFO', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Data missing for this date: [${missingRoot.join(', ')}]`);
-            skipped.add(calc.name);
-            return false;
-        }
-
-        // 4. (For Meta Calcs) Check computed dependencies
-        if (calc.type === 'meta') {
-            const missingDeps = (calc.dependencies || []).map(normalizeName).filter(d => !existingResults[d]);
-            if (missingDeps.length > 0) {
-                logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${calc.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`);
-                skipped.add(calc.name);
-                return false;
-            }
-        }
+        if (!canRun) {logger.log('INFO', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Data missing for this date: [${missingRoot.join(', ')}]`);skipped.add(calc.name); return false;}
         
-        // If it passed all checks, run it.
+        if (calc.type === 'meta') { const missingDeps = (calc.dependencies || []).map(normalizeName).filter(d => !existingResults[d]); if (missingDeps.length > 0) { logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${calc.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`); skipped.add(calc.name); return false;} }
         return true;
     };
-
     const standardCalcsToRun = standardCalcs.filter(filterCalc);
     const metaCalcsToRun = metaCalcs.filter(filterCalc);
-
     return { standardCalcsToRun, metaCalcsToRun };
 }
 
@@ -183,141 +115,164 @@ function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingRe
 function initializeCalculators(calcs, logger) { const state = {}; for (const c of calcs) { const name=normalizeName(c.name), Cl=c.class; if(typeof Cl==='function') try { const inst=new Cl(); inst.manifest=c; state[name]=inst; } catch(e){logger.warn(`Init failed ${name}`,{errorMessage:e.message}); state[name]=null;} else {logger.warn(`Class missing ${name}`); state[name]=null;} } return state; }
 
 /** * Stage 7: Load historical data required for calculations 
- * --- THIS FUNCTION IS NOW FIXED ---
  */
+// --- MODIFIED: Stage 7: Load ONLY non-streaming historical data ---
 async function loadHistoricalData(date, calcs, config, deps, rootData) { 
-    const { logger } = deps; // <--- THIS WAS THE MISSING LINE
-    const updated = {...rootData}, dStr=date.toISOString().slice(0,10); const tasks = [];
-    
-    const needsYesterdayPortfolio = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('portfolio'));
-    const needsTodayHistory = calcs.some(c => c.rootDataDependencies.includes('history'));
-    const needsYesterdayHistory = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('history'));
-    const needsYesterdayInsights = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('insights'));
-    const needsYesterdaySocial = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('social'));
+    const { logger }              = deps;
+    const updated                 = {...rootData};
+    const tasks = [];
     
-    // --- MODIFIED: Be smarter about loading data ---
-    if(needsYesterdayPortfolio) {
-        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10);
-            logger.log('INFO', `[PassRunner] Loading YESTERDAY portfolio data for ${prevStr}`);
-            updated.yesterdayPortfolios=await loadFullDayMap(config,deps,await getPortfolioPartRefs(config,deps,prevStr)); 
-        })());
-    }
-    if(needsTodayHistory) {
-        tasks.push((async()=>{ 
-            logger.log('INFO', `[PassRunner] Loading TODAY history data for ${dStr}`);
-            updated.todayHistoryData=await loadFullDayMap(config,deps,rootData.historyRefs); 
-        })());
-    }
-    if(needsYesterdayHistory) {
-        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
-            logger.log('INFO', `[PassRunner] Loading YESTERDAY history data for ${prevStr}`);
-            updated.yesterdayHistoryData=await loadFullDayMap(config,deps,await getHistoryPartRefs(config,deps,prevStr)); 
-        })());
-    }
+    // --- REMOVED: needsYesterdayPortfolio ---
+    // --- REMOVED: needsTodayHistory ---
+    // --- REMOVED: needsYesterdayHistory ---
+    const needsYesterdayInsights  = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('insights'));
+    const needsYesterdaySocial    = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('social'));
+
+    // --- REMOVED: All async tasks for portfolio and history data ---
+
     if(needsYesterdayInsights) {
         tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
-            logger.log('INFO', `[PassRunner] Loading YESTERDAY insights data for ${prevStr}`);
-            updated.yesterdayInsights=await loadDailyInsights(config,deps,prevStr); 
-        })());
-    }
+        logger.log('INFO', `[PassRunner] Loading YESTERDAY insights data for ${prevStr}`);
+        updated.yesterdayInsights=await loadDailyInsights(config,deps,prevStr); })());}
     if(needsYesterdaySocial) {
         tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
-            logger.log('INFO', `[PassRunner] Loading YESTERDAY social data for ${prevStr}`);
-            updated.yesterdaySocialPostInsights=await loadDailySocialPostInsights(config,deps,prevStr); 
-        })());
-    }
-
-    await Promise.all(tasks); return updated;
+        logger.log('INFO', `[PassRunner] Loading YESTERDAY social data for ${prevStr}`);
+        updated.yesterdaySocialPostInsights=await loadDailySocialPostInsights(config,deps,prevStr); })());}
+    
+    await Promise.all(tasks); 
+    return updated; // This no longer contains the large data maps
 }
 
-
-/** * --- MODIFIED: Stage 8: Stream and process data for standard calculations ---
- * This function now uses an async generator to stream portfolio data
- * instead of loading it all into memory.
+/**
+ * --- REFACTORED: Stage 8: Stream and process data for standard calculations ---
+ * This function now streams today's portfolios, yesterday's portfolios, 
+ * and today's history data in parallel to avoid OOM errors.
+ * It loads chunks of all three streams, processes UIDs found in the
+ * main (today's portfolio) stream, and then deletes processed users
+ * from the historical maps to free memory.
  */
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData) { 
     const { logger, calculationUtils } = deps;
-    // --- MODIFIED: yesterdayInsights/Social are now loaded by loadHistoricalData ---
-    // --- THIS WILL NOW WORK, as rootData contains 'todayInsights' from Stage 3 ---
-    const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, todayHistoryData, yesterdayHistoryData, yesterdayPortfolios } = rootData;
     
-    // --- NEW: Check if streaming is even needed ---
-    const calcsThatStreamPortfolio = Object.values(state).filter(calc => calc && calc.manifest && (calc.manifest.rootDataDependencies.includes('portfolio') || calc.manifest.category === 'speculators'));
+    // --- MODIFIED: yesterdayPortfolios & todayHistoryData are no longer in rootData ---
+    const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights } = rootData;
     
+    const calcsThatStreamPortfolio = Object.values(state).filter(calc => calc && calc.manifest && (calc.manifest.rootDataDependencies.includes('portfolio') || calc.manifest.category === 'speculators'));
     const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config};
-    
-    // --- MODIFIED: Run non-streaming calcs first (social/insights) ---
-    // This allows them to run even if portfolio data is missing
-    let firstUser=true; // Used to run them only once
+    let firstUser=true; 
+
+    // --- (Non-streaming (insights/social) calculation logic remains unchanged) ---
     for(const name in state){ 
         const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
         const cat=calc.manifest.category;
         if(cat==='socialPosts'||cat==='insights') {
             if (firstUser) {
                 logger.log('INFO', `[${passName}] Running non-streaming calc: ${name}`);
-                // --- This 'args' array will now receive the correct data ---
-                let args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData];
-                // Pass historical data if needed
+                // (Using 'null' for hT and hY as they aren't relevant for these calcs)
+                let args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,null,null];
                 if(calc.manifest.isHistorical) {
-                    args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData];
+                    args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,null,null];
                 }
                 try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} (non-stream)`,{err:e.message});}
             }
         }
     }
-
+    // --- (End of non-streaming calc logic) ---
 
     if (calcsThatStreamPortfolio.length === 0) {
         logger.log('INFO', `[${passName}] No portfolio-streaming calcs to run for ${dateStr}. Skipping stream.`);
-        return; // Exit stream function
+        return; 
+    }
+
+    logger.log('INFO', `[${passName}] Streaming portfolio & historical data for ${calcsThatStreamPortfolio.length} calcs...`);
+
+    // --- NEW: Prepare iterators and maps for parallel streaming ---
+    const prevDate = new Date(dateStr + 'T00:00:00Z');
+    prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+    const prevDateStr = prevDate.toISOString().slice(0, 10);
+
+    const needsYesterdayPortfolio = Object.values(state).some(c => c && c.manifest.isHistorical && c.manifest.rootDataDependencies.includes('portfolio'));
+    const needsTodayHistory = Object.values(state).some(c => c && c.manifest.rootDataDependencies.includes('history'));
+
+    // Get the async iterators
+    const yP_iterator = needsYesterdayPortfolio ? streamPortfolioData(config, deps, prevDateStr) : null;
+    const hT_iterator = needsTodayHistory ? streamHistoryData(config, deps, dateStr) : null;
+
+    // These maps will accumulate data chunk-by-chunk
+    let yesterdayPortfolios = {};
+    let todayHistoryData = {};
+
+    // Load the FIRST chunk of historical data before the loop starts
+    if (yP_iterator) { 
+        Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {}); 
+        logger.log('INFO', `[${passName}] Loaded first chunk of yesterday's portfolios.`);
+    }
+    if (hT_iterator) { 
+        Object.assign(todayHistoryData, (await hT_iterator.next()).value || {}); 
+        logger.log('INFO', `[${passName}] Loaded first chunk of today's history.`);
     }
     
-    logger.log('INFO', `[${passName}] Streaming portfolio data for ${calcsThatStreamPortfolio.length} calcs...`);
-    
+    // --- MODIFIED: Main streaming loop (driven by TODAY's portfolio stream) ---
     for await (const chunk of streamPortfolioData(config, deps, dateStr)) {
-        for(const uid in chunk){ const p=chunk[uid]; if(!p) continue; 
+        
+        // --- NEW: Load the NEXT chunk of historical data ---
+        // This keeps the historical maps populated as the main stream progresses
+        if (yP_iterator) { 
+            Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {}); 
+        }
+        if (hT_iterator) { 
+            Object.assign(todayHistoryData, (await hT_iterator.next()).value || {}); 
+        }
+
+        for(const uid in chunk){ // Iterate through today's portfolio chunk
+            const p = chunk[uid]; if(!p) continue; 
             const userType=p.PublicPositions?'speculator':'normal'; 
             context.userType=userType; 
+
+            // --- NEW: Look up corresponding historical data for THIS user ---
+            const pY = yesterdayPortfolios[uid] || null; // Yesterday's Portfolio
+            const hT = todayHistoryData[uid] || null;    // Today's History
+            // (Note: yesterdayHistoryData (hY) would require another stream if needed)
+
             for(const name in state){ 
                 const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
                 const cat=calc.manifest.category, isSocialOrInsights=cat==='socialPosts'||cat==='insights', isHistorical=calc.manifest.isHistorical, isSpec=cat==='speculators'; 
                 
-                // --- MODIFIED: Skip social/insights here, they ran above ---
-                if(isSocialOrInsights) continue; 
-                
-                // --- This 'args' array will now receive the correct data ---
-                let args=[p,null,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; 
+                if(isSocialOrInsights) continue; // Skip non-streaming calcs
+
+                // --- MODIFIED: Arguments now use streamed historical data (hT) ---
+                let args=[p,null,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,hT,null]; 
                 
                 if(isHistorical){ 
-                    const pY=yesterdayPortfolios ? yesterdayPortfolios[uid] : null; // Check if yesterdayPortfolios exists
-                    // V3 behavioural calcs (like history aggregator) *can* run without pY
+                    // pY is now the streamed yesterday's portfolio for this user
                     if(!pY && (cat !== 'behavioural' && name !== 'historical-performance-aggregator')) continue; 
-                    args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; 
+                    args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,hT,null]; 
                 } 
+
                 if((userType==='normal'&&isSpec)||(userType==='speculator'&&!isSpec&&name!=='users-processed')) continue; 
-                try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} for ${uid}`,{err:e.message});} } 
+                
+                try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} for ${uid}`,{err:e.message});} 
+            } 
             firstUser=false; 
+
+            // --- NEW: Clear this user from historical maps to free memory ---
+            if (pY) { delete yesterdayPortfolios[uid]; }
+            if (hT) { delete todayHistoryData[uid]; }
         } 
     }
+    logger.log('INFO', `[${passName}] Finished streaming data for ${dateStr}.`);
 }
 
 /** Stage 9: Run standard computations */
 async function runStandardComputationPass(date, calcs, passName, config, deps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
-    // --- THIS IS THE CRITICAL CHANGE ---
-    // If 'calcs' is empty *because of the new filter*, this log won't even appear.
     if (calcs.length === 0) {
         logger.log('INFO', `[${passName}] No standard calcs to run for ${dStr} after filtering.`);
-        return; 
-    }
-    // This log now only appears if there is *actually* work to do.
+        return; }
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
-
     const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
     const state = initializeCalculators(calcs, logger);
-    
     await streamAndProcess(dStr, state, passName, config, deps, fullRoot);
-    
     let success = 0;
     const standardWrites = [];
     const shardedWrites = {}; 
@@ -354,8 +309,8 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
     if (standardWrites.length > 0) {
         await commitBatchInChunks(config, deps, standardWrites, `${passName} Standard ${dStr}`);
     }
-    for (const docPath in shardedWrites) { // 'docPath' is the key, e.g., 'user_profile_history_shard_0' or 'social_.../history'
-        const docData = shardedWrites[docPath]; // 'docData' is the object to write, e.g., { profiles: ... }
+    for (const docPath in shardedWrites) { 
+        const docData = shardedWrites[docPath]; 
         const shardedDocWrites = [];
         let docRef;
         if (docPath.includes('/')) {
@@ -381,17 +336,12 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
 /** Stage 10: Run meta computations */
 async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
-    // --- THIS IS THE CRITICAL CHANGE ---
     if (calcs.length === 0) {
         logger.log('INFO', `[${passName}] No meta calcs to run for ${dStr} after filtering.`);
         return;
     }
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
-    
-    // --- NEW: Load historical data for meta calcs if needed ---
-    // (This might be redundant if standard pass ran, but meta-calcs can run standalone)
     const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
-
     let success = 0;
     const standardWrites = [];
     const shardedWrites = {}; 
@@ -400,7 +350,6 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
         if (typeof Cl !== 'function') {logger.log('ERROR', `Invalid class ${name}`);continue;}
         const inst = new Cl();
         try {
-            // --- MODIFIED: Pass fullRoot to meta calcs ---
             const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData: fullRoot }, config, fetchedDeps));
             if (result && Object.keys(result).length > 0) {const standardResult = {}; for (const key in result) {
                     if (key.startsWith('sharded_')) {const shardedData = result[key];for (const collectionName in shardedData) 

package/functions/computation-system/utils/data_loader.js

@@ -198,6 +198,28 @@ async function* streamPortfolioData(config, deps, dateString) {
 }
 // --- END: Stage 7 ---
 
+/**
+ * --- NEW: Stage 8: Stream history data in chunks ---
+ * Streams history data in chunks for a given date.
+ */
+async function* streamHistoryData(config, deps, dateString) {
+    const { logger } = deps;
+    const refs = await getHistoryPartRefs(config, deps, dateString); // <-- Uses history refs
+    if (refs.length === 0) {
+        logger.log('WARN', `[streamHistoryData] No history refs found for ${dateString}. Stream is empty.`);
+        return;
+    }
+
+    const batchSize = config.partRefBatchSize || 50; 
+    logger.log('INFO', `[streamHistoryData] Streaming ${refs.length} history parts in chunks of ${batchSize}...`);
+
+    for (let i = 0; i < refs.length; i += batchSize) {
+        const batchRefs = refs.slice(i, i + batchSize);
+        const data = await loadDataByRefs(config, deps, batchRefs);
+        yield data; 
+    }
+    logger.log('INFO', `[streamHistoryData] Finished streaming for ${dateString}.`);
+}
 
 module.exports = {
     getPortfolioPartRefs,
@@ -207,4 +229,5 @@ module.exports = {
     loadDailySocialPostInsights,
     getHistoryPartRefs,
     streamPortfolioData, // <-- EXPORT NEW FUNCTION
+    streamHistoryData  // <-- EXPORT NEW FUNCTION
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.144",
+  "version": "1.0.146",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [