bulltrackers-module 1.0.145 → 1.0.146

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,6 +1,6 @@
 const { FieldPath } = require('@google-cloud/firestore');
 // --- MODIFIED: Import streamPortfolioData ---
-const { getPortfolioPartRefs, loadFullDayMap, loadDataByRefs, loadDailyInsights, loadDailySocialPostInsights, getHistoryPartRefs, streamPortfolioData } = require('../utils/data_loader.js');
+const { getPortfolioPartRefs, loadFullDayMap, loadDataByRefs, loadDailyInsights, loadDailySocialPostInsights, getHistoryPartRefs, streamPortfolioData, streamHistoryData } = require('../utils/data_loader.js');
 const { normalizeName, commitBatchInChunks } = require('../utils/utils.js');
 
 /** Stage 1: Group manifest by pass number */
@@ -116,26 +116,20 @@ function initializeCalculators(calcs, logger) { const state = {}; for (const c o
 
 /** * Stage 7: Load historical data required for calculations 
  */
+// --- MODIFIED: Stage 7: Load ONLY non-streaming historical data ---
 async function loadHistoricalData(date, calcs, config, deps, rootData) { 
     const { logger }              = deps;
-    const updated                 = {...rootData}, dStr=date.toISOString().slice(0,10); const tasks = [];
-    const needsYesterdayPortfolio = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('portfolio'));
-    const needsTodayHistory       = calcs.some(c => c.rootDataDependencies.includes('history'));
-    const needsYesterdayHistory   = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('history'));
+    const updated                 = {...rootData};
+    const tasks = [];
+    
+    // --- REMOVED: needsYesterdayPortfolio ---
+    // --- REMOVED: needsTodayHistory ---
+    // --- REMOVED: needsYesterdayHistory ---
     const needsYesterdayInsights  = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('insights'));
     const needsYesterdaySocial    = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('social'));
-    if(needsYesterdayPortfolio) {
-        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10);
-        logger.log('INFO', `[PassRunner] Loading YESTERDAY portfolio data for ${prevStr}`);
-        updated.yesterdayPortfolios=await loadFullDayMap(config,deps,await getPortfolioPartRefs(config,deps,prevStr)); })());}
-    if(needsTodayHistory) {
-        tasks.push((async()=>{ 
-        logger.log('INFO', `[PassRunner] Loading TODAY history data for ${dStr}`);
-        updated.todayHistoryData=await loadFullDayMap(config,deps,rootData.historyRefs); })());}
-    if(needsYesterdayHistory) {
-        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
-        logger.log('INFO', `[PassRunner] Loading YESTERDAY history data for ${prevStr}`);
-        updated.yesterdayHistoryData=await loadFullDayMap(config,deps,await getHistoryPartRefs(config,deps,prevStr)); })());}
+
+    // --- REMOVED: All async tasks for portfolio and history data ---
+
     if(needsYesterdayInsights) {
         tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
         logger.log('INFO', `[PassRunner] Loading YESTERDAY insights data for ${prevStr}`);
@@ -144,56 +138,129 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
         tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
         logger.log('INFO', `[PassRunner] Loading YESTERDAY social data for ${prevStr}`);
         updated.yesterdaySocialPostInsights=await loadDailySocialPostInsights(config,deps,prevStr); })());}
-    await Promise.all(tasks); return updated;
+    
+    await Promise.all(tasks); 
+    return updated; // This no longer contains the large data maps
 }
 
-
-/** * --- MODIFIED: Stage 8: Stream and process data for standard calculations ---
- * This function now uses an async generator to stream portfolio data
- * instead of loading it all into memory.
+/**
+ * --- REFACTORED: Stage 8: Stream and process data for standard calculations ---
+ * This function now streams today's portfolios, yesterday's portfolios, 
+ * and today's history data in parallel to avoid OOM errors.
+ * It loads chunks of all three streams, processes UIDs found in the
+ * main (today's portfolio) stream, and then deletes processed users
+ * from the historical maps to free memory.
  */
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData) { 
     const { logger, calculationUtils } = deps;
-    const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, todayHistoryData, yesterdayHistoryData, yesterdayPortfolios } = rootData;
+    
+    // --- MODIFIED: yesterdayPortfolios & todayHistoryData are no longer in rootData ---
+    const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights } = rootData;
+    
     const calcsThatStreamPortfolio = Object.values(state).filter(calc => calc && calc.manifest && (calc.manifest.rootDataDependencies.includes('portfolio') || calc.manifest.category === 'speculators'));
     const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config};
     let firstUser=true; 
+
+    // --- (Non-streaming (insights/social) calculation logic remains unchanged) ---
     for(const name in state){ 
         const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
         const cat=calc.manifest.category;
         if(cat==='socialPosts'||cat==='insights') {
             if (firstUser) {
                 logger.log('INFO', `[${passName}] Running non-streaming calc: ${name}`);
-                let args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData];
+                // (Using 'null' for hT and hY as they aren't relevant for these calcs)
+                let args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,null,null];
                 if(calc.manifest.isHistorical) {
-                    args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData];
+                    args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,null,null];
                 }
                 try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} (non-stream)`,{err:e.message});}
             }
         }
     }
+    // --- (End of non-streaming calc logic) ---
+
     if (calcsThatStreamPortfolio.length === 0) {
         logger.log('INFO', `[${passName}] No portfolio-streaming calcs to run for ${dateStr}. Skipping stream.`);
-        return; }
-    logger.log('INFO', `[${passName}] Streaming portfolio data for ${calcsThatStreamPortfolio.length} calcs...`);
+        return; 
+    }
+
+    logger.log('INFO', `[${passName}] Streaming portfolio & historical data for ${calcsThatStreamPortfolio.length} calcs...`);
+
+    // --- NEW: Prepare iterators and maps for parallel streaming ---
+    const prevDate = new Date(dateStr + 'T00:00:00Z');
+    prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+    const prevDateStr = prevDate.toISOString().slice(0, 10);
+
+    const needsYesterdayPortfolio = Object.values(state).some(c => c && c.manifest.isHistorical && c.manifest.rootDataDependencies.includes('portfolio'));
+    const needsTodayHistory = Object.values(state).some(c => c && c.manifest.rootDataDependencies.includes('history'));
+
+    // Get the async iterators
+    const yP_iterator = needsYesterdayPortfolio ? streamPortfolioData(config, deps, prevDateStr) : null;
+    const hT_iterator = needsTodayHistory ? streamHistoryData(config, deps, dateStr) : null;
+
+    // These maps will accumulate data chunk-by-chunk
+    let yesterdayPortfolios = {};
+    let todayHistoryData = {};
+
+    // Load the FIRST chunk of historical data before the loop starts
+    if (yP_iterator) { 
+        Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {}); 
+        logger.log('INFO', `[${passName}] Loaded first chunk of yesterday's portfolios.`);
+    }
+    if (hT_iterator) { 
+        Object.assign(todayHistoryData, (await hT_iterator.next()).value || {}); 
+        logger.log('INFO', `[${passName}] Loaded first chunk of today's history.`);
+    }
+    
+    // --- MODIFIED: Main streaming loop (driven by TODAY's portfolio stream) ---
     for await (const chunk of streamPortfolioData(config, deps, dateStr)) {
-        for(const uid in chunk){ const p=chunk[uid]; if(!p) continue; 
+        
+        // --- NEW: Load the NEXT chunk of historical data ---
+        // This keeps the historical maps populated as the main stream progresses
+        if (yP_iterator) { 
+            Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {}); 
+        }
+        if (hT_iterator) { 
+            Object.assign(todayHistoryData, (await hT_iterator.next()).value || {}); 
+        }
+
+        for(const uid in chunk){ // Iterate through today's portfolio chunk
+            const p = chunk[uid]; if(!p) continue; 
             const userType=p.PublicPositions?'speculator':'normal'; 
             context.userType=userType; 
+
+            // --- NEW: Look up corresponding historical data for THIS user ---
+            const pY = yesterdayPortfolios[uid] || null; // Yesterday's Portfolio
+            const hT = todayHistoryData[uid] || null;    // Today's History
+            // (Note: yesterdayHistoryData (hY) would require another stream if needed)
+
             for(const name in state){ 
                 const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
                 const cat=calc.manifest.category, isSocialOrInsights=cat==='socialPosts'||cat==='insights', isHistorical=calc.manifest.isHistorical, isSpec=cat==='speculators'; 
-                if(isSocialOrInsights) continue; 
-                let args=[p,null,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; 
+                
+                if(isSocialOrInsights) continue; // Skip non-streaming calcs
+
+                // --- MODIFIED: Arguments now use streamed historical data (hT) ---
+                let args=[p,null,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,hT,null]; 
+                
                 if(isHistorical){ 
-                    const pY=yesterdayPortfolios ? yesterdayPortfolios[uid] : null;
+                    // pY is now the streamed yesterday's portfolio for this user
                     if(!pY && (cat !== 'behavioural' && name !== 'historical-performance-aggregator')) continue; 
-                    args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; } 
+                    args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,hT,null]; 
+                } 
+
                 if((userType==='normal'&&isSpec)||(userType==='speculator'&&!isSpec&&name!=='users-processed')) continue; 
-                try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} for ${uid}`,{err:e.message});} } 
+                
+                try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} for ${uid}`,{err:e.message});} 
+            } 
             firstUser=false; 
+
+            // --- NEW: Clear this user from historical maps to free memory ---
+            if (pY) { delete yesterdayPortfolios[uid]; }
+            if (hT) { delete todayHistoryData[uid]; }
         } 
     }
+    logger.log('INFO', `[${passName}] Finished streaming data for ${dateStr}.`);
 }
 
 /** Stage 9: Run standard computations */

package/functions/computation-system/utils/data_loader.js

@@ -198,6 +198,28 @@ async function* streamPortfolioData(config, deps, dateString) {
 }
 // --- END: Stage 7 ---
 
+/**
+ * --- NEW: Stage 8: Stream history data in chunks ---
+ * Streams history data in chunks for a given date.
+ */
+async function* streamHistoryData(config, deps, dateString) {
+    const { logger } = deps;
+    const refs = await getHistoryPartRefs(config, deps, dateString); // <-- Uses history refs
+    if (refs.length === 0) {
+        logger.log('WARN', `[streamHistoryData] No history refs found for ${dateString}. Stream is empty.`);
+        return;
+    }
+
+    const batchSize = config.partRefBatchSize || 50; 
+    logger.log('INFO', `[streamHistoryData] Streaming ${refs.length} history parts in chunks of ${batchSize}...`);
+
+    for (let i = 0; i < refs.length; i += batchSize) {
+        const batchRefs = refs.slice(i, i + batchSize);
+        const data = await loadDataByRefs(config, deps, batchRefs);
+        yield data; 
+    }
+    logger.log('INFO', `[streamHistoryData] Finished streaming for ${dateString}.`);
+}
 
 module.exports = {
     getPortfolioPartRefs,
@@ -207,4 +229,5 @@ module.exports = {
     loadDailySocialPostInsights,
     getHistoryPartRefs,
     streamPortfolioData, // <-- EXPORT NEW FUNCTION
+    streamHistoryData  // <-- EXPORT NEW FUNCTION
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.145",
+  "version": "1.0.146",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [