bulltrackers-module 1.0.150 → 1.0.152

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -40,7 +40,7 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
         if (dateToProcess >= earliestDates.social)    {tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(res => {socialData = res;hasSocial = !!res;}));}
         if (dateToProcess >= earliestDates.history)   {tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(res => {historyRefs = res;hasHistory = !!(res?.length);}));}
         await Promise.all(tasks);
-        logger.log('INFO', `[PassRunner] Data availability for ${dateStr}: P:${hasPortfolio}, I:${insightsData_}, S:${socialData_}, H:${hasHistory}`);
+        logger.log('INFO', `[PassRunner] Data availability for ${dateStr}: P:${hasPortfolio}, I:${hasInsights}, S:${hasSocial}, H:${hasHistory}`); //Fixed bug, was logging insightsData and socialData objects
         if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) { logger.log('WARN', `[PassRunner] No root data at all for ${dateStr}.`); return null; }
         return { portfolioRefs, todayInsights: insightsData, todaySocialPostInsights: socialData, historyRefs, status: { hasPortfolio, hasInsights, hasSocial, hasHistory } };
     } catch (err) { logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message }); return null; }
@@ -111,6 +111,13 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
     const needsYesterdayInsights  = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('insights'));
     const needsYesterdaySocial    = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('social'));
 
+    // --- ADD THIS ---
+    const needsYesterdayPortfolio = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('portfolio'));
+    const prev = new Date(date);
+    prev.setUTCDate(prev.getUTCDate() - 1);
+    const prevStr = prev.toISOString().slice(0, 10);
+    // --- END ADD ---
+
     if(needsYesterdayInsights) {
         tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
         logger.log('INFO', `[PassRunner] Loading YESTERDAY insights data for ${prevStr}`);
@@ -120,6 +127,16 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
         logger.log('INFO', `[PassRunner] Loading YESTERDAY social data for ${prevStr}`);
         updated.yesterdaySocialPostInsights=await loadDailySocialPostInsights(config,deps,prevStr); })());}
     
+    // --- ADD THIS BLOCK ---
+    if(needsYesterdayPortfolio) {
+        tasks.push((async()=>{ 
+            logger.log('INFO', `[PassRunner] Getting YESTERDAY portfolio refs for ${prevStr}`);
+            // This adds the refs to the 'fullRoot' object for later
+            updated.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr); 
+        })());
+    }
+    // --- END ADD ---
+
     await Promise.all(tasks); 
     return updated; 
 }
@@ -131,8 +148,9 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
  * It loads chunks of all three streams, processes UIDs found in the
  * main (today's portfolio) stream, and then deletes processed users
  * from the historical maps to free memory.
+ * --- FIX: Added portfolioRefs and historyRefs to signature ---
  */
-async function streamAndProcess(dateStr, state, passName, config, deps, rootData) { 
+async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs) { 
     const { logger, calculationUtils } = deps;
     const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights } = rootData;
     const calcsThatStreamPortfolio = Object.values(state).filter(calc => calc && calc.manifest && (calc.manifest.rootDataDependencies.includes('portfolio') || calc.manifest.category === 'speculators'));
@@ -168,8 +186,12 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     const needsYesterdayPortfolio = Object.values(state).some(c => c && c.manifest.isHistorical && c.manifest.rootDataDependencies.includes('portfolio'));
     const needsTodayHistory = Object.values(state).some(c => c && c.manifest.rootDataDependencies.includes('history'));
 
-    const yP_iterator = needsYesterdayPortfolio ? streamPortfolioData(config, deps, prevDateStr) : null;
-    const hT_iterator = needsTodayHistory ? streamHistoryData(config, deps, dateStr) : null;
+    // --- FIX: Pass pre-fetched refs to the generators ---
+    // Use the refs from fullRoot (which is rootData + yesterdayPortfolioRefs)
+    const yP_iterator = needsYesterdayPortfolio ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
+    // Use the historyRefs argument
+    const hT_iterator = needsTodayHistory ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
+    // --- END FIX ---
 
     let yesterdayPortfolios = {};
     let todayHistoryData = {};
@@ -183,7 +205,9 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
         logger.log('INFO', `[${passName}] Loaded first chunk of today's history.`);
     }
 
-    for await (const chunk of streamPortfolioData(config, deps, dateStr)) {
+    // --- FIX: Pass pre-fetched portfolioRefs to the main loop generator ---
+    for await (const chunk of streamPortfolioData(config, deps, dateStr, portfolioRefs)) {
+    // --- END FIX ---
         
         if (yP_iterator) { 
             Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {}); 
@@ -237,7 +261,10 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
     const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
     const state = initializeCalculators(calcs, logger);
-    await streamAndProcess(dStr, state, passName, config, deps, fullRoot);
+    
+    // --- FIX: Pass portfolioRefs and historyRefs from rootData ---
+    await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs);
+    // --- END FIX ---
         
     let success = 0;
     const failedCalcs = [];

package/functions/computation-system/utils/data_loader.js

@@ -2,6 +2,7 @@
  * @fileoverview Data loader sub-pipes for the Computation System.
  * REFACTORED: Now stateless and receive dependencies.
  * --- NEW: Added streamPortfolioData async generator ---
+ * --- FIXED: streamPortfolioData and streamHistoryData now accept optional 'providedRefs' ---
  */
 
 // <<< FIX: REMOVED all top-level 'require' and 'dependencies' lines >>>
@@ -173,10 +174,13 @@ async function getHistoryPartRefs(config, deps, dateString) {
  * @param {object} config - The computation system configuration object.
  * @param {object} deps - Contains db, logger, calculationUtils.
  * @param {string} dateString - The date in YYYY-MM-DD format.
+ * @param {Array<Firestore.DocumentReference> | null} [providedRefs=null] - Optional pre-fetched refs.
  */
-async function* streamPortfolioData(config, deps, dateString) {
+async function* streamPortfolioData(config, deps, dateString, providedRefs = null) {
     const { logger } = deps;
-    const refs = await getPortfolioPartRefs(config, deps, dateString);
+    // --- FIX: Use providedRefs if available, otherwise fetch them ---
+    const refs = providedRefs || (await getPortfolioPartRefs(config, deps, dateString));
+    
     if (refs.length === 0) {
         logger.log('WARN', `[streamPortfolioData] No portfolio refs found for ${dateString}. Stream is empty.`);
         return;
@@ -201,10 +205,16 @@ async function* streamPortfolioData(config, deps, dateString) {
 /**
  * --- NEW: Stage 8: Stream history data in chunks ---
  * Streams history data in chunks for a given date.
+ * @param {object} config - The computation system configuration object.
+ * @param {object} deps - Contains db, logger, calculationUtils.
+ * @param {string} dateString - The date in YYYY-MM-DD format.
+ * @param {Array<Firestore.DocumentReference> | null} [providedRefs=null] - Optional pre-fetched refs.
  */
-async function* streamHistoryData(config, deps, dateString) {
+async function* streamHistoryData(config, deps, dateString, providedRefs = null) {
     const { logger } = deps;
-    const refs = await getHistoryPartRefs(config, deps, dateString); // <-- Uses history refs
+    // --- FIX: Use providedRefs if available, otherwise fetch them ---
+    const refs = providedRefs || (await getHistoryPartRefs(config, deps, dateString)); // <-- Uses history refs
+    
     if (refs.length === 0) {
         logger.log('WARN', `[streamHistoryData] No history refs found for ${dateString}. Stream is empty.`);
         return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.150",
+  "version": "1.0.152",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [