bulltrackers-module 1.0.124 → 1.0.126

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -13,10 +13,11 @@
 const { FieldPath } = require('@google-cloud/firestore');
 const { 
     getPortfolioPartRefs, 
-    loadFullDayMap, 
+    loadFullDayMap, // <-- Ensure loadFullDayMap is imported
     loadDataByRefs, 
     loadDailyInsights,
-    loadDailySocialPostInsights
+    loadDailySocialPostInsights,
+    getHistoryPartRefs // <-- IMPORT NEW FUNCTION
 } = require('../utils/data_loader.js');
 
 const {
@@ -53,47 +54,53 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
         if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) return false;
         if (dep === 'insights' && !rootDataStatus.hasInsights) return false;
         if (dep === 'social' && !rootDataStatus.hasSocial) return false;
+        if (dep === 'history' && !rootDataStatus.hasHistory) return false; // <-- ADDED THIS LINE
     }
     return true; // All dependencies were met
 }
 
 
 /**
- * Checks if the root data (portfolios, insights, social) exists for a given day.
+ * Checks if the root data (portfolios, insights, social, history) exists for a given day.
  * @param {string} dateStr - The date string to check (YYYY-MM-DD).
  * @param {object} config - The computation system configuration object.
  * @param {object} dependencies - Contains db, logger, calculationUtils.
- * @returns {Promise<object>} { portfolioRefs, insightsData, socialData, hasPortfolio, hasInsights, hasSocial }
+ * @returns {Promise<object>} { portfolioRefs, insightsData, socialData, historyRefs, hasPortfolio, hasInsights, hasSocial, hasHistory }
  */
 async function checkRootDataAvailability(dateStr, config, dependencies) {
     const { logger } = dependencies;
     logger.log('INFO', `[PassRunner] Checking root data availability for ${dateStr}...`);
 
     try {
-        const [portfolioRefs, insightsData, socialData] = await Promise.all([
+        // --- MODIFIED: Add getHistoryPartRefs to the parallel load ---
+        const [portfolioRefs, insightsData, socialData, historyRefs] = await Promise.all([
             getPortfolioPartRefs(config, dependencies, dateStr),
             loadDailyInsights(config, dependencies, dateStr),
-            loadDailySocialPostInsights(config, dependencies, dateStr)
+            loadDailySocialPostInsights(config, dependencies, dateStr),
+            getHistoryPartRefs(config, dependencies, dateStr) // <-- ADD THIS
         ]);
 
         const hasPortfolio = (portfolioRefs && portfolioRefs.length > 0);
         const hasInsights = !!insightsData;
         const hasSocial = !!socialData;
+        const hasHistory = (historyRefs && historyRefs.length > 0); // <-- ADD THIS
 
         return {
             portfolioRefs: portfolioRefs || [],
             insightsData: insightsData || null,
             socialData: socialData || null,
+            historyRefs: historyRefs || [], // <-- ADD THIS
             hasPortfolio: hasPortfolio,
             hasInsights: hasInsights,
-            hasSocial: hasSocial
+            hasSocial: hasSocial,
+            hasHistory: hasHistory // <-- ADD THIS
         };
 
     } catch (err) {
         logger.log('ERROR', `[PassRunner] Error checking data availability for ${dateStr}`, { errorMessage: err.message });
         return { 
-            portfolioRefs: [], insightsData: null, socialData: null,
-            hasPortfolio: false, hasInsights: false, hasSocial: false 
+            portfolioRefs: [], insightsData: null, socialData: null, historyRefs: [], // <-- ADD historyRefs
+            hasPortfolio: false, hasInsights: false, hasSocial: false, hasHistory: false // <-- ADD hasHistory
         };
     }
 }
@@ -223,15 +230,16 @@ async function runComputationPass(config, dependencies, computationManifest) {
     for (const dateStr of allExpectedDates) {
         const dateToProcess = new Date(dateStr + 'T00:00:00Z');
         
-        // 1. Check for root data (portfolios, insights, social)
+        // 1. Check for root data (portfolios, insights, social, AND HISTORY)
         const rootData = await checkRootDataAvailability(dateStr, config, dependencies);
         const rootDataStatus = {
             hasPortfolio: rootData.hasPortfolio,
             hasInsights: rootData.hasInsights,
-            hasSocial: rootData.hasSocial
+            hasSocial: rootData.hasSocial,
+            hasHistory: rootData.hasHistory // <-- ADD THIS
         };
 
-        if (!rootData.hasPortfolio && !rootData.hasInsights && !rootData.hasSocial) {
+        if (!rootData.hasPortfolio && !rootData.hasInsights && !rootData.hasSocial && !rootData.hasHistory) {
             logger.log('WARN', `[PassRunner] Skipping Pass ${passToRun} for ${dateStr} due to missing all root data.`);
             continue; // Skip to the next day
         }
@@ -363,7 +371,9 @@ async function streamAndProcess(
     todayInsights = null, 
     yesterdayInsights = null,
     todaySocialPostInsights = null,
-    yesterdaySocialPostInsights = null
+    yesterdaySocialPostInsights = null,
+    todayHistoryData = null, // <-- ADD THIS
+    yesterdayHistoryData = null // <-- ADD THIS
 ) {
     const { logger, calculationUtils } = dependencies;
     logger.log('INFO', `[${passName}] Streaming ${todayRefs.length} 'today' part docs for ${dateStr}...`);
@@ -405,12 +415,16 @@ async function streamAndProcess(
                 const isHistoricalCalc = manifestCalc.isHistorical === true;
                 const isSpeculatorCalc = manifestCalc.category === 'speculators';
                 let processArgs;
+                
+                // --- MODIFIED: Add history data to the context args ---
                 const allContextArgs = [
                     context, 
                     todayInsights, 
                     yesterdayInsights, 
                     todaySocialPostInsights,
-                    yesterdaySocialPostInsights
+                    yesterdaySocialPostInsights,
+                    todayHistoryData,      // <-- ADD THIS
+                    yesterdayHistoryData   // <-- ADD THIS
                 ];
 
                 if (isSocialOrInsights) {
@@ -447,7 +461,18 @@ async function streamAndProcess(
     // Handle case where there are no users but we still need to run insights/social calcs
     if (todayRefs.length === 0 && isFirstUser) {
         logger.log('INFO', `[${passName}] No user portfolios found for ${dateStr}. Running insights/social calcs once.`);
-        const allContextArgs = [ context, todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights ];
+        
+        // --- MODIFIED: Add null history data ---
+        const allContextArgs = [ 
+            context, 
+            todayInsights, 
+            yesterdayInsights, 
+            todaySocialPostInsights, 
+            yesterdaySocialPostInsights,
+            todayHistoryData,      // <-- ADD THIS
+            yesterdayHistoryData   // <-- ADD THIS
+        ];
+        
         for (const calcName in state) {
             const calc = state[calcName];
             if (!calc || typeof calc.process !== 'function') continue;
@@ -474,23 +499,32 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
     logger.log('INFO', `[${passName}] Starting run for ${dateStr} with ${calculationsToRun.length} calcs.`);
 
     try {
+        // --- NEW: Get history root data from the orchestrator ---
         const {
             portfolioRefs: todayRefs,
             insightsData: todayInsightsData,
-            socialData: todaySocialPostInsightsData
+            socialData: todaySocialPostInsightsData,
+            historyRefs: todayHistoryRefs // <-- ADD THIS
         } = rootData;
         
         let yesterdayPortfolios = {};
         let yesterdayInsightsData = null; 
         let yesterdaySocialPostInsightsData = null; 
+        let todayHistoryData = null; // <-- ADD THIS
+        let yesterdayHistoryData = null; // <-- ADD THIS
         
+        // --- MODIFIED: Add checks for history data ---
         const requiresYesterdayPortfolio = calculationsToRun.some(c => c.isHistorical === true);
         const requiresYesterdayInsights = calculationsToRun.some(c => c.class.prototype.process.toString().includes('yesterdayInsights'));
         const requiresYesterdaySocialPosts = calculationsToRun.some(c => c.class.prototype.process.toString().includes('yesterdaySocialPostInsights'));
+        const requiresHistory = calculationsToRun.some(c => c.rootDataDependencies.includes('history'));
+        const requiresYesterdayHistory = calculationsToRun.some(c => c.isHistorical === true && c.class.prototype.process.toString().includes('yesterdayHistoryData'));
+        // --- END MODIFICATION ---
+
 
-        if (requiresYesterdayPortfolio || requiresYesterdayInsights || requiresYesterdaySocialPosts) {
-            // ... [Identical logic for fetching yesterday's data as in original file] ...
-            // This logic is complex and correct, so it is preserved.
+        // --- (This block is now much larger, handling all "yesterday" and "history" data) ---
+        if (requiresYesterdayPortfolio || requiresYesterdayInsights || requiresYesterdaySocialPosts || requiresHistory || requiresYesterdayHistory) { 
+            
              if(requiresYesterdayInsights) {
                 let daysAgo = 1; const maxLookback = 30;
                 while (!yesterdayInsightsData && daysAgo <= maxLookback) {
@@ -524,9 +558,32 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
                     logger.log('WARN', `[${passName}] Yesterday's (${prevStr}) portfolio data not found. Historical calcs requiring it will be skipped.`);
                 }
             }
+
+            // --- NEW: Add logic to load Today's History Data (if needed) ---
+            if (requiresHistory && todayHistoryRefs.length > 0) {
+                 logger.log('INFO', `[${passName}] Loading today's (${dateStr}) history map...`);
+                 todayHistoryData = await loadFullDayMap(config, dependencies, todayHistoryRefs);
+                 logger.log('INFO', `[${passName}] Loaded today's history map with ${Object.keys(todayHistoryData).length} users.`);
+            }
+            
+            // --- NEW: Add logic to load Yesterday's History Data (if needed) ---
+            if (requiresYesterdayHistory) {
+                const prev = new Date(dateToProcess);
+                prev.setUTCDate(prev.getUTCDate() - 1);
+                const prevStr = prev.toISOString().slice(0, 10);
+                const yesterdayHistoryRefs = await getHistoryPartRefs(config, dependencies, prevStr);
+                if (yesterdayHistoryRefs.length > 0) {
+                    yesterdayHistoryData = await loadFullDayMap(config, dependencies, yesterdayHistoryRefs);
+                     logger.log('INFO', `[${passName}] Loaded yesterday's (${prevStr}) history map for historical calcs.`);
+                } else {
+                    logger.log('WARN', `[${passName}] Yesterday's (${prevStr}) history data not found. Historical calcs requiring it will be skipped.`);
+                }
+            }
         }
         
         const state = initializeCalculators(calculationsToRun, logger);
+        
+        // --- MODIFIED: Pass new history data ---
         await streamAndProcess(
             dateStr, 
             todayRefs, 
@@ -538,7 +595,9 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
             todayInsightsData, 
             yesterdayInsightsData,
             todaySocialPostInsightsData,
-            yesterdaySocialPostInsightsData
+            yesterdaySocialPostInsightsData,
+            todayHistoryData,         // <-- ADD THIS
+            yesterdayHistoryData      // <-- ADD THIS
         );
 
         let successCount = 0;

package/functions/computation-system/utils/data_loader.js

@@ -183,10 +183,60 @@ async function loadDailySocialPostInsights(config, dependencies, dateString) {
 // --- END NEW ---
 
 
+
+/**
+ * --- NEW ---
+ * Sub-pipe: pipe.computationSystem.dataLoader.getHistoryPartRefs
+ * @param {object} config - The computation system configuration object.
+ * @param {object} dependencies - Contains db, logger, calculationUtils.
+ * @param {string} dateString - The date in YYYY-MM-DD format.
+ * @returns {Promise<Firestore.DocumentReference[]>} An array of DocumentReferences.
+ */
+async function getHistoryPartRefs(config, dependencies, dateString) {
+    const { db, logger, calculationUtils } = dependencies;
+    const { withRetry } = calculationUtils;
+
+    logger.log('INFO', `Getting history part references for date: ${dateString}`);
+    const allPartRefs = [];
+    // --- MODIFIED: Use new history collection config keys ---
+    const collectionsToQuery = [config.normalUserHistoryCollection, config.speculatorHistoryCollection];
+
+    for (const collectionName of collectionsToQuery) {
+        if (!collectionName) { // Add a check in case config is missing
+             logger.log('WARN', `History collection name is undefined. Skipping.`);
+             continue;
+        }
+        const blockDocsQuery = db.collection(collectionName);
+        const blockDocRefs = await withRetry(
+            () => blockDocsQuery.listDocuments(),
+            `listDocuments(${collectionName})`
+        );
+
+        if (blockDocRefs.length === 0) {
+             logger.log('WARN', `No block documents found in collection: ${collectionName}`);
+             continue;
+         }
+
+        for (const blockDocRef of blockDocRefs) {
+            const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection).doc(dateString).collection(config.partsSubcollection);
+            const partDocs = await withRetry(
+                () => partsCollectionRef.listDocuments(),
+                `listDocuments(${partsCollectionRef.path})`
+            );
+            allPartRefs.push(...partDocs);
+        }
+    }
+
+    logger.log('INFO', `Found ${allPartRefs.length} history part document references for ${dateString}.`);
+    return allPartRefs;
+}
+
+
 module.exports = {
     getPortfolioPartRefs,
     loadDataByRefs,
     loadFullDayMap,
     loadDailyInsights,
     loadDailySocialPostInsights, 
+    getHistoryPartRefs, // <-- EXPORT NEW FUNCTION
 };

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -120,10 +120,11 @@ class FirestoreBatchManager {
         this.tradingHistoryBatch[basePath][userId] = historyData;
 
         // Trigger flush based on portfolio batch size, assuming they'll be roughly equal
-        const totalUsersInBatch = Object.values(this.portfolioBatch).reduce((sum, users) => sum + Object.keys(users).length, 0);
+       const totalUsersInBatch = Object.values(this.tradingHistoryBatch).reduce((sum, users) => sum + Object.keys(users).length, 0);
 
         if (totalUsersInBatch >= this.config.TASK_ENGINE_MAX_BATCH_SIZE) {
-            await this.flushBatches();
+            // DO NOT await a flush. Just schedule one.
+            this._scheduleFlush();
         } else {
             this._scheduleFlush();
         }
@@ -153,7 +154,8 @@ class FirestoreBatchManager {
         const totalUsersInBatch = Object.values(this.portfolioBatch).reduce((sum, users) => sum + Object.keys(users).length, 0);
 
         if (totalUsersInBatch >= this.config.TASK_ENGINE_MAX_BATCH_SIZE) {
-            await this.flushBatches();
+            // DO NOT await a flush. Just schedule one.
+            this._scheduleFlush();
         } else {
             this._scheduleFlush();
         }
@@ -180,7 +182,8 @@ class FirestoreBatchManager {
         this.timestampBatch[docPath][timestampKey] = new Date();
 
         if (Object.keys(this.timestampBatch[docPath]).length >= this.config.TASK_ENGINE_MAX_BATCH_SIZE) {
-            await this.flushBatches();
+        // DO NOT await a flush. Just schedule one.
+            this._scheduleFlush();
         } else {
             this._scheduleFlush();
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.124",
+  "version": "1.0.126",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [