bulltrackers-module 1.0.184 → 1.0.185

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -13,7 +13,7 @@ const {
     checkRootDependencies
 } = require('./orchestration_helpers.js');
 
-const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
+const { getExpectedDateStrings, normalizeName, getEarliestDataDates } = require('../utils/utils.js');
 
 const PARALLEL_BATCH_SIZE = 7;
 
@@ -25,8 +25,7 @@ async function runComputationPass(config, dependencies, computationManifest) {
     
     logger.log('INFO', `🚀 Starting PASS ${passToRun} (Targeting /computation_status/{YYYY-MM-DD})...`);
 
-    // Hardcoded earliest dates
-    const earliestDates = { portfolio: new Date('2025-09-25T00:00:00Z'), history: new Date('2025-11-05T00:00:00Z'), social: new Date('2025-10-30T00:00:00Z'), insights: new Date('2025-08-26T00:00:00Z') };
+    const earliestDates = await getEarliestDataDates(config, dependencies); // Now not hardcoded.
     earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a,b) => a < b ? a : b);
 
     const passes = groupByPass(computationManifest);

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -28,6 +28,7 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
         else if (dep === 'insights'  && !rootDataStatus.hasInsights)  missing.push('insights');
         else if (dep === 'social'    && !rootDataStatus.hasSocial)    missing.push('social');
         else if (dep === 'history'   && !rootDataStatus.hasHistory)   missing.push('history');
+        else if (dep === 'price'     && !rootDataStatus.hasPrices)    missing.push('price'); // NEW
     }
     return { canRun: missing.length === 0, missing };
 }
@@ -36,26 +37,34 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
  * Checks for the availability of all required root data for a specific date.
  */
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
-    const { logger }    = dependencies;
+    const { logger, db } = dependencies;
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-    let portfolioRefs   = [],    historyRefs = [];
-    let hasPortfolio    = false, hasInsights = false, hasSocial = false, hasHistory = false, insightsData = null , socialData  = null;
+    let portfolioRefs = [], historyRefs = [];
+    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
+    let insightsData = null, socialData = null;
 
     try {
         const tasks = [];
-        if (dateToProcess >= earliestDates.portfolio)  tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r =>        { portfolioRefs = r; hasPortfolio = !!r.length; }));
-        if (dateToProcess >= earliestDates.insights)   tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r =>           { insightsData  = r; hasInsights  = !!r; }));
-        if (dateToProcess >= earliestDates.social)     tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData    = r; hasSocial    = !!r; }));
-        if (dateToProcess >= earliestDates.history)    tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r =>          { historyRefs   = r; hasHistory   = !!r.length; }));
+        if (dateToProcess >= earliestDates.portfolio) tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio = !!r.length; }));
+        if (dateToProcess >= earliestDates.insights) tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
+        if (dateToProcess >= earliestDates.social) tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
+        if (dateToProcess >= earliestDates.history) tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
+        
+        // NEW: Check if price data exists (proper validation)
+        if (dateToProcess >= (earliestDates.price || earliestDates.absoluteEarliest)) {
+            tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; }));
+        }
         
         await Promise.all(tasks);
         
-        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) return null;
+        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory || hasPrices)) return null;
         
         return { 
-            portfolioRefs, historyRefs, 
-            todayInsights: insightsData, todaySocialPostInsights: socialData, 
-            status: { hasPortfolio, hasInsights, hasSocial, hasHistory } 
+            portfolioRefs, 
+            historyRefs, 
+            todayInsights: insightsData, 
+            todaySocialPostInsights: socialData, 
+            status: { hasPortfolio, hasInsights, hasSocial, hasHistory, hasPrices }
         };
 
     } catch (err) {
@@ -64,6 +73,40 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
     }
 }
 
+/**
+ * NEW HELPER: Check if price data collection has any data
+ */
+async function checkPriceDataAvailability(config, dependencies) {
+    const { db, logger } = dependencies;
+    const collection = config.priceCollection || 'asset_prices';
+    
+    try {
+        // Check if the collection has at least one shard document
+        const snapshot = await db.collection(collection).limit(1).get();
+        
+        if (snapshot.empty) {
+            logger.log('WARN', `[checkPriceData] No price shards found in ${collection}`);
+            return false;
+        }
+        
+        // Check if the first shard actually has price data
+        const firstDoc = snapshot.docs[0];
+        const data = firstDoc.data();
+        
+        if (!data || Object.keys(data).length === 0) {
+            logger.log('WARN', `[checkPriceData] Price shard exists but is empty`);
+            return false;
+        }
+        
+        logger.log('TRACE', `[checkPriceData] Price data available in ${collection}`);
+        return true;
+        
+    } catch (e) {
+        logger.log('ERROR', `[checkPriceData] Failed to check price availability: ${e.message}`);
+        return false;
+    }
+}
+
 async function fetchComputationStatus(dateStr, config, { db }) {
     const collection = config.computationStatusCollection || 'computation_status';
     const docRef     = db.collection(collection).doc(dateStr);

package/functions/computation-system/utils/utils.js

@@ -75,25 +75,125 @@ async function getFirstDateFromCollection(config, deps, collectionName) {
 async function getEarliestDataDates(config, deps) {
     const { logger } = deps;
     logger.log('INFO', 'Querying for earliest date from ALL source data collections...');
-    const [ investorDate, speculatorDate, investorHistoryDate, speculatorHistoryDate, insightsDate, socialDate ] = await Promise.all([
-        getFirstDateFromCollection      (config, deps, config.normalUserPortfolioCollection),
-        getFirstDateFromCollection      (config, deps, config.speculatorPortfolioCollection),
-        getFirstDateFromCollection      (config, deps, config.normalUserHistoryCollection),
-        getFirstDateFromCollection      (config, deps, config.speculatorHistoryCollection),
+    
+    const [ 
+        investorDate, 
+        speculatorDate, 
+        investorHistoryDate, 
+        speculatorHistoryDate, 
+        insightsDate, 
+        socialDate,
+        priceDate  // NEW
+    ] = await Promise.all([
+        getFirstDateFromCollection(config, deps, config.normalUserPortfolioCollection),
+        getFirstDateFromCollection(config, deps, config.speculatorPortfolioCollection),
+        getFirstDateFromCollection(config, deps, config.normalUserHistoryCollection),
+        getFirstDateFromCollection(config, deps, config.speculatorHistoryCollection),
         getFirstDateFromSimpleCollection(config, deps, config.insightsCollectionName),
-        getFirstDateFromSimpleCollection(config, deps, config.socialInsightsCollectionName)
+        getFirstDateFromSimpleCollection(config, deps, config.socialInsightsCollectionName),
+        getFirstDateFromPriceCollection(config, deps)  // NEW
     ]);
     
-    const getMinDate            = (...dates) => { const validDates = dates.filter(Boolean);  if (validDates.length === 0) return null; return new Date(Math.min(...validDates)); };
+    const getMinDate = (...dates) => { 
+        const validDates = dates.filter(Boolean); 
+        if (validDates.length === 0) return null; 
+        return new Date(Math.min(...validDates)); 
+    };
+    
     const earliestPortfolioDate = getMinDate(investorDate, speculatorDate);
-    const earliestHistoryDate   = getMinDate(investorHistoryDate, speculatorHistoryDate);
-    const earliestInsightsDate  = getMinDate(insightsDate);
-    const earliestSocialDate    = getMinDate(socialDate); 
-    const absoluteEarliest      = getMinDate(earliestPortfolioDate, earliestHistoryDate, earliestInsightsDate, earliestSocialDate );
-    const fallbackDate          = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
-    const result                = { portfolio: earliestPortfolioDate || new Date('2999-12-31'),  history: earliestHistoryDate || new Date('2999-12-31'), insights: earliestInsightsDate || new Date('2999-12-31'), social: earliestSocialDate || new Date('2999-12-31'), absoluteEarliest: absoluteEarliest || fallbackDate };
-    logger.log('INFO', 'Earliest data availability map built:', { portfolio: result.portfolio.toISOString().slice(0, 10), history: result.history.toISOString().slice(0, 10), insights: result.insights.toISOString().slice(0, 10), social: result.social.toISOString().slice(0, 10), absoluteEarliest: result.absoluteEarliest.toISOString().slice(0, 10) });
+    const earliestHistoryDate = getMinDate(investorHistoryDate, speculatorHistoryDate);
+    const earliestInsightsDate = getMinDate(insightsDate);
+    const earliestSocialDate = getMinDate(socialDate); 
+    const earliestPriceDate = getMinDate(priceDate);  // NEW
+    const absoluteEarliest = getMinDate(
+        earliestPortfolioDate, 
+        earliestHistoryDate, 
+        earliestInsightsDate, 
+        earliestSocialDate,
+        earliestPriceDate  // NEW
+    );
+    
+    const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
+    
+    const result = { 
+        portfolio: earliestPortfolioDate || new Date('2999-12-31'), 
+        history: earliestHistoryDate || new Date('2999-12-31'), 
+        insights: earliestInsightsDate || new Date('2999-12-31'), 
+        social: earliestSocialDate || new Date('2999-12-31'),
+        price: earliestPriceDate || new Date('2999-12-31'),  // NEW
+        absoluteEarliest: absoluteEarliest || fallbackDate 
+    };
+    
+    logger.log('INFO', 'Earliest data availability map built:', { 
+        portfolio: result.portfolio.toISOString().slice(0, 10), 
+        history: result.history.toISOString().slice(0, 10), 
+        insights: result.insights.toISOString().slice(0, 10), 
+        social: result.social.toISOString().slice(0, 10),
+        price: result.price.toISOString().slice(0, 10),  // NEW
+        absoluteEarliest: result.absoluteEarliest.toISOString().slice(0, 10) 
+    });
+    
     return result;
 }
 
+/**
+ * NEW HELPER: Get the earliest date from price collection
+ * Price data is sharded differently - each shard contains instrumentId -> {prices: {date: price}}
+ */
+async function getFirstDateFromPriceCollection(config, deps) {
+    const { db, logger, calculationUtils } = deps;
+    const { withRetry } = calculationUtils;
+    const collection = config.priceCollection || 'asset_prices';
+    
+    try {
+        logger.log('TRACE', `[getFirstDateFromPriceCollection] Querying ${collection}...`);
+        
+        // Get all shards (limit to first few for performance)
+        const snapshot = await withRetry(
+            () => db.collection(collection).limit(10).get(), 
+            `GetPriceShards(${collection})`
+        );
+        
+        if (snapshot.empty) {
+            logger.log('WARN', `No price shards found in ${collection}`);
+            return null;
+        }
+        
+        let earliestDate = null;
+        
+        // Iterate through shards to find the earliest date across all instruments
+        snapshot.forEach(doc => {
+            const shardData = doc.data();
+            
+            // Each shard has structure: { instrumentId: { ticker, prices: { "YYYY-MM-DD": price } } }
+            for (const instrumentId in shardData) {
+                const instrumentData = shardData[instrumentId];
+                if (!instrumentData.prices) continue;
+                
+                // Get all dates for this instrument
+                const dates = Object.keys(instrumentData.prices)
+                    .filter(d => /^\d{4}-\d{2}-\d{2}$/.test(d))
+                    .sort();
+                
+                if (dates.length > 0) {
+                    const firstDate = new Date(dates[0] + 'T00:00:00Z');
+                    if (!earliestDate || firstDate < earliestDate) {
+                        earliestDate = firstDate;
+                    }
+                }
+            }
+        });
+        
+        if (earliestDate) {
+            logger.log('TRACE', `[getFirstDateFromPriceCollection] Earliest price date: ${earliestDate.toISOString().slice(0, 10)}`);
+        }
+        
+        return earliestDate;
+        
+    } catch (e) {
+        logger.log('ERROR', `Failed to get earliest price date from ${collection}`, { errorMessage: e.message });
+        return null;
+    }
+}
+
 module.exports = { FieldValue, FieldPath, normalizeName, commitBatchInChunks, getExpectedDateStrings, getEarliestDataDates };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.184",
+  "version": "1.0.185",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [