bulltrackers-module 1.0.183 → 1.0.185

package/functions/computation-system/controllers/computation_controller.js

@@ -1,6 +1,6 @@
 /**
  * FIXED: computation_controller.js
- * V3.3: Adds Price Loading for Meta Context & Fixes Context Injection
+ * V3.4: Adds Price Loading & Context Injection.
  */
 
 const { DataExtractor, 
@@ -55,34 +55,25 @@ class DataLoader {
         
         try {
             const snapshot = await db.collection(collection).get();
-            if (snapshot.empty) return { history: [] };
+            if (snapshot.empty) return { history: {} };
 
-            // Flatten shards into a single array for the context
-            // Structure expected by calculation: Array of { instrumentId, prices: {...} }
-            const allPrices = [];
+            const historyMap = {};
             
             snapshot.forEach(doc => {
                 const shardData = doc.data();
-                // Iterate keys in shard (instrumentIds)
-                for (const [instId, data] of Object.entries(shardData)) {
-                    if (data && data.prices) {
-                        allPrices.push({
-                            instrumentId: instId,
-                            ...data 
-                        });
-                    }
-                }
+                // Merge shard keys (instrumentIds) into main map
+                if (shardData) Object.assign(historyMap, shardData);
             });
 
-            logger.log('INFO', `[DataLoader] Loaded prices for ${allPrices.length} instruments.`);
+            logger.log('INFO', `[DataLoader] Loaded prices for ${Object.keys(historyMap).length} instruments.`);
             
-            // Cache as an object with 'history' array to match context expectations
-            this.cache.prices = { history: allPrices };
+            // Cache as an object with 'history' map to match priceExtractor expectations
+            this.cache.prices = { history: historyMap };
             return this.cache.prices;
 
         } catch (e) {
             logger.log('ERROR', `[DataLoader] Failed to load prices: ${e.message}`);
-            return { history: [] };
+            return { history: {} };
         }
     }
 }

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -13,7 +13,7 @@ const {
     checkRootDependencies
 } = require('./orchestration_helpers.js');
 
-const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
+const { getExpectedDateStrings, normalizeName, getEarliestDataDates } = require('../utils/utils.js');
 
 const PARALLEL_BATCH_SIZE = 7;
 
@@ -25,8 +25,7 @@ async function runComputationPass(config, dependencies, computationManifest) {
     
     logger.log('INFO', `🚀 Starting PASS ${passToRun} (Targeting /computation_status/{YYYY-MM-DD})...`);
 
-    // Hardcoded earliest dates
-    const earliestDates = { portfolio: new Date('2025-09-25T00:00:00Z'), history: new Date('2025-11-05T00:00:00Z'), social: new Date('2025-10-30T00:00:00Z'), insights: new Date('2025-08-26T00:00:00Z') };
+    const earliestDates = await getEarliestDataDates(config, dependencies); // Now not hardcoded.
     earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a,b) => a < b ? a : b);
 
     const passes = groupByPass(computationManifest);

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,6 +1,6 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/orchestration_helpers.js
- * FIXED: Only marks computations as TRUE if they actually store results.
+ * FIXED: Explicit Logging + Honest Status Updates
  */
 
 const { ComputationController }              = require('../controllers/computation_controller');
@@ -28,8 +28,7 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
         else if (dep === 'insights'  && !rootDataStatus.hasInsights)  missing.push('insights');
         else if (dep === 'social'    && !rootDataStatus.hasSocial)    missing.push('social');
         else if (dep === 'history'   && !rootDataStatus.hasHistory)   missing.push('history');
-        // Note: 'price' is typically not a blocking root check in this specific function logic unless added,
-        // but usually prices are treated as auxiliary. If you want to block on prices, add it here.
+        else if (dep === 'price'     && !rootDataStatus.hasPrices)    missing.push('price'); // NEW
     }
     return { canRun: missing.length === 0, missing };
 }
@@ -38,27 +37,34 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
  * Checks for the availability of all required root data for a specific date.
  */
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
-    const { logger }    = dependencies;
+    const { logger, db } = dependencies;
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-    let portfolioRefs   = [],    historyRefs = [];
-    let hasPortfolio    = false, hasInsights = false, hasSocial = false, hasHistory = false, insightsData = null , socialData  = null;
+    let portfolioRefs = [], historyRefs = [];
+    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
+    let insightsData = null, socialData = null;
 
     try {
         const tasks = [];
-        if (dateToProcess >= earliestDates.portfolio)  tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r =>        { portfolioRefs = r; hasPortfolio = !!r.length; }));
-        if (dateToProcess >= earliestDates.insights)   tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r =>           { insightsData  = r; hasInsights  = !!r; }));
-        if (dateToProcess >= earliestDates.social)     tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData    = r; hasSocial    = !!r; }));
-        if (dateToProcess >= earliestDates.history)    tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r =>          { historyRefs   = r; hasHistory   = !!r.length; }));
+        if (dateToProcess >= earliestDates.portfolio) tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio = !!r.length; }));
+        if (dateToProcess >= earliestDates.insights) tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
+        if (dateToProcess >= earliestDates.social) tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
+        if (dateToProcess >= earliestDates.history) tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
+        
+        // NEW: Check if price data exists (proper validation)
+        if (dateToProcess >= (earliestDates.price || earliestDates.absoluteEarliest)) {
+            tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; }));
+        }
         
         await Promise.all(tasks);
         
-        // We allow running if ANY data is present. Specific calcs filter themselves using checkRootDependencies.
-        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) return null;
+        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory || hasPrices)) return null;
         
         return { 
-            portfolioRefs, historyRefs, 
-            todayInsights: insightsData, todaySocialPostInsights: socialData, 
-            status: { hasPortfolio, hasInsights, hasSocial, hasHistory } 
+            portfolioRefs, 
+            historyRefs, 
+            todayInsights: insightsData, 
+            todaySocialPostInsights: socialData, 
+            status: { hasPortfolio, hasInsights, hasSocial, hasHistory, hasPrices }
         };
 
     } catch (err) {
@@ -67,6 +73,40 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
     }
 }
 
+/**
+ * NEW HELPER: Check if price data collection has any data
+ */
+async function checkPriceDataAvailability(config, dependencies) {
+    const { db, logger } = dependencies;
+    const collection = config.priceCollection || 'asset_prices';
+    
+    try {
+        // Check if the collection has at least one shard document
+        const snapshot = await db.collection(collection).limit(1).get();
+        
+        if (snapshot.empty) {
+            logger.log('WARN', `[checkPriceData] No price shards found in ${collection}`);
+            return false;
+        }
+        
+        // Check if the first shard actually has price data
+        const firstDoc = snapshot.docs[0];
+        const data = firstDoc.data();
+        
+        if (!data || Object.keys(data).length === 0) {
+            logger.log('WARN', `[checkPriceData] Price shard exists but is empty`);
+            return false;
+        }
+        
+        logger.log('TRACE', `[checkPriceData] Price data available in ${collection}`);
+        return true;
+        
+    } catch (e) {
+        logger.log('ERROR', `[checkPriceData] Failed to check price availability: ${e.message}`);
+        return false;
+    }
+}
+
 async function fetchComputationStatus(dateStr, config, { db }) {
     const collection = config.computationStatusCollection || 'computation_status';
     const docRef     = db.collection(collection).doc(dateStr);
@@ -205,6 +245,9 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
             const inst = new c.class(); 
             inst.manifest = c; 
             state[normalizeName(c.name)] = inst; 
+            
+            // LOG: Explicitly say what calculation is being processed (Initialized)
+            logger.log('INFO', `${c.name} calculation running for ${dStr}`);
         } 
         catch(e)  { 
             logger.log('WARN', `Failed to init ${c.name}`); 
@@ -223,6 +266,9 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
 
     for (const mCalc of calcs) {
         try {
+            // LOG: Explicitly say what calculation is being processed
+            deps.logger.log('INFO', `${mCalc.name} calculation running for ${dStr}`);
+
             const inst = new mCalc.class();
             inst.manifest = mCalc;
             await controller.executor.executeOncePerDay(inst, mCalc, dStr, fetchedDeps, previousFetchedDeps);
@@ -234,8 +280,8 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
 }
 
 /**
- * --- FIXED: commitResults ---
- * Only marks 'successUpdates' if data is actually written.
+ * --- UPDATED: commitResults ---
+ * Includes Explicit Result Logging and Honest Status Reporting.
  */
 async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
     const writes = [], schemas = [], sharded = {};
@@ -245,10 +291,15 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
         const calc = stateObj[name];
         try {
             const result = await calc.getResult(); 
-            if (!result) continue;
+            
+            // If null/undefined, log as Failed/Unknown immediately
+            if (!result) {
+                deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Failed (Empty Result)`);
+                continue;
+            }
             
             const standardRes = {};
-            let hasData = false; // Track if this calc produced data
+            let hasData = false;
 
             for (const key in result) {
                 if (key.startsWith('sharded_')) {
@@ -284,15 +335,19 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 });
             }
             
-            // FIX: Only mark as successful if we actually had data to write.
+            // --- EXPLICIT LOGGING & STATUS UPDATE ---
             if (hasData) {
                 successUpdates[name] = true;
+                deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Succeeded`);
             } else {
-                // Optional: Log that it produced no data
-                // deps.logger.log('INFO', `Calc ${name} produced no data. Skipping status update.`);
+                // It ran without error, but produced no content (e.g. no data met criteria)
+                deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Unknown (No Data Written)`);
             }
 
-        } catch (e) { deps.logger.log('ERROR', `Commit failed ${name}: ${e.message}`); }
+        } catch (e) { 
+            deps.logger.log('ERROR', `Commit failed ${name}: ${e.message}`);
+            deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Failed (Exception)`);
+        }
     }
 
     if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(()=>{});

package/functions/computation-system/utils/utils.js

@@ -75,25 +75,125 @@ async function getFirstDateFromCollection(config, deps, collectionName) {
 async function getEarliestDataDates(config, deps) {
     const { logger } = deps;
     logger.log('INFO', 'Querying for earliest date from ALL source data collections...');
-    const [ investorDate, speculatorDate, investorHistoryDate, speculatorHistoryDate, insightsDate, socialDate ] = await Promise.all([
-        getFirstDateFromCollection      (config, deps, config.normalUserPortfolioCollection),
-        getFirstDateFromCollection      (config, deps, config.speculatorPortfolioCollection),
-        getFirstDateFromCollection      (config, deps, config.normalUserHistoryCollection),
-        getFirstDateFromCollection      (config, deps, config.speculatorHistoryCollection),
+    
+    const [ 
+        investorDate, 
+        speculatorDate, 
+        investorHistoryDate, 
+        speculatorHistoryDate, 
+        insightsDate, 
+        socialDate,
+        priceDate  // NEW
+    ] = await Promise.all([
+        getFirstDateFromCollection(config, deps, config.normalUserPortfolioCollection),
+        getFirstDateFromCollection(config, deps, config.speculatorPortfolioCollection),
+        getFirstDateFromCollection(config, deps, config.normalUserHistoryCollection),
+        getFirstDateFromCollection(config, deps, config.speculatorHistoryCollection),
         getFirstDateFromSimpleCollection(config, deps, config.insightsCollectionName),
-        getFirstDateFromSimpleCollection(config, deps, config.socialInsightsCollectionName)
+        getFirstDateFromSimpleCollection(config, deps, config.socialInsightsCollectionName),
+        getFirstDateFromPriceCollection(config, deps)  // NEW
     ]);
     
-    const getMinDate            = (...dates) => { const validDates = dates.filter(Boolean);  if (validDates.length === 0) return null; return new Date(Math.min(...validDates)); };
+    const getMinDate = (...dates) => { 
+        const validDates = dates.filter(Boolean); 
+        if (validDates.length === 0) return null; 
+        return new Date(Math.min(...validDates)); 
+    };
+    
     const earliestPortfolioDate = getMinDate(investorDate, speculatorDate);
-    const earliestHistoryDate   = getMinDate(investorHistoryDate, speculatorHistoryDate);
-    const earliestInsightsDate  = getMinDate(insightsDate);
-    const earliestSocialDate    = getMinDate(socialDate); 
-    const absoluteEarliest      = getMinDate(earliestPortfolioDate, earliestHistoryDate, earliestInsightsDate, earliestSocialDate );
-    const fallbackDate          = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
-    const result                = { portfolio: earliestPortfolioDate || new Date('2999-12-31'),  history: earliestHistoryDate || new Date('2999-12-31'), insights: earliestInsightsDate || new Date('2999-12-31'), social: earliestSocialDate || new Date('2999-12-31'), absoluteEarliest: absoluteEarliest || fallbackDate };
-    logger.log('INFO', 'Earliest data availability map built:', { portfolio: result.portfolio.toISOString().slice(0, 10), history: result.history.toISOString().slice(0, 10), insights: result.insights.toISOString().slice(0, 10), social: result.social.toISOString().slice(0, 10), absoluteEarliest: result.absoluteEarliest.toISOString().slice(0, 10) });
+    const earliestHistoryDate = getMinDate(investorHistoryDate, speculatorHistoryDate);
+    const earliestInsightsDate = getMinDate(insightsDate);
+    const earliestSocialDate = getMinDate(socialDate); 
+    const earliestPriceDate = getMinDate(priceDate);  // NEW
+    const absoluteEarliest = getMinDate(
+        earliestPortfolioDate, 
+        earliestHistoryDate, 
+        earliestInsightsDate, 
+        earliestSocialDate,
+        earliestPriceDate  // NEW
+    );
+    
+    const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
+    
+    const result = { 
+        portfolio: earliestPortfolioDate || new Date('2999-12-31'), 
+        history: earliestHistoryDate || new Date('2999-12-31'), 
+        insights: earliestInsightsDate || new Date('2999-12-31'), 
+        social: earliestSocialDate || new Date('2999-12-31'),
+        price: earliestPriceDate || new Date('2999-12-31'),  // NEW
+        absoluteEarliest: absoluteEarliest || fallbackDate 
+    };
+    
+    logger.log('INFO', 'Earliest data availability map built:', { 
+        portfolio: result.portfolio.toISOString().slice(0, 10), 
+        history: result.history.toISOString().slice(0, 10), 
+        insights: result.insights.toISOString().slice(0, 10), 
+        social: result.social.toISOString().slice(0, 10),
+        price: result.price.toISOString().slice(0, 10),  // NEW
+        absoluteEarliest: result.absoluteEarliest.toISOString().slice(0, 10) 
+    });
+    
     return result;
 }
 
+/**
+ * NEW HELPER: Get the earliest date from price collection
+ * Price data is sharded differently - each shard contains instrumentId -> {prices: {date: price}}
+ */
+async function getFirstDateFromPriceCollection(config, deps) {
+    const { db, logger, calculationUtils } = deps;
+    const { withRetry } = calculationUtils;
+    const collection = config.priceCollection || 'asset_prices';
+    
+    try {
+        logger.log('TRACE', `[getFirstDateFromPriceCollection] Querying ${collection}...`);
+        
+        // Get all shards (limit to first few for performance)
+        const snapshot = await withRetry(
+            () => db.collection(collection).limit(10).get(), 
+            `GetPriceShards(${collection})`
+        );
+        
+        if (snapshot.empty) {
+            logger.log('WARN', `No price shards found in ${collection}`);
+            return null;
+        }
+        
+        let earliestDate = null;
+        
+        // Iterate through shards to find the earliest date across all instruments
+        snapshot.forEach(doc => {
+            const shardData = doc.data();
+            
+            // Each shard has structure: { instrumentId: { ticker, prices: { "YYYY-MM-DD": price } } }
+            for (const instrumentId in shardData) {
+                const instrumentData = shardData[instrumentId];
+                if (!instrumentData.prices) continue;
+                
+                // Get all dates for this instrument
+                const dates = Object.keys(instrumentData.prices)
+                    .filter(d => /^\d{4}-\d{2}-\d{2}$/.test(d))
+                    .sort();
+                
+                if (dates.length > 0) {
+                    const firstDate = new Date(dates[0] + 'T00:00:00Z');
+                    if (!earliestDate || firstDate < earliestDate) {
+                        earliestDate = firstDate;
+                    }
+                }
+            }
+        });
+        
+        if (earliestDate) {
+            logger.log('TRACE', `[getFirstDateFromPriceCollection] Earliest price date: ${earliestDate.toISOString().slice(0, 10)}`);
+        }
+        
+        return earliestDate;
+        
+    } catch (e) {
+        logger.log('ERROR', `Failed to get earliest price date from ${collection}`, { errorMessage: e.message });
+        return null;
+    }
+}
+
 module.exports = { FieldValue, FieldPath, normalizeName, commitBatchInChunks, getExpectedDateStrings, getEarliestDataDates };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.183",
+  "version": "1.0.185",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [