bulltrackers-module 1.0.131 → 1.0.132

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -6,9 +6,11 @@
  * It reads its pass number from the config and executes only those calculations.
  * This file contains the high-level "manual" of steps. The "how-to" logic
  * is extracted into 'computation_system_utils.js'.
+ * --- MODIFIED: To use getEarliestDataDates and pass the date map to the orchestrator helpers. ---
  */
 const { groupByPass, checkRootDataAvailability, fetchDependenciesForPass, filterCalculations, runStandardComputationPass, runMetaComputationPass } = require('./orchestration_helpers.js');
-const { getExpectedDateStrings, getFirstDateFromSourceData } = require('../utils/utils.js');
+// --- MODIFIED: Import getEarliestDataDates ---
+const { getExpectedDateStrings, getEarliestDataDates } = require('../utils/utils.js');
 
 async function runComputationPass(config, dependencies, computationManifest) {
     const { logger } = dependencies;
@@ -17,7 +19,12 @@ async function runComputationPass(config, dependencies, computationManifest) {
 
     const yesterday = new Date(); yesterday.setUTCDate(yesterday.getUTCDate()-1); 
     const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
-    const firstDate = await getFirstDateFromSourceData(config, dependencies);
+    
+    // --- MODIFIED: Call new date function ---
+    const earliestDates = await getEarliestDataDates(config, dependencies);
+    const firstDate = earliestDates.absoluteEarliest; // Use the absolute earliest for the loop
+    // --- END MODIFICATION ---
+    
     const startDateUTC = firstDate ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate())) : new Date(config.earliestComputationDate+'T00:00:00Z');
     const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
 
@@ -30,7 +37,10 @@ async function runComputationPass(config, dependencies, computationManifest) {
     for (const dateStr of allExpectedDates) {
         const dateToProcess = new Date(dateStr+'T00:00:00Z');
         try {
-            const rootData = await checkRootDataAvailability(dateStr, config, dependencies); if (!rootData) continue;
+            // --- MODIFIED: Pass earliestDates map to checkRootDataAvailability ---
+            const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates); if (!rootData) continue;
+            // --- END MODIFICATION ---
+            
             const fetchedDeps = await fetchDependenciesForPass(dateStr, calcsInThisPass, computationManifest, config, dependencies);
             const { standardCalcsToRun, metaCalcsToRun } = filterCalculations(standardCalcs, metaCalcs, rootData.status, fetchedDeps, passToRun, dateStr, logger);
             if (standardCalcsToRun.length) await runStandardComputationPass(dateToProcess, standardCalcsToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData);
@@ -44,4 +54,4 @@ async function runComputationPass(config, dependencies, computationManifest) {
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 
-module.exports = { runComputationPass };
+module.exports = { runComputationPass };

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -5,30 +5,96 @@ const { normalizeName, commitBatchInChunks } = require('../utils/utils.js');
 /** Stage 1: Group manifest by pass number */
 function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
 
-/** Stage 2: Check root data dependencies for a calc */
-function checkRootDependencies(calcManifest, rootDataStatus) { if (!calcManifest.rootDataDependencies || !calcManifest.rootDataDependencies.length) return true;
-    for (const dep of calcManifest.rootDataDependencies) if ((dep==='portfolio'&&!rootDataStatus.hasPortfolio)||(dep==='insights'&&!rootDataStatus.hasInsights)||(dep==='social'&&!rootDataStatus.hasSocial)||(dep==='history'&&!rootDataStatus.hasHistory)) return false;
-    return true;
+/** * --- MODIFIED: Returns detailed missing dependencies for logging ---
+ * Stage 2: Check root data dependencies for a calc 
+ */
+function checkRootDependencies(calcManifest, rootDataStatus) {
+    const missing = [];
+    if (!calcManifest.rootDataDependencies || !calcManifest.rootDataDependencies.length) {
+        return { canRun: true, missing };
+    }
+    for (const dep of calcManifest.rootDataDependencies) {
+        if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
+        else if (dep === 'insights' && !rootDataStatus.hasInsights) missing.push('insights');
+        else if (dep === 'social' && !rootDataStatus.hasSocial) missing.push('social');
+        else if (dep === 'history' && !rootDataStatus.hasHistory) missing.push('history');
+    }
+    return { canRun: missing.length === 0, missing };
 }
 
-/** Stage 3: Check root data availability for a date */
-// --- FIX: Passes the full 'dependencies' object down ---
-async function checkRootDataAvailability(dateStr, config, dependencies) {
-    const { logger } = dependencies; // Destructure logger for local use
+/** * --- MODIFIED: Uses earliestDates map to avoid unnecessary queries ---
+ * Stage 3: Check root data availability for a date 
+ */
+async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
+    const { logger } = dependencies;
     logger.log('INFO', `[PassRunner] Checking root data for ${dateStr}...`);
+
+    const dateToProcess = new Date(dateStr + 'T00:00:00Z');
+    let portfolioRefs = [], insightsData = null, socialData = null, historyRefs = [];
+    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false;
+
     try {
-        const [portfolioRefs, insightsData, socialData, historyRefs] = await Promise.all([
-            getPortfolioPartRefs(config, dependencies, dateStr),        // Pass full 'dependencies'
-            loadDailyInsights(config, dependencies, dateStr),         // Pass full 'dependencies'
-            loadDailySocialPostInsights(config, dependencies, dateStr), // Pass full 'dependencies'
-            getHistoryPartRefs(config, dependencies, dateStr)           // Pass full 'dependencies'
-        ]);
-        const hasPortfolio = !!(portfolioRefs?.length), hasInsights = !!insightsData, hasSocial = !!socialData, hasHistory = !!(historyRefs?.length);
-        if (!(hasPortfolio||hasInsights||hasSocial||hasHistory)) { logger.log('WARN', `[PassRunner] No root data for ${dateStr}.`); return null; }
-        return { portfolioRefs: portfolioRefs||[], insightsData: insightsData||null, socialData: socialData||null, historyRefs: historyRefs||[], status: { hasPortfolio, hasInsights, hasSocial, hasHistory } };
-    } catch (err) { logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message }); return null; }
+        const tasks = [];
+
+        if (dateToProcess >= earliestDates.portfolio) {
+            tasks.push(
+                getPortfolioPartRefs(config, dependencies, dateStr).then(res => {
+                    portfolioRefs = res;
+                    hasPortfolio = !!(res?.length);
+                })
+            );
+        }
+
+        if (dateToProcess >= earliestDates.insights) {
+            tasks.push(
+                loadDailyInsights(config, dependencies, dateStr).then(res => {
+                    insightsData = res;
+                    hasInsights = !!res;
+                })
+            );
+        }
+
+        if (dateToProcess >= earliestDates.social) {
+            tasks.push(
+                loadDailySocialPostInsights(config, dependencies, dateStr).then(res => {
+                    socialData = res;
+                    hasSocial = !!res;
+                })
+            );
+        }
+
+        if (dateToProcess >= earliestDates.history) {
+            tasks.push(
+                getHistoryPartRefs(config, dependencies, dateStr).then(res => {
+                    historyRefs = res;
+                    hasHistory = !!(res?.length);
+                })
+            );
+        }
+
+        await Promise.all(tasks);
+
+        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) {
+            logger.log('WARN', `[PassRunner] No root data for ${dateStr}.`);
+            return null;
+        }
+
+        return {
+            portfolioRefs,
+            insightsData,
+            socialData,
+            historyRefs,
+            status: { hasPortfolio, hasInsights, hasSocial, hasHistory }
+        };
+
+    } catch (err) {
+        logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message });
+        return null;
+    }
 }
 
+
+
 /** Stage 4: Fetch computed dependencies from Firestore */
 async function fetchDependenciesForPass(dateStr, calcsInPass, fullManifest, config, { db, logger }) {
     const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
@@ -47,14 +113,47 @@ async function fetchDependenciesForPass(dateStr, calcsInPass, fullManifest, conf
     return fetched;
 }
 
-/** Stage 5: Filter calculations based on available root data and dependencies */
+/** * --- MODIFIED: Added detailed logging ---
+ * Stage 5: Filter calculations based on available root data and dependencies 
+ */
 function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, fetchedDeps, passToRun, dateStr, logger) {
     const skipped = new Set();
-    const standardCalcsToRun = standardCalcs.filter(c => checkRootDependencies(c, rootDataStatus) || (logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} missing root data`), skipped.add(c.name), false));
-    const metaCalcsToRun = metaCalcs.filter(c => checkRootDependencies(c, rootDataStatus) && (c.dependencies||[]).every(d=>fetchedDeps[normalizeName(d)]) || (logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${c.name} missing dep`), skipped.add(c.name), false));
+
+    // Filter Standard Calcs
+    const standardCalcsToRun = standardCalcs.filter(c => {
+        const { canRun, missing } = checkRootDependencies(c, rootDataStatus);
+        if (canRun) return true;
+        
+        logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Missing root data: [${missing.join(', ')}]`);
+        skipped.add(c.name);
+        return false;
+    });
+
+    // Filter Meta Calcs
+    const metaCalcsToRun = metaCalcs.filter(c => {
+        // 1. Check root data
+        const { canRun, missing: missingRoot } = checkRootDependencies(c, rootDataStatus);
+        if (!canRun) {
+            logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing root data: [${missingRoot.join(', ')}]`);
+            skipped.add(c.name);
+            return false;
+        }
+        
+        // 2. Check computed dependencies
+        const missingDeps = (c.dependencies || []).map(normalizeName).filter(d => !fetchedDeps[d]);
+        if (missingDeps.length > 0) {
+            logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`);
+            skipped.add(c.name);
+            return false;
+        }
+        
+        return true; // All checks passed
+    });
+    
     return { standardCalcsToRun, metaCalcsToRun };
 }
 
+
 /** Stage 6: Initialize calculator instances */
 function initializeCalculators(calcs, logger) { const state = {}; for (const c of calcs) { const name=normalizeName(c.name), Cl=c.class; if(typeof Cl==='function') try { const inst=new Cl(); inst.manifest=c; state[name]=inst; } catch(e){logger.warn(`Init failed ${name}`,{errorMessage:e.message}); state[name]=null;} else {logger.warn(`Class missing ${name}`); state[name]=null;} } return state; }
 

package/functions/computation-system/utils/utils.js

@@ -2,8 +2,10 @@
  * @fileoverview Computation system sub-pipes and utils.
  * REFACTORED: Now stateless and receive dependencies where needed.
  * DYNAMIC: Categorization logic is removed, replaced by manifest.
+ * --- MODIFIED: getFirstDateFromSourceData is now getEarliestDataDates
+ * and queries all data sources to build an availability map. ---
  */
-/** --- Computation System Sub-Pipes & Utils (Stateless) --- */
+/** --- Computation System Sub-Pipes & Utils (Stateless, Dependency-Injection) --- */
 
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
 
@@ -46,13 +48,46 @@ function getExpectedDateStrings(startDate, endDate) {
     return dateStrings;
 }
 
-/** Stage 4: Get the earliest date in a collection */
+/**
+ * --- NEW HELPER ---
+ * Stage 4: Get the earliest date in a *flat* collection where doc IDs are dates.
+ */
+async function getFirstDateFromSimpleCollection(config, deps, collectionName) {
+    const { db, logger, calculationUtils } = deps;
+    const { withRetry } = calculationUtils;
+
+    try {
+        if (!collectionName) {
+            logger.log('WARN', `[Core Utils] Collection name not provided for simple date query.`);
+            return null;
+        }
+        const query = db.collection(collectionName)
+            .where(FieldPath.documentId(), '>=', '2000-01-01')
+            .orderBy(FieldPath.documentId(), 'asc')
+            .limit(1);
+        
+        const snapshot = await withRetry(() => query.get(), `GetEarliestDoc(${collectionName})`);
+        
+        if (!snapshot.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshot.docs[0].id)) {
+            return new Date(snapshot.docs[0].id + 'T00:00:00Z');
+        }
+    } catch (e) {
+        logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message });
+    }
+    return null;
+}
+
+/** Stage 4: Get the earliest date in a sharded collection */
 async function getFirstDateFromCollection(config, deps, collectionName) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
 
     let earliestDate = null;
     try {
+        if (!collectionName) {
+            logger.log('WARN', `[Core Utils] Collection name not provided for sharded date query.`);
+            return null;
+        }
         const blockDocRefs = await withRetry(() => db.collection(collectionName).listDocuments(), `GetBlocks(${collectionName})`);
         if (!blockDocRefs.length) { logger.log('WARN', `No block documents in collection: ${collectionName}`); return null; }
 
@@ -75,26 +110,69 @@ async function getFirstDateFromCollection(config, deps, collectionName) {
     return earliestDate;
 }
 
-/** Stage 5: Determine the earliest date from source data across both user types */
-async function getFirstDateFromSourceData(config, deps) {
+/** * --- MODIFIED FUNCTION ---
+ * Stage 5: Determine the earliest date from *all* source data.
+ */
+async function getEarliestDataDates(config, deps) {
     const { logger } = deps;
-    logger.log('INFO', 'Querying for earliest date from source portfolio data...');
-
-    const investorDate = await getFirstDateFromCollection(config, deps, config.normalUserPortfolioCollection);
-    const speculatorDate = await getFirstDateFromCollection(config, deps, config.speculatorPortfolioCollection);
-
-    let earliestDate;
-    if (investorDate && speculatorDate) earliestDate = investorDate < speculatorDate ? investorDate : speculatorDate;
-    else earliestDate = investorDate || speculatorDate;
-
-    if (earliestDate) {
-        logger.log('INFO', `Found earliest source data date: ${earliestDate.toISOString().slice(0, 10)}`);
-        return earliestDate;
-    } else {
-        const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
-        logger.log('WARN', `No source data found. Defaulting first date to: ${fallbackDate.toISOString().slice(0, 10)}`);
-        return fallbackDate;
-    }
+    logger.log('INFO', 'Querying for earliest date from ALL source data collections...');
+
+    // These return null on error or if empty
+    const [
+        investorDate,
+        speculatorDate,
+        investorHistoryDate,
+        speculatorHistoryDate,
+        insightsDate,
+        socialDate
+    ] = await Promise.all([
+        getFirstDateFromCollection(config, deps, config.normalUserPortfolioCollection),
+        getFirstDateFromCollection(config, deps, config.speculatorPortfolioCollection),
+        getFirstDateFromCollection(config, deps, config.normalUserHistoryCollection),
+        getFirstDateFromCollection(config, deps, config.speculatorHistoryCollection),
+        getFirstDateFromSimpleCollection(config, deps, config.insightsCollectionName),
+        getFirstDateFromSimpleCollection(config, deps, config.socialInsightsCollectionName)
+    ]);
+    
+    // Helper to find the minimum (earliest) of a set of dates
+    const getMinDate = (...dates) => {
+        const validDates = dates.filter(Boolean); // Filter out nulls
+        if (validDates.length === 0) return null;
+        return new Date(Math.min(...validDates));
+    };
+
+    const earliestPortfolioDate = getMinDate(investorDate, speculatorDate);
+    const earliestHistoryDate = getMinDate(investorHistoryDate, speculatorHistoryDate);
+    const earliestInsightsDate = getMinDate(insightsDate); // Already a single date
+    const earliestSocialDate = getMinDate(socialDate); // Already a single date
+
+    const absoluteEarliest = getMinDate(
+        earliestPortfolioDate,
+        earliestHistoryDate,
+        earliestInsightsDate,
+        earliestSocialDate
+    );
+
+    // Fallback date
+    const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
+
+    const result = {
+        portfolio: earliestPortfolioDate || new Date('2999-12-31'), // Use a 'far future' date if null
+        history: earliestHistoryDate || new Date('2999-12-31'),
+        insights: earliestInsightsDate || new Date('2999-12-31'),
+        social: earliestSocialDate || new Date('2999-12-31'),
+        absoluteEarliest: absoluteEarliest || fallbackDate // Use fallback for the main loop
+    };
+
+    logger.log('INFO', 'Earliest data availability map built:', {
+        portfolio: result.portfolio.toISOString().slice(0, 10),
+        history: result.history.toISOString().slice(0, 10),
+        insights: result.insights.toISOString().slice(0, 10),
+        social: result.social.toISOString().slice(0, 10),
+        absoluteEarliest: result.absoluteEarliest.toISOString().slice(0, 10)
+    });
+
+    return result;
 }
 
 module.exports = {
@@ -103,5 +181,6 @@ module.exports = {
     normalizeName,
     commitBatchInChunks,
     getExpectedDateStrings,
-    getFirstDateFromSourceData,
-};
+    // getFirstDateFromSourceData, // This is replaced
+    getEarliestDataDates, // <-- EXPORT NEW FUNCTION
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.131",
+  "version": "1.0.132",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [