bulltrackers-module 1.0.140 → 1.0.142

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -7,7 +7,8 @@ const { normalizeName, commitBatchInChunks } = require('../utils/utils.js');
 function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
 
 /** * --- MODIFIED: Returns detailed missing dependencies for logging ---
- * Stage 2: Check root data dependencies for a calc 
+ * Stage 2: Check root data dependencies for a calc
+ * --- THIS FUNCTION IS NOW MORE GRANULAR ---
  */
 function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
@@ -23,7 +24,7 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
 }
 
 /** * --- MODIFIED: Uses earliestDates map to avoid unnecessary queries ---
- * Stage 3: Check root data availability for a date 
+ * Stage 3: Check root data availability for a date
  */
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
     const { logger } = dependencies;
@@ -33,7 +34,9 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
     let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false;
     try {
         const tasks = [];
-        if (dateToProcess >= earliestDates.portfolio) 
+        // This logic is correct. It *avoids* calling get...Refs
+        // if the dateToProcess is before the earliest known data.
+        if (dateToProcess >= earliestDates.portfolio)
             {tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(res => {portfolioRefs = res;hasPortfolio = !!(res?.length);}));}
         if (dateToProcess >= earliestDates.insights) {
             tasks.push(loadDailyInsights(config, dependencies, dateStr).then(res => {insightsData = res;hasInsights = !!res;}));}
@@ -41,8 +44,17 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
             tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(res => {socialData = res;hasSocial = !!res;}));}
         if (dateToProcess >= earliestDates.history) {
             tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(res => {historyRefs = res;hasHistory = !!(res?.length);}));}
+
         await Promise.all(tasks);
-        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) {logger.log('WARN', `[PassRunner] No root data for ${dateStr}.`); return null;}
+
+        // --- NEW: Log what was *actually* found ---
+        logger.log('INFO', `[PassRunner] Data availability for ${dateStr}: P:${hasPortfolio}, I:${hasInsights}, S:${hasSocial}, H:${hasHistory}`);
+
+        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) {
+            logger.log('WARN', `[PassRunner] No root data at all for ${dateStr}.`);
+            // We return null to skip the entire day
+            return null;
+        }
         return {portfolioRefs, insightsData,socialData,historyRefs,status: { hasPortfolio, hasInsights, hasSocial, hasHistory }};
     } catch (err) { logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message }); return null; }
 }
@@ -70,34 +82,85 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
     return fetched;
 }
 
-/** --- MODIFIED: Stage 5: Filter calculations to skip completed work ---
+/**
+ * --- ENTIRELY REWRITTEN: Stage 5: Filter calculations ---
+ * This function now implements your "even better design".
+ * It calculates the *true earliest run date* for every calculation
+ * and filters them out *before* the "Running..." log ever appears.
  */
-function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, logger, isFirstDayOfBackfill = false) { 
+function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, earliestDates, logger) {
     const skipped = new Set();
-    const standardCalcsToRun = standardCalcs.filter(c => {
-        if (existingResults[c.name]) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Result already exists.`);return false;}
-        if (isFirstDayOfBackfill && c.isHistorical) {
-            logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Historical calc on the first day of backfill (no yesterday).`);
-            skipped.add(c.name);
+    const dateToProcess = new Date(dateStr + 'T00:00:00Z');
+
+    /**
+     * --- CORRECTED LOGIC ---
+     * Helper to get the true earliest date a calc can run.
+     */
+    const getTrueEarliestRunDate = (calc) => {
+        let earliestRunDate = new Date('1970-01-01T00:00:00Z'); // Start at zero
+        const dependencies = calc.rootDataDependencies || [];
+
+        // 1. Find the LATEST "today" dependency
+        // This is the date where all *today* data is first available
+        for (const dep of dependencies) {
+            if (dep === 'portfolio' && earliestDates.portfolio > earliestRunDate) earliestRunDate = earliestDates.portfolio;
+            if (dep === 'history' && earliestDates.history > earliestRunDate) earliestRunDate = earliestDates.history;
+            if (dep === 'social' && earliestDates.social > earliestRunDate) earliestRunDate = earliestDates.social;
+            if (dep === 'insights' && earliestDates.insights > earliestRunDate) earliestRunDate = earliestDates.insights;
+        }
+
+        // 2. If the calc is historical, shift the *final* date by +1
+        // This universally applies the "+1" rule if *any* yesterday data is needed,
+        // (as long as we found a dependency in step 1).
+        if (calc.isHistorical && earliestRunDate.getTime() > 0) {
+            earliestRunDate.setUTCDate(earliestRunDate.getUTCDate() + 1);
+        }
+        
+        return earliestRunDate;
+    };
+
+    const filterCalc = (calc) => {
+        // 1. Skip if result already exists
+        if (existingResults[calc.name]) {
+            logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Result already exists.`);
+            skipped.add(calc.name);
             return false;
         }
-    });
-    // Filter Meta Calcs
-    const metaCalcsToRun = metaCalcs.filter(c => {
-        if (existingResults[c.name]) {logger.log('TRACE', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Result already exists.`);skipped.add(c.name);return false;}
-        // --- START: RECOMMENDED ADDITION ---
-        if (isFirstDayOfBackfill && c.isHistorical) {
-            logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Historical calc on the first day of backfill (no yesterday).`);
-            skipped.add(c.name);
+
+        // 2. Check *true* earliest run date
+        const earliestRunDate = getTrueEarliestRunDate(calc);
+        if (dateToProcess < earliestRunDate) {
+            logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Date is before true earliest run date (${earliestRunDate.toISOString().slice(0, 10)}).`);
+            skipped.add(calc.name);
+            return false;
+        }
+
+        // 3. Check if *today's* root data was *actually* found
+        // This handles gaps *after* the earliest date
+        const { canRun, missing: missingRoot } = checkRootDependencies(calc, rootDataStatus);
+        if (!canRun) {
+            logger.log('INFO', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Data missing for this date: [${missingRoot.join(', ')}]`);
+            skipped.add(calc.name);
             return false;
         }
-        // --- END: RECOMMENDED ADDITION ---
-        // 1. Check root data
-        const { canRun, missing: missingRoot } = checkRootDependencies(c, rootDataStatus);
-        if (!canRun) {logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing root data: [${missingRoot.join(', ')}]`);skipped.add(c.name);return false;}
-        // 2. Check computed dependencies
-        const missingDeps = (c.dependencies || []).map(normalizeName).filter(d => !existingResults[d]);
-        if (missingDeps.length > 0) {logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`);skipped.add(c.name);return false;} return true;});
+
+        // 4. (For Meta Calcs) Check computed dependencies
+        if (calc.type === 'meta') {
+            const missingDeps = (calc.dependencies || []).map(normalizeName).filter(d => !existingResults[d]);
+            if (missingDeps.length > 0) {
+                logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${calc.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`);
+                skipped.add(calc.name);
+                return false;
+            }
+        }
+        
+        // If it passed all checks, run it.
+        return true;
+    };
+
+    const standardCalcsToRun = standardCalcs.filter(filterCalc);
+    const metaCalcsToRun = metaCalcs.filter(filterCalc);
+
     return { standardCalcsToRun, metaCalcsToRun };
 }
 
@@ -107,21 +170,91 @@ function initializeCalculators(calcs, logger) { const state = {}; for (const c o
 
 /** Stage 7: Load historical data required for calculations */
 async function loadHistoricalData(date, calcs, config, deps, rootData) { const updated = {...rootData}, dStr=date.toISOString().slice(0,10); const tasks = [];
-    if(calcs.some(c=>c.isHistorical)) tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); updated.yesterdayPortfolios=await loadFullDayMap(config,deps,await getPortfolioPartRefs(config,deps,prevStr)); })());
-    if(calcs.some(c=>c.rootDataDependencies.includes('history'))) tasks.push((async()=>{ updated.todayHistoryData=await loadFullDayMap(config,deps,rootData.historyRefs); })());
-    if(calcs.some(c=>c.isHistorical)) tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); updated.yesterdayHistoryData=await loadFullDayMap(config,deps,await getHistoryPartRefs(config,deps,prevStr)); })());
+    const needsYesterdayPortfolio = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('portfolio'));
+    const needsTodayHistory = calcs.some(c => c.rootDataDependencies.includes('history'));
+    const needsYesterdayHistory = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('history'));
+    // --- NEW: Add checks for historical insights and social ---
+    const needsYesterdayInsights = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('insights'));
+    const needsYesterdaySocial = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('social'));
+    
+    // --- MODIFIED: Be smarter about loading data ---
+    if(needsYesterdayPortfolio) {
+        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10);
+            logger.log('INFO', `[PassRunner] Loading YESTERDAY portfolio data for ${prevStr}`);
+            updated.yesterdayPortfolios=await loadFullDayMap(config,deps,await getPortfolioPartRefs(config,deps,prevStr)); 
+        })());
+    }
+    if(needsTodayHistory) {
+        tasks.push((async()=>{ 
+            logger.log('INFO', `[PassRunner] Loading TODAY history data for ${dStr}`);
+            updated.todayHistoryData=await loadFullDayMap(config,deps,rootData.historyRefs); 
+        })());
+    }
+    if(needsYesterdayHistory) {
+        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
+            logger.log('INFO', `[PassRunner] Loading YESTERDAY history data for ${prevStr}`);
+            updated.yesterdayHistoryData=await loadFullDayMap(config,deps,await getHistoryPartRefs(config,deps,prevStr)); 
+        })());
+    }
+    // --- NEW: Load historical insights/social ---
+    if(needsYesterdayInsights) {
+        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
+            logger.log('INFO', `[PassRunner] Loading YESTERDAY insights data for ${prevStr}`);
+            updated.yesterdayInsights=await loadDailyInsights(config,deps,prevStr); 
+        })());
+    }
+    if(needsYesterdaySocial) {
+        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
+            logger.log('INFO', `[PassRunner] Loading YESTERDAY social data for ${prevStr}`);
+            updated.yesterdaySocialPostInsights=await loadDailySocialPostInsights(config,deps,prevStr); 
+        })());
+    }
+
     await Promise.all(tasks); return updated;
 }
 
+
 /** * --- MODIFIED: Stage 8: Stream and process data for standard calculations ---
  * This function now uses an async generator to stream portfolio data
  * instead of loading it all into memory.
  */
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData) { 
     const { logger, calculationUtils } = deps;
+    // --- MODIFIED: yesterdayInsights/Social are now loaded by loadHistoricalData ---
     const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, todayHistoryData, yesterdayHistoryData, yesterdayPortfolios } = rootData;
-    let firstUser=true;
+    
+    // --- NEW: Check if streaming is even needed ---
+    const calcsThatStreamPortfolio = Object.values(state).filter(calc => calc && calc.manifest && (calc.manifest.rootDataDependencies.includes('portfolio') || calc.manifest.category === 'speculators'));
+    
     const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config};
+    
+    // --- MODIFIED: Run non-streaming calcs first (social/insights) ---
+    // This allows them to run even if portfolio data is missing
+    let firstUser=true; // Used to run them only once
+    for(const name in state){ 
+        const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
+        const cat=calc.manifest.category;
+        if(cat==='socialPosts'||cat==='insights') {
+            if (firstUser) {
+                logger.log('INFO', `[${passName}] Running non-streaming calc: ${name}`);
+                let args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData];
+                // Pass historical data if needed
+                if(calc.manifest.isHistorical) {
+                    args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData];
+                }
+                try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} (non-stream)`,{err:e.message});}
+            }
+        }
+    }
+
+
+    if (calcsThatStreamPortfolio.length === 0) {
+        logger.log('INFO', `[${passName}] No portfolio-streaming calcs to run for ${dateStr}. Skipping stream.`);
+        return; // Exit stream function
+    }
+    
+    logger.log('INFO', `[${passName}] Streaming portfolio data for ${calcsThatStreamPortfolio.length} calcs...`);
+    
     for await (const chunk of streamPortfolioData(config, deps, dateStr)) {
         for(const uid in chunk){ const p=chunk[uid]; if(!p) continue; 
             const userType=p.PublicPositions?'speculator':'normal'; 
@@ -129,9 +262,18 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
             for(const name in state){ 
                 const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
                 const cat=calc.manifest.category, isSocialOrInsights=cat==='socialPosts'||cat==='insights', isHistorical=calc.manifest.isHistorical, isSpec=cat==='speculators'; 
+                
+                // --- MODIFIED: Skip social/insights here, they ran above ---
+                if(isSocialOrInsights) continue; 
+                
                 let args=[p,null,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; 
-                if(isSocialOrInsights&&!firstUser) continue; 
-                if(isHistorical){ const pY=yesterdayPortfolios[uid]; if(!pY) continue; args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; } 
+                
+                if(isHistorical){ 
+                    const pY=yesterdayPortfolios ? yesterdayPortfolios[uid] : null; // Check if yesterdayPortfolios exists
+                    // V3 behavioural calcs (like history aggregator) *can* run without pY
+                    if(!pY && (cat !== 'behavioural' && name !== 'historical-performance-aggregator')) continue; 
+                    args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; 
+                } 
                 if((userType==='normal'&&isSpec)||(userType==='speculator'&&!isSpec&&name!=='users-processed')) continue; 
                 try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} for ${uid}`,{err:e.message});} } 
             firstUser=false; 
@@ -142,11 +284,20 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
 /** Stage 9: Run standard computations */
 async function runStandardComputationPass(date, calcs, passName, config, deps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
-    if (calcs.length === 0) return; 
+    // --- THIS IS THE CRITICAL CHANGE ---
+    // If 'calcs' is empty *because of the new filter*, this log won't even appear.
+    if (calcs.length === 0) {
+        logger.log('INFO', `[${passName}] No standard calcs to run for ${dateStr} after filtering.`);
+        return; 
+    }
+    // This log now only appears if there is *actually* work to do.
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
+
     const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
     const state = initializeCalculators(calcs, logger);
+    
     await streamAndProcess(dStr, state, passName, config, deps, fullRoot);
+    
     let success = 0;
     const standardWrites = [];
     const shardedWrites = {}; 
@@ -188,25 +339,19 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
         const shardedDocWrites = [];
         let docRef;
         if (docPath.includes('/')) {
-            // Path is absolute, e.g., 'social_prediction_regime_state/history'
             docRef = deps.db.doc(docPath);
         } else {
-            // Path is a docId, e.g., 'user_profile_history_shard_0'
-            // We must infer its collection from config.
             const collection = (docPath.startsWith('user_profile_history')) 
-                ? config.shardedUserProfileCollection // 'user_profile_history'
-                : config.shardedProfitabilityCollection; // Fallback
+                ? config.shardedUserProfileCollection
+                : config.shardedProfitabilityCollection;
             docRef = deps.db.collection(collection).doc(docPath);
         }
-        // Ensure data is a valid object before pushing
         if (docData && typeof docData === 'object' && !Array.isArray(docData)) {
              shardedDocWrites.push({ ref: docRef, data: docData });
         } else {
              logger.log('ERROR', `[${passName}] Invalid sharded document data for ${docPath}. Not an object.`, { data: docData });
         }
-        // Commit this single document write (or small batch if logic is changed later)
         if (shardedDocWrites.length > 0) {
-            // Use the docPath in the operation name for clearer logging
             await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${docPath} ${dStr}`);
         }
     }
@@ -216,8 +361,17 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
 /** Stage 10: Run meta computations */
 async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
-    if (calcs.length === 0) return;
+    // --- THIS IS THE CRITICAL CHANGE ---
+    if (calcs.length === 0) {
+        logger.log('INFO', `[${passName}] No meta calcs to run for ${dStr} after filtering.`);
+        return;
+    }
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
+    
+    // --- NEW: Load historical data for meta calcs if needed ---
+    // (This might be redundant if standard pass ran, but meta-calcs can run standalone)
+    const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
+
     let success = 0;
     const standardWrites = [];
     const shardedWrites = {}; 
@@ -226,7 +380,8 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
         if (typeof Cl !== 'function') {logger.log('ERROR', `Invalid class ${name}`);continue;}
         const inst = new Cl();
         try {
-            const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData }, config, fetchedDeps));
+            // --- MODIFIED: Pass fullRoot to meta calcs ---
+            const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData: fullRoot }, config, fetchedDeps));
             if (result && Object.keys(result).length > 0) {const standardResult = {}; for (const key in result) {
                     if (key.startsWith('sharded_')) {const shardedData = result[key];for (const collectionName in shardedData) 
                         {if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};Object.assign(shardedWrites[collectionName], shardedData[collectionName]);}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.140",
+  "version": "1.0.142",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [