bulltrackers-module 1.0.139 → 1.0.141

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -1,125 +1,210 @@
 /**
- * @fileoverview
- * Main "Pass Runner" for the V2 Computation System.
- *
- * This orchestrator is designed to be run by a separate Cloud Function for each "pass".
- * It reads its pass number from the config and executes only those calculations.
- * This file contains the high-level "manual" of steps. The "how-to" logic
- * is extracted into 'computation_system_utils.js'.
- * --- MODIFIED: To use getEarliestDataDates and pass the date map to the orchestrator helpers. ---
- * --- MODIFIED: To run date processing in parallel batches. ---
- * --- MODIFIED: To fetch ALL existing results to enable incremental (skip) logic. ---
+ * FIXED: computation_pass_runner.js
+ * Now calculates earliest date PER CALCULATION, not per pass
  */
 
-// --- MODIFIED: Renamed fetchDependenciesForPass to fetchExistingResults ---
 const { groupByPass, checkRootDataAvailability, fetchExistingResults, filterCalculations, runStandardComputationPass, runMetaComputationPass } = require('./orchestration_helpers.js');
-// --- MODIFIED: Import getEarliestDataDates ---
 const { getExpectedDateStrings, getEarliestDataDates } = require('../utils/utils.js');
 
-// --- NEW: Parallel processing batch size ---
-const PARALLEL_BATCH_SIZE = 7; // Process a week at a time
+const PARALLEL_BATCH_SIZE = 7;
 
 async function runComputationPass(config, dependencies, computationManifest) {
     const { logger } = dependencies;
-    const passToRun = String(config.COMPUTATION_PASS_TO_RUN); if (!passToRun) return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
+    const passToRun = String(config.COMPUTATION_PASS_TO_RUN);
+    if (!passToRun) return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
+    
     logger.log('INFO', `🚀 Starting PASS ${passToRun}...`);
-    const yesterday = new Date(); yesterday.setUTCDate(yesterday.getUTCDate()-1); 
+    
+    const yesterday = new Date();
+    yesterday.setUTCDate(yesterday.getUTCDate() - 1);
     const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
     
-    // const earliestDates = await getEarliestDataDates(config, dependencies); // <-- 1. COMMENT OUT OR REMOVE THIS BROKEN CALL
-
-    // --- START: 2. NEW HARDCODED DATES (V2 - CORRECTED) ---
+    // Hardcoded earliest dates
     logger.log('INFO', 'Using hardcoded earliest data dates to bypass faulty discovery.');
-
-    // Use the exact dates you provided:
-    const earliestPortfolio = new Date('2025-09-25T00:00:00Z'); // Earliest of Normal (09-25) and Spec (09-29)
-    const earliestHistory = new Date('2025-11-05T00:00:00Z');   // Both Normal and Spec history
-    const earliestSocial = new Date('2025-10-30T00:00:00Z');    // daily_social_insights
-    const earliestInsights = new Date('2025-08-26T00:00:00Z');   // daily_instrument_insights (Your new date)
+    const earliestPortfolio = new Date('2025-09-25T00:00:00Z');
+    const earliestHistory = new Date('2025-11-05T00:00:00Z');
+    const earliestSocial = new Date('2025-10-30T00:00:00Z');
+    const earliestInsights = new Date('2025-08-26T00:00:00Z');
     
     const earliestDates = {
         portfolio: earliestPortfolio,
         history: earliestHistory,
         social: earliestSocial,
-        insights: earliestInsights, // <-- Use the real date
-        // Calculate absoluteEarliest based on all real data
+        insights: earliestInsights,
         absoluteEarliest: [earliestPortfolio, earliestHistory, earliestSocial, earliestInsights].reduce((a, b) => a < b ? a : b)
     };
     
     logger.log('INFO', `Hardcoded map: portfolio=${earliestDates.portfolio.toISOString().slice(0,10)}, history=${earliestDates.history.toISOString().slice(0,10)}, social=${earliestDates.social.toISOString().slice(0,10)}, insights=${earliestDates.insights.toISOString().slice(0,10)}`);
-    // --- END: NEW HARDCODED DATES ---
-
-
+    
     const passes = groupByPass(computationManifest);
-    const calcsInThisPass = passes[passToRun] || []; if (!calcsInThisPass.length) return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
-    
-    // --- 3. THIS LOGIC BLOCK (from our last fix) IS NOW CORRECT ---
-    // It will consume the clean, hardcoded 'earliestDates' map and work as intended.
-    const requiredRootData = new Set();
-    calcsInThisPass.forEach(c => {
-        (c.rootDataDependencies || []).forEach(dep => requiredRootData.add(dep));
-    });
-
-    let earliestStartDateForPass = null;
-    if (requiredRootData.size > 0) {
-        let latestOfEarliestDates = new Date(0); // Start at epoch
-        const farFutureSentinelYear = 2999; // Keep as a safety check for any future issues
-        let hasAtLeastOneValidDate = false; 
-
-        requiredRootData.forEach(dep => {
-            const earliestDateForDep = earliestDates[dep];
-            
-            // This check is now just a safeguard
-            if (earliestDateForDep && earliestDateForDep.getUTCFullYear() < farFutureSentinelYear) {
-                if (earliestDateForDep > latestOfEarliestDates) {
-                    latestOfEarliestDates = earliestDateForDep;
-                }
-                hasAtLeastOneValidDate = true; 
-            } else if (earliestDateForDep) {
-                logger.log('INFO', `[PassRunner] Dependency '${dep}' is not available (date: ${earliestDateForDep.toISOString().slice(0, 10)}). Calcs requiring it will be skipped.`);
-            } else {
-                 logger.log('WARN', `[PassRunner] Dependency '${dep}' has no earliest date defined in hardcoded map.`);
+    const calcsInThisPass = passes[passToRun] || [];
+    if (!calcsInThisPass.length) return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
+    
+    // ============================================
+    // NEW: Calculate earliest date PER CALCULATION
+    // ============================================
+    const calcEarliestDates = new Map();
+    
+    for (const calc of calcsInThisPass) {
+        const deps = calc.rootDataDependencies || [];
+        
+        if (deps.length === 0) {
+            // No dependencies = can run from the absolute earliest
+            calcEarliestDates.set(calc.name, earliestDates.absoluteEarliest);
+            continue;
+        }
+        
+        // Find the LATEST earliest date among THIS calculation's dependencies
+        let latestEarliest = new Date(0);
+        for (const dep of deps) {
+            const depDate = earliestDates[dep];
+            if (depDate && depDate > latestEarliest) {
+                latestEarliest = depDate;
             }
-        });
+        }
         
-        if (hasAtLeastOneValidDate) {
-             earliestStartDateForPass = latestOfEarliestDates;
+        // If this is a historical calculation, add 1 day (needs yesterday)
+        if (calc.isHistorical) {
+            const adjusted = new Date(latestEarliest);
+            adjusted.setUTCDate(adjusted.getUTCDate() + 1);
+            calcEarliestDates.set(calc.name, adjusted);
+            logger.log('TRACE', `[PassRunner] ${calc.name}: earliest=${adjusted.toISOString().slice(0,10)} (historical, needs ${latestEarliest.toISOString().slice(0,10)} + 1 day)`);
+        } else {
+            calcEarliestDates.set(calc.name, latestEarliest);
+            logger.log('TRACE', `[PassRunner] ${calc.name}: earliest=${latestEarliest.toISOString().slice(0,10)}`);
         }
     }
-
-    // Use the pass-specific date. Fall back to absolute earliest, then config.
-    const firstDate = earliestStartDateForPass || earliestDates.absoluteEarliest;
-    const startDateUTC = firstDate 
-        ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate())) 
-        : new Date(config.earliestComputationDate+'T00:00:00Z');
-    
-    logger.log('INFO', `[PassRunner] Pass ${passToRun} requires data: [${Array.from(requiredRootData).join(', ')}].`);
-    logger.log('INFO', `[PassRunner] Determined start date for this pass: ${startDateUTC.toISOString().slice(0, 10)}`);
-    // --- END OF LOGIC BLOCK ---
-
+    
+    // The pass can start from the EARLIEST calculation's start date
+    const passEarliestDate = new Date(Math.min(...Array.from(calcEarliestDates.values()).map(d => d.getTime())));
+    
+    logger.log('INFO', `[PassRunner] Pass ${passToRun} analysis:`);
+    logger.log('INFO', `  Total calculations: ${calcsInThisPass.length}`);
+    logger.log('INFO', `  Pass can start from: ${passEarliestDate.toISOString().slice(0,10)}`);
+    logger.log('INFO', `  Individual calculation date ranges calculated.`);
+    
+    // ============================================
+    // Generate date range from earliest to yesterday
+    // ============================================
+    const startDateUTC = new Date(Date.UTC(passEarliestDate.getUTCFullYear(), passEarliestDate.getUTCMonth(), passEarliestDate.getUTCDate()));
     const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
-    const firstDayOfBackfill = allExpectedDates.length > 0 ? allExpectedDates[0] : null; // --- MOVED FROM ABOVE ---
     
-    const standardCalcs = calcsInThisPass.filter(c => c.type==='standard'); 
-    const metaCalcs = calcsInThisPass.filter(c => c.type==='meta');
+    logger.log('INFO', `[PassRunner] Processing ${allExpectedDates.length} total dates from ${allExpectedDates[0]} to ${allExpectedDates[allExpectedDates.length-1]}`);
+    
+    const standardCalcs = calcsInThisPass.filter(c => c.type === 'standard');
+    const metaCalcs = calcsInThisPass.filter(c => c.type === 'meta');
+    
+    // ============================================
+    // Process each date
+    // ============================================
     const processDate = async (dateStr) => {
-        const dateToProcess = new Date(dateStr+'T00:00:00Z');
+        const dateToProcess = new Date(dateStr + 'T00:00:00Z');
+        
         try {
+            // Check root data availability
             const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
-            if (!rootData) { logger.log('WARN', `[PassRunner] Skipping ${dateStr} for Pass ${passToRun}: No root data.`);return;}
+            if (!rootData) {
+                logger.log('WARN', `[PassRunner] Skipping ${dateStr} for Pass ${passToRun}: No root data.`);
+                return;
+            }
+            
+            // Fetch existing results
             const existingResults = await fetchExistingResults(dateStr, calcsInThisPass, computationManifest, config, dependencies);
-            const { standardCalcsToRun, metaCalcsToRun } = filterCalculations(standardCalcs, metaCalcs, rootData.status, existingResults, passToRun, dateStr, logger,dateStr === firstDayOfBackfill );            
-            if (standardCalcsToRun.length === 0 && metaCalcsToRun.length === 0) {logger.log('INFO', `[PassRunner] All calcs for ${dateStr} Pass ${passToRun} are already complete. Skipping.`);return;}
-            if (standardCalcsToRun.length) await runStandardComputationPass(dateToProcess, standardCalcsToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData);
-            if (metaCalcsToRun.length) await runMetaComputationPass(dateToProcess, metaCalcsToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, rootData);
-        } catch (err) {logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack });}
+            
+            // ============================================
+            // NEW: Filter based on per-calculation earliest dates
+            // ============================================
+            const standardCalcsToRun = standardCalcs.filter(c => {
+                // Skip if already exists
+                if (existingResults[c.name]) {
+                    logger.log('TRACE', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Result already exists.`);
+                    return false;
+                }
+                
+                // Skip if date is before this calculation's earliest date
+                const calcEarliestDate = calcEarliestDates.get(c.name);
+                if (calcEarliestDate && dateToProcess < calcEarliestDate) {
+                    logger.log('TRACE', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Date before calc's earliest (${calcEarliestDate.toISOString().slice(0,10)}).`);
+                    return false;
+                }
+                
+                // Check root data dependencies
+                const missingDeps = (c.rootDataDependencies || []).filter(dep => {
+                    if (dep === 'portfolio') return !rootData.status.hasPortfolio;
+                    if (dep === 'insights') return !rootData.status.hasInsights;
+                    if (dep === 'social') return !rootData.status.hasSocial;
+                    if (dep === 'history') return !rootData.status.hasHistory;
+                    return false;
+                });
+                
+                if (missingDeps.length > 0) {
+                    logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Missing deps: ${missingDeps.join(', ')}`);
+                    return false;
+                }
+                
+                return true;
+            });
+            
+            const metaCalcsToRun = metaCalcs.filter(c => {
+                if (existingResults[c.name]) return false;
+                
+                const calcEarliestDate = calcEarliestDates.get(c.name);
+                if (calcEarliestDate && dateToProcess < calcEarliestDate) {
+                    logger.log('TRACE', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Date before calc's earliest.`);
+                    return false;
+                }
+                
+                // Check root dependencies
+                const missingRootDeps = (c.rootDataDependencies || []).filter(dep => {
+                    if (dep === 'portfolio') return !rootData.status.hasPortfolio;
+                    if (dep === 'insights') return !rootData.status.hasInsights;
+                    if (dep === 'social') return !rootData.status.hasSocial;
+                    if (dep === 'history') return !rootData.status.hasHistory;
+                    return false;
+                });
+                
+                if (missingRootDeps.length > 0) {
+                    logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing root deps: ${missingRootDeps.join(', ')}`);
+                    return false;
+                }
+                
+                // Check computed dependencies
+                const missingComputedDeps = (c.dependencies || []).filter(d => !existingResults[d]);
+                if (missingComputedDeps.length > 0) {
+                    logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing computed deps: ${missingComputedDeps.join(', ')}`);
+                    return false;
+                }
+                
+                return true;
+            });
+            
+            if (standardCalcsToRun.length === 0 && metaCalcsToRun.length === 0) {
+                logger.log('INFO', `[PassRunner] All eligible calcs for ${dateStr} Pass ${passToRun} are already complete. Skipping.`);
+                return;
+            }
+            
+            logger.log('INFO', `[PassRunner] Running ${dateStr}: ${standardCalcsToRun.length} standard, ${metaCalcsToRun.length} meta`);
+            
+            if (standardCalcsToRun.length) {
+                await runStandardComputationPass(dateToProcess, standardCalcsToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData);
+            }
+            if (metaCalcsToRun.length) {
+                await runMetaComputationPass(dateToProcess, metaCalcsToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, rootData);
+            }
+            
+        } catch (err) {
+            logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
+        }
     };
+    
+    // Process in batches
     logger.log('INFO', `[PassRunner] Processing ${allExpectedDates.length} total dates in batches of ${PARALLEL_BATCH_SIZE}...`);
     for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) {
         const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
         logger.log('INFO', `[PassRunner] Processing batch ${Math.floor(i / PARALLEL_BATCH_SIZE) + 1}/${Math.ceil(allExpectedDates.length / PARALLEL_BATCH_SIZE)} (Dates: ${batch[0]}...${batch[batch.length-1]})`);
         await Promise.all(batch.map(dateStr => processDate(dateStr)));
     }
+    
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -7,7 +7,8 @@ const { normalizeName, commitBatchInChunks } = require('../utils/utils.js');
 function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
 
 /** * --- MODIFIED: Returns detailed missing dependencies for logging ---
- * Stage 2: Check root data dependencies for a calc 
+ * Stage 2: Check root data dependencies for a calc
+ * --- THIS FUNCTION IS NOW MORE GRANULAR ---
  */
 function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
@@ -23,7 +24,7 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
 }
 
 /** * --- MODIFIED: Uses earliestDates map to avoid unnecessary queries ---
- * Stage 3: Check root data availability for a date 
+ * Stage 3: Check root data availability for a date
  */
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
     const { logger } = dependencies;
@@ -33,7 +34,9 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
     let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false;
     try {
         const tasks = [];
-        if (dateToProcess >= earliestDates.portfolio) 
+        // This logic is correct. It *avoids* calling get...Refs
+        // if the dateToProcess is before the earliest known data.
+        if (dateToProcess >= earliestDates.portfolio)
             {tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(res => {portfolioRefs = res;hasPortfolio = !!(res?.length);}));}
         if (dateToProcess >= earliestDates.insights) {
             tasks.push(loadDailyInsights(config, dependencies, dateStr).then(res => {insightsData = res;hasInsights = !!res;}));}
@@ -41,8 +44,17 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
             tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(res => {socialData = res;hasSocial = !!res;}));}
         if (dateToProcess >= earliestDates.history) {
             tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(res => {historyRefs = res;hasHistory = !!(res?.length);}));}
+
         await Promise.all(tasks);
-        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) {logger.log('WARN', `[PassRunner] No root data for ${dateStr}.`); return null;}
+
+        // --- NEW: Log what was *actually* found ---
+        logger.log('INFO', `[PassRunner] Data availability for ${dateStr}: P:${hasPortfolio}, I:${hasInsights}, S:${hasSocial}, H:${hasHistory}`);
+
+        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) {
+            logger.log('WARN', `[PassRunner] No root data at all for ${dateStr}.`);
+            // We return null to skip the entire day
+            return null;
+        }
         return {portfolioRefs, insightsData,socialData,historyRefs,status: { hasPortfolio, hasInsights, hasSocial, hasHistory }};
     } catch (err) { logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message }); return null; }
 }
@@ -70,34 +82,85 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
     return fetched;
 }
 
-/** --- MODIFIED: Stage 5: Filter calculations to skip completed work ---
+/**
+ * --- ENTIRELY REWRITTEN: Stage 5: Filter calculations ---
+ * This function now implements your "even better design".
+ * It calculates the *true earliest run date* for every calculation
+ * and filters them out *before* the "Running..." log ever appears.
  */
-function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, logger, isFirstDayOfBackfill = false) { 
+function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, earliestDates, logger) {
     const skipped = new Set();
-    const standardCalcsToRun = standardCalcs.filter(c => {
-        if (existingResults[c.name]) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Result already exists.`);return false;}
-        if (isFirstDayOfBackfill && c.isHistorical) {
-            logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Historical calc on the first day of backfill (no yesterday).`);
-            skipped.add(c.name);
+    const dateToProcess = new Date(dateStr + 'T00:00:00Z');
+
+    /**
+     * --- CORRECTED LOGIC ---
+     * Helper to get the true earliest date a calc can run.
+     */
+    const getTrueEarliestRunDate = (calc) => {
+        let earliestRunDate = new Date('1970-01-01T00:00:00Z'); // Start at zero
+        const dependencies = calc.rootDataDependencies || [];
+
+        // 1. Find the LATEST "today" dependency
+        // This is the date where all *today* data is first available
+        for (const dep of dependencies) {
+            if (dep === 'portfolio' && earliestDates.portfolio > earliestRunDate) earliestRunDate = earliestDates.portfolio;
+            if (dep === 'history' && earliestDates.history > earliestRunDate) earliestRunDate = earliestDates.history;
+            if (dep === 'social' && earliestDates.social > earliestRunDate) earliestRunDate = earliestDates.social;
+            if (dep === 'insights' && earliestDates.insights > earliestRunDate) earliestRunDate = earliestDates.insights;
+        }
+
+        // 2. If the calc is historical, shift the *final* date by +1
+        // This universally applies the "+1" rule if *any* yesterday data is needed,
+        // (as long as we found a dependency in step 1).
+        if (calc.isHistorical && earliestRunDate.getTime() > 0) {
+            earliestRunDate.setUTCDate(earliestRunDate.getUTCDate() + 1);
+        }
+        
+        return earliestRunDate;
+    };
+
+    const filterCalc = (calc) => {
+        // 1. Skip if result already exists
+        if (existingResults[calc.name]) {
+            logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Result already exists.`);
+            skipped.add(calc.name);
             return false;
         }
-    });
-    // Filter Meta Calcs
-    const metaCalcsToRun = metaCalcs.filter(c => {
-        if (existingResults[c.name]) {logger.log('TRACE', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Result already exists.`);skipped.add(c.name);return false;}
-        // --- START: RECOMMENDED ADDITION ---
-        if (isFirstDayOfBackfill && c.isHistorical) {
-            logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Historical calc on the first day of backfill (no yesterday).`);
-            skipped.add(c.name);
+
+        // 2. Check *true* earliest run date
+        const earliestRunDate = getTrueEarliestRunDate(calc);
+        if (dateToProcess < earliestRunDate) {
+            logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Date is before true earliest run date (${earliestRunDate.toISOString().slice(0, 10)}).`);
+            skipped.add(calc.name);
+            return false;
+        }
+
+        // 3. Check if *today's* root data was *actually* found
+        // This handles gaps *after* the earliest date
+        const { canRun, missing: missingRoot } = checkRootDependencies(calc, rootDataStatus);
+        if (!canRun) {
+            logger.log('INFO', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Data missing for this date: [${missingRoot.join(', ')}]`);
+            skipped.add(calc.name);
             return false;
         }
-        // --- END: RECOMMENDED ADDITION ---
-        // 1. Check root data
-        const { canRun, missing: missingRoot } = checkRootDependencies(c, rootDataStatus);
-        if (!canRun) {logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing root data: [${missingRoot.join(', ')}]`);skipped.add(c.name);return false;}
-        // 2. Check computed dependencies
-        const missingDeps = (c.dependencies || []).map(normalizeName).filter(d => !existingResults[d]);
-        if (missingDeps.length > 0) {logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`);skipped.add(c.name);return false;} return true;});
+
+        // 4. (For Meta Calcs) Check computed dependencies
+        if (calc.type === 'meta') {
+            const missingDeps = (calc.dependencies || []).map(normalizeName).filter(d => !existingResults[d]);
+            if (missingDeps.length > 0) {
+                logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${calc.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`);
+                skipped.add(calc.name);
+                return false;
+            }
+        }
+        
+        // If it passed all checks, run it.
+        return true;
+    };
+
+    const standardCalcsToRun = standardCalcs.filter(filterCalc);
+    const metaCalcsToRun = metaCalcs.filter(filterCalc);
+
     return { standardCalcsToRun, metaCalcsToRun };
 }
 
@@ -107,21 +170,91 @@ function initializeCalculators(calcs, logger) { const state = {}; for (const c o
 
 /** Stage 7: Load historical data required for calculations */
 async function loadHistoricalData(date, calcs, config, deps, rootData) { const updated = {...rootData}, dStr=date.toISOString().slice(0,10); const tasks = [];
-    if(calcs.some(c=>c.isHistorical)) tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); updated.yesterdayPortfolios=await loadFullDayMap(config,deps,await getPortfolioPartRefs(config,deps,prevStr)); })());
-    if(calcs.some(c=>c.rootDataDependencies.includes('history'))) tasks.push((async()=>{ updated.todayHistoryData=await loadFullDayMap(config,deps,rootData.historyRefs); })());
-    if(calcs.some(c=>c.isHistorical)) tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); updated.yesterdayHistoryData=await loadFullDayMap(config,deps,await getHistoryPartRefs(config,deps,prevStr)); })());
+    const needsYesterdayPortfolio = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('portfolio'));
+    const needsTodayHistory = calcs.some(c => c.rootDataDependencies.includes('history'));
+    const needsYesterdayHistory = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('history'));
+    // --- NEW: Add checks for historical insights and social ---
+    const needsYesterdayInsights = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('insights'));
+    const needsYesterdaySocial = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('social'));
+    
+    // --- MODIFIED: Be smarter about loading data ---
+    if(needsYesterdayPortfolio) {
+        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10);
+            logger.log('INFO', `[PassRunner] Loading YESTERDAY portfolio data for ${prevStr}`);
+            updated.yesterdayPortfolios=await loadFullDayMap(config,deps,await getPortfolioPartRefs(config,deps,prevStr)); 
+        })());
+    }
+    if(needsTodayHistory) {
+        tasks.push((async()=>{ 
+            logger.log('INFO', `[PassRunner] Loading TODAY history data for ${dStr}`);
+            updated.todayHistoryData=await loadFullDayMap(config,deps,rootData.historyRefs); 
+        })());
+    }
+    if(needsYesterdayHistory) {
+        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
+            logger.log('INFO', `[PassRunner] Loading YESTERDAY history data for ${prevStr}`);
+            updated.yesterdayHistoryData=await loadFullDayMap(config,deps,await getHistoryPartRefs(config,deps,prevStr)); 
+        })());
+    }
+    // --- NEW: Load historical insights/social ---
+    if(needsYesterdayInsights) {
+        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
+            logger.log('INFO', `[PassRunner] Loading YESTERDAY insights data for ${prevStr}`);
+            updated.yesterdayInsights=await loadDailyInsights(config,deps,prevStr); 
+        })());
+    }
+    if(needsYesterdaySocial) {
+        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
+            logger.log('INFO', `[PassRunner] Loading YESTERDAY social data for ${prevStr}`);
+            updated.yesterdaySocialPostInsights=await loadDailySocialPostInsights(config,deps,prevStr); 
+        })());
+    }
+
     await Promise.all(tasks); return updated;
 }
 
+
 /** * --- MODIFIED: Stage 8: Stream and process data for standard calculations ---
  * This function now uses an async generator to stream portfolio data
  * instead of loading it all into memory.
  */
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData) { 
     const { logger, calculationUtils } = deps;
+    // --- MODIFIED: yesterdayInsights/Social are now loaded by loadHistoricalData ---
     const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, todayHistoryData, yesterdayHistoryData, yesterdayPortfolios } = rootData;
-    let firstUser=true;
+    
+    // --- NEW: Check if streaming is even needed ---
+    const calcsThatStreamPortfolio = Object.values(state).filter(calc => calc && calc.manifest && (calc.manifest.rootDataDependencies.includes('portfolio') || calc.manifest.category === 'speculators'));
+    
     const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config};
+    
+    // --- MODIFIED: Run non-streaming calcs first (social/insights) ---
+    // This allows them to run even if portfolio data is missing
+    let firstUser=true; // Used to run them only once
+    for(const name in state){ 
+        const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
+        const cat=calc.manifest.category;
+        if(cat==='socialPosts'||cat==='insights') {
+            if (firstUser) {
+                logger.log('INFO', `[${passName}] Running non-streaming calc: ${name}`);
+                let args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData];
+                // Pass historical data if needed
+                if(calc.manifest.isHistorical) {
+                    args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData];
+                }
+                try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} (non-stream)`,{err:e.message});}
+            }
+        }
+    }
+
+
+    if (calcsThatStreamPortfolio.length === 0) {
+        logger.log('INFO', `[${passName}] No portfolio-streaming calcs to run for ${dateStr}. Skipping stream.`);
+        return; // Exit stream function
+    }
+    
+    logger.log('INFO', `[${passName}] Streaming portfolio data for ${calcsThatStreamPortfolio.length} calcs...`);
+    
     for await (const chunk of streamPortfolioData(config, deps, dateStr)) {
         for(const uid in chunk){ const p=chunk[uid]; if(!p) continue; 
             const userType=p.PublicPositions?'speculator':'normal'; 
@@ -129,9 +262,18 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
             for(const name in state){ 
                 const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
                 const cat=calc.manifest.category, isSocialOrInsights=cat==='socialPosts'||cat==='insights', isHistorical=calc.manifest.isHistorical, isSpec=cat==='speculators'; 
+                
+                // --- MODIFIED: Skip social/insights here, they ran above ---
+                if(isSocialOrInsights) continue; 
+                
                 let args=[p,null,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; 
-                if(isSocialOrInsights&&!firstUser) continue; 
-                if(isHistorical){ const pY=yesterdayPortfolios[uid]; if(!pY) continue; args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; } 
+                
+                if(isHistorical){ 
+                    const pY=yesterdayPortfolios ? yesterdayPortfolios[uid] : null; // Check if yesterdayPortfolios exists
+                    // V3 behavioural calcs (like history aggregator) *can* run without pY
+                    if(!pY && (cat !== 'behavioural' && name !== 'historical-performance-aggregator')) continue; 
+                    args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; 
+                } 
                 if((userType==='normal'&&isSpec)||(userType==='speculator'&&!isSpec&&name!=='users-processed')) continue; 
                 try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} for ${uid}`,{err:e.message});} } 
             firstUser=false; 
@@ -142,11 +284,20 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
 /** Stage 9: Run standard computations */
 async function runStandardComputationPass(date, calcs, passName, config, deps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
-    if (calcs.length === 0) return; 
+    // --- THIS IS THE CRITICAL CHANGE ---
+    // If 'calcs' is empty *because of the new filter*, this log won't even appear.
+    if (calcs.length === 0) {
+        logger.log('INFO', `[${passName}] No standard calcs to run for ${dateStr} after filtering.`);
+        return; 
+    }
+    // This log now only appears if there is *actually* work to do.
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
+
     const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
     const state = initializeCalculators(calcs, logger);
+    
     await streamAndProcess(dStr, state, passName, config, deps, fullRoot);
+    
     let success = 0;
     const standardWrites = [];
     const shardedWrites = {}; 
@@ -188,25 +339,19 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
         const shardedDocWrites = [];
         let docRef;
         if (docPath.includes('/')) {
-            // Path is absolute, e.g., 'social_prediction_regime_state/history'
             docRef = deps.db.doc(docPath);
         } else {
-            // Path is a docId, e.g., 'user_profile_history_shard_0'
-            // We must infer its collection from config.
             const collection = (docPath.startsWith('user_profile_history')) 
-                ? config.shardedUserProfileCollection // 'user_profile_history'
-                : config.shardedProfitabilityCollection; // Fallback
+                ? config.shardedUserProfileCollection
+                : config.shardedProfitabilityCollection;
             docRef = deps.db.collection(collection).doc(docPath);
         }
-        // Ensure data is a valid object before pushing
         if (docData && typeof docData === 'object' && !Array.isArray(docData)) {
              shardedDocWrites.push({ ref: docRef, data: docData });
         } else {
              logger.log('ERROR', `[${passName}] Invalid sharded document data for ${docPath}. Not an object.`, { data: docData });
         }
-        // Commit this single document write (or small batch if logic is changed later)
         if (shardedDocWrites.length > 0) {
-            // Use the docPath in the operation name for clearer logging
             await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${docPath} ${dStr}`);
         }
     }
@@ -216,8 +361,17 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
 /** Stage 10: Run meta computations */
 async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
-    if (calcs.length === 0) return;
+    // --- THIS IS THE CRITICAL CHANGE ---
+    if (calcs.length === 0) {
+        logger.log('INFO', `[${passName}] No meta calcs to run for ${dStr} after filtering.`);
+        return;
+    }
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
+    
+    // --- NEW: Load historical data for meta calcs if needed ---
+    // (This might be redundant if standard pass ran, but meta-calcs can run standalone)
+    const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
+
     let success = 0;
     const standardWrites = [];
     const shardedWrites = {}; 
@@ -226,7 +380,8 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
         if (typeof Cl !== 'function') {logger.log('ERROR', `Invalid class ${name}`);continue;}
         const inst = new Cl();
         try {
-            const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData }, config, fetchedDeps));
+            // --- MODIFIED: Pass fullRoot to meta calcs ---
+            const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData: fullRoot }, config, fetchedDeps));
             if (result && Object.keys(result).length > 0) {const standardResult = {}; for (const key in result) {
                     if (key.startsWith('sharded_')) {const shardedData = result[key];for (const collectionName in shardedData) 
                         {if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};Object.assign(shardedWrites[collectionName], shardedData[collectionName]);}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.139",
+  "version": "1.0.141",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [