bulltrackers-module 1.0.139 → 1.0.140

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -1,125 +1,210 @@
 /**
- * @fileoverview
- * Main "Pass Runner" for the V2 Computation System.
- *
- * This orchestrator is designed to be run by a separate Cloud Function for each "pass".
- * It reads its pass number from the config and executes only those calculations.
- * This file contains the high-level "manual" of steps. The "how-to" logic
- * is extracted into 'computation_system_utils.js'.
- * --- MODIFIED: To use getEarliestDataDates and pass the date map to the orchestrator helpers. ---
- * --- MODIFIED: To run date processing in parallel batches. ---
- * --- MODIFIED: To fetch ALL existing results to enable incremental (skip) logic. ---
+ * FIXED: computation_pass_runner.js
+ * Now calculates earliest date PER CALCULATION, not per pass
  */
 
-// --- MODIFIED: Renamed fetchDependenciesForPass to fetchExistingResults ---
 const { groupByPass, checkRootDataAvailability, fetchExistingResults, filterCalculations, runStandardComputationPass, runMetaComputationPass } = require('./orchestration_helpers.js');
-// --- MODIFIED: Import getEarliestDataDates ---
 const { getExpectedDateStrings, getEarliestDataDates } = require('../utils/utils.js');
 
-// --- NEW: Parallel processing batch size ---
-const PARALLEL_BATCH_SIZE = 7; // Process a week at a time
+const PARALLEL_BATCH_SIZE = 7;
 
 async function runComputationPass(config, dependencies, computationManifest) {
     const { logger } = dependencies;
-    const passToRun = String(config.COMPUTATION_PASS_TO_RUN); if (!passToRun) return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
+    const passToRun = String(config.COMPUTATION_PASS_TO_RUN);
+    if (!passToRun) return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
+    
     logger.log('INFO', `🚀 Starting PASS ${passToRun}...`);
-    const yesterday = new Date(); yesterday.setUTCDate(yesterday.getUTCDate()-1); 
+    
+    const yesterday = new Date();
+    yesterday.setUTCDate(yesterday.getUTCDate() - 1);
     const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
     
-    // const earliestDates = await getEarliestDataDates(config, dependencies); // <-- 1. COMMENT OUT OR REMOVE THIS BROKEN CALL
-
-    // --- START: 2. NEW HARDCODED DATES (V2 - CORRECTED) ---
+    // Hardcoded earliest dates
     logger.log('INFO', 'Using hardcoded earliest data dates to bypass faulty discovery.');
-
-    // Use the exact dates you provided:
-    const earliestPortfolio = new Date('2025-09-25T00:00:00Z'); // Earliest of Normal (09-25) and Spec (09-29)
-    const earliestHistory = new Date('2025-11-05T00:00:00Z');   // Both Normal and Spec history
-    const earliestSocial = new Date('2025-10-30T00:00:00Z');    // daily_social_insights
-    const earliestInsights = new Date('2025-08-26T00:00:00Z');   // daily_instrument_insights (Your new date)
+    const earliestPortfolio = new Date('2025-09-25T00:00:00Z');
+    const earliestHistory = new Date('2025-11-05T00:00:00Z');
+    const earliestSocial = new Date('2025-10-30T00:00:00Z');
+    const earliestInsights = new Date('2025-08-26T00:00:00Z');
     
     const earliestDates = {
         portfolio: earliestPortfolio,
         history: earliestHistory,
         social: earliestSocial,
-        insights: earliestInsights, // <-- Use the real date
-        // Calculate absoluteEarliest based on all real data
+        insights: earliestInsights,
         absoluteEarliest: [earliestPortfolio, earliestHistory, earliestSocial, earliestInsights].reduce((a, b) => a < b ? a : b)
     };
     
     logger.log('INFO', `Hardcoded map: portfolio=${earliestDates.portfolio.toISOString().slice(0,10)}, history=${earliestDates.history.toISOString().slice(0,10)}, social=${earliestDates.social.toISOString().slice(0,10)}, insights=${earliestDates.insights.toISOString().slice(0,10)}`);
-    // --- END: NEW HARDCODED DATES ---
-
-
+    
     const passes = groupByPass(computationManifest);
-    const calcsInThisPass = passes[passToRun] || []; if (!calcsInThisPass.length) return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
-    
-    // --- 3. THIS LOGIC BLOCK (from our last fix) IS NOW CORRECT ---
-    // It will consume the clean, hardcoded 'earliestDates' map and work as intended.
-    const requiredRootData = new Set();
-    calcsInThisPass.forEach(c => {
-        (c.rootDataDependencies || []).forEach(dep => requiredRootData.add(dep));
-    });
-
-    let earliestStartDateForPass = null;
-    if (requiredRootData.size > 0) {
-        let latestOfEarliestDates = new Date(0); // Start at epoch
-        const farFutureSentinelYear = 2999; // Keep as a safety check for any future issues
-        let hasAtLeastOneValidDate = false; 
-
-        requiredRootData.forEach(dep => {
-            const earliestDateForDep = earliestDates[dep];
-            
-            // This check is now just a safeguard
-            if (earliestDateForDep && earliestDateForDep.getUTCFullYear() < farFutureSentinelYear) {
-                if (earliestDateForDep > latestOfEarliestDates) {
-                    latestOfEarliestDates = earliestDateForDep;
-                }
-                hasAtLeastOneValidDate = true; 
-            } else if (earliestDateForDep) {
-                logger.log('INFO', `[PassRunner] Dependency '${dep}' is not available (date: ${earliestDateForDep.toISOString().slice(0, 10)}). Calcs requiring it will be skipped.`);
-            } else {
-                 logger.log('WARN', `[PassRunner] Dependency '${dep}' has no earliest date defined in hardcoded map.`);
+    const calcsInThisPass = passes[passToRun] || [];
+    if (!calcsInThisPass.length) return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
+    
+    // ============================================
+    // NEW: Calculate earliest date PER CALCULATION
+    // ============================================
+    const calcEarliestDates = new Map();
+    
+    for (const calc of calcsInThisPass) {
+        const deps = calc.rootDataDependencies || [];
+        
+        if (deps.length === 0) {
+            // No dependencies = can run from the absolute earliest
+            calcEarliestDates.set(calc.name, earliestDates.absoluteEarliest);
+            continue;
+        }
+        
+        // Find the LATEST earliest date among THIS calculation's dependencies
+        let latestEarliest = new Date(0);
+        for (const dep of deps) {
+            const depDate = earliestDates[dep];
+            if (depDate && depDate > latestEarliest) {
+                latestEarliest = depDate;
             }
-        });
+        }
         
-        if (hasAtLeastOneValidDate) {
-             earliestStartDateForPass = latestOfEarliestDates;
+        // If this is a historical calculation, add 1 day (needs yesterday)
+        if (calc.isHistorical) {
+            const adjusted = new Date(latestEarliest);
+            adjusted.setUTCDate(adjusted.getUTCDate() + 1);
+            calcEarliestDates.set(calc.name, adjusted);
+            logger.log('TRACE', `[PassRunner] ${calc.name}: earliest=${adjusted.toISOString().slice(0,10)} (historical, needs ${latestEarliest.toISOString().slice(0,10)} + 1 day)`);
+        } else {
+            calcEarliestDates.set(calc.name, latestEarliest);
+            logger.log('TRACE', `[PassRunner] ${calc.name}: earliest=${latestEarliest.toISOString().slice(0,10)}`);
         }
     }
-
-    // Use the pass-specific date. Fall back to absolute earliest, then config.
-    const firstDate = earliestStartDateForPass || earliestDates.absoluteEarliest;
-    const startDateUTC = firstDate 
-        ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate())) 
-        : new Date(config.earliestComputationDate+'T00:00:00Z');
-    
-    logger.log('INFO', `[PassRunner] Pass ${passToRun} requires data: [${Array.from(requiredRootData).join(', ')}].`);
-    logger.log('INFO', `[PassRunner] Determined start date for this pass: ${startDateUTC.toISOString().slice(0, 10)}`);
-    // --- END OF LOGIC BLOCK ---
-
+    
+    // The pass can start from the EARLIEST calculation's start date
+    const passEarliestDate = new Date(Math.min(...Array.from(calcEarliestDates.values()).map(d => d.getTime())));
+    
+    logger.log('INFO', `[PassRunner] Pass ${passToRun} analysis:`);
+    logger.log('INFO', `  Total calculations: ${calcsInThisPass.length}`);
+    logger.log('INFO', `  Pass can start from: ${passEarliestDate.toISOString().slice(0,10)}`);
+    logger.log('INFO', `  Individual calculation date ranges calculated.`);
+    
+    // ============================================
+    // Generate date range from earliest to yesterday
+    // ============================================
+    const startDateUTC = new Date(Date.UTC(passEarliestDate.getUTCFullYear(), passEarliestDate.getUTCMonth(), passEarliestDate.getUTCDate()));
     const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
-    const firstDayOfBackfill = allExpectedDates.length > 0 ? allExpectedDates[0] : null; // --- MOVED FROM ABOVE ---
     
-    const standardCalcs = calcsInThisPass.filter(c => c.type==='standard'); 
-    const metaCalcs = calcsInThisPass.filter(c => c.type==='meta');
+    logger.log('INFO', `[PassRunner] Processing ${allExpectedDates.length} total dates from ${allExpectedDates[0]} to ${allExpectedDates[allExpectedDates.length-1]}`);
+    
+    const standardCalcs = calcsInThisPass.filter(c => c.type === 'standard');
+    const metaCalcs = calcsInThisPass.filter(c => c.type === 'meta');
+    
+    // ============================================
+    // Process each date
+    // ============================================
     const processDate = async (dateStr) => {
-        const dateToProcess = new Date(dateStr+'T00:00:00Z');
+        const dateToProcess = new Date(dateStr + 'T00:00:00Z');
+        
         try {
+            // Check root data availability
             const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
-            if (!rootData) { logger.log('WARN', `[PassRunner] Skipping ${dateStr} for Pass ${passToRun}: No root data.`);return;}
+            if (!rootData) {
+                logger.log('WARN', `[PassRunner] Skipping ${dateStr} for Pass ${passToRun}: No root data.`);
+                return;
+            }
+            
+            // Fetch existing results
             const existingResults = await fetchExistingResults(dateStr, calcsInThisPass, computationManifest, config, dependencies);
-            const { standardCalcsToRun, metaCalcsToRun } = filterCalculations(standardCalcs, metaCalcs, rootData.status, existingResults, passToRun, dateStr, logger,dateStr === firstDayOfBackfill );            
-            if (standardCalcsToRun.length === 0 && metaCalcsToRun.length === 0) {logger.log('INFO', `[PassRunner] All calcs for ${dateStr} Pass ${passToRun} are already complete. Skipping.`);return;}
-            if (standardCalcsToRun.length) await runStandardComputationPass(dateToProcess, standardCalcsToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData);
-            if (metaCalcsToRun.length) await runMetaComputationPass(dateToProcess, metaCalcsToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, rootData);
-        } catch (err) {logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack });}
+            
+            // ============================================
+            // NEW: Filter based on per-calculation earliest dates
+            // ============================================
+            const standardCalcsToRun = standardCalcs.filter(c => {
+                // Skip if already exists
+                if (existingResults[c.name]) {
+                    logger.log('TRACE', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Result already exists.`);
+                    return false;
+                }
+                
+                // Skip if date is before this calculation's earliest date
+                const calcEarliestDate = calcEarliestDates.get(c.name);
+                if (calcEarliestDate && dateToProcess < calcEarliestDate) {
+                    logger.log('TRACE', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Date before calc's earliest (${calcEarliestDate.toISOString().slice(0,10)}).`);
+                    return false;
+                }
+                
+                // Check root data dependencies
+                const missingDeps = (c.rootDataDependencies || []).filter(dep => {
+                    if (dep === 'portfolio') return !rootData.status.hasPortfolio;
+                    if (dep === 'insights') return !rootData.status.hasInsights;
+                    if (dep === 'social') return !rootData.status.hasSocial;
+                    if (dep === 'history') return !rootData.status.hasHistory;
+                    return false;
+                });
+                
+                if (missingDeps.length > 0) {
+                    logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Missing deps: ${missingDeps.join(', ')}`);
+                    return false;
+                }
+                
+                return true;
+            });
+            
+            const metaCalcsToRun = metaCalcs.filter(c => {
+                if (existingResults[c.name]) return false;
+                
+                const calcEarliestDate = calcEarliestDates.get(c.name);
+                if (calcEarliestDate && dateToProcess < calcEarliestDate) {
+                    logger.log('TRACE', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Date before calc's earliest.`);
+                    return false;
+                }
+                
+                // Check root dependencies
+                const missingRootDeps = (c.rootDataDependencies || []).filter(dep => {
+                    if (dep === 'portfolio') return !rootData.status.hasPortfolio;
+                    if (dep === 'insights') return !rootData.status.hasInsights;
+                    if (dep === 'social') return !rootData.status.hasSocial;
+                    if (dep === 'history') return !rootData.status.hasHistory;
+                    return false;
+                });
+                
+                if (missingRootDeps.length > 0) {
+                    logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing root deps: ${missingRootDeps.join(', ')}`);
+                    return false;
+                }
+                
+                // Check computed dependencies
+                const missingComputedDeps = (c.dependencies || []).filter(d => !existingResults[d]);
+                if (missingComputedDeps.length > 0) {
+                    logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing computed deps: ${missingComputedDeps.join(', ')}`);
+                    return false;
+                }
+                
+                return true;
+            });
+            
+            if (standardCalcsToRun.length === 0 && metaCalcsToRun.length === 0) {
+                logger.log('INFO', `[PassRunner] All eligible calcs for ${dateStr} Pass ${passToRun} are already complete. Skipping.`);
+                return;
+            }
+            
+            logger.log('INFO', `[PassRunner] Running ${dateStr}: ${standardCalcsToRun.length} standard, ${metaCalcsToRun.length} meta`);
+            
+            if (standardCalcsToRun.length) {
+                await runStandardComputationPass(dateToProcess, standardCalcsToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData);
+            }
+            if (metaCalcsToRun.length) {
+                await runMetaComputationPass(dateToProcess, metaCalcsToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, rootData);
+            }
+            
+        } catch (err) {
+            logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
+        }
     };
+    
+    // Process in batches
     logger.log('INFO', `[PassRunner] Processing ${allExpectedDates.length} total dates in batches of ${PARALLEL_BATCH_SIZE}...`);
     for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) {
         const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
         logger.log('INFO', `[PassRunner] Processing batch ${Math.floor(i / PARALLEL_BATCH_SIZE) + 1}/${Math.ceil(allExpectedDates.length / PARALLEL_BATCH_SIZE)} (Dates: ${batch[0]}...${batch[batch.length-1]})`);
         await Promise.all(batch.map(dateStr => processDate(dateStr)));
     }
+    
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.139",
+  "version": "1.0.140",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [