bulltrackers-module 1.0.137 → 1.0.139

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -25,12 +25,81 @@ async function runComputationPass(config, dependencies, computationManifest) {
     logger.log('INFO', `🚀 Starting PASS ${passToRun}...`);
     const yesterday = new Date(); yesterday.setUTCDate(yesterday.getUTCDate()-1); 
     const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
-    const earliestDates = await getEarliestDataDates(config, dependencies);
-    const firstDate = earliestDates.absoluteEarliest; 
-    const startDateUTC = firstDate ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate())) : new Date(config.earliestComputationDate+'T00:00:00Z');
-    const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
+    
+    // const earliestDates = await getEarliestDataDates(config, dependencies); // <-- 1. COMMENT OUT OR REMOVE THIS BROKEN CALL
+
+    // --- START: 2. NEW HARDCODED DATES (V2 - CORRECTED) ---
+    logger.log('INFO', 'Using hardcoded earliest data dates to bypass faulty discovery.');
+
+    // Use the exact dates you provided:
+    const earliestPortfolio = new Date('2025-09-25T00:00:00Z'); // Earliest of Normal (09-25) and Spec (09-29)
+    const earliestHistory = new Date('2025-11-05T00:00:00Z');   // Both Normal and Spec history
+    const earliestSocial = new Date('2025-10-30T00:00:00Z');    // daily_social_insights
+    const earliestInsights = new Date('2025-08-26T00:00:00Z');   // daily_instrument_insights (Your new date)
+    
+    const earliestDates = {
+        portfolio: earliestPortfolio,
+        history: earliestHistory,
+        social: earliestSocial,
+        insights: earliestInsights, // <-- Use the real date
+        // Calculate absoluteEarliest based on all real data
+        absoluteEarliest: [earliestPortfolio, earliestHistory, earliestSocial, earliestInsights].reduce((a, b) => a < b ? a : b)
+    };
+    
+    logger.log('INFO', `Hardcoded map: portfolio=${earliestDates.portfolio.toISOString().slice(0,10)}, history=${earliestDates.history.toISOString().slice(0,10)}, social=${earliestDates.social.toISOString().slice(0,10)}, insights=${earliestDates.insights.toISOString().slice(0,10)}`);
+    // --- END: NEW HARDCODED DATES ---
+
+
     const passes = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || []; if (!calcsInThisPass.length) return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
+    
+    // --- 3. THIS LOGIC BLOCK (from our last fix) IS NOW CORRECT ---
+    // It will consume the clean, hardcoded 'earliestDates' map and work as intended.
+    const requiredRootData = new Set();
+    calcsInThisPass.forEach(c => {
+        (c.rootDataDependencies || []).forEach(dep => requiredRootData.add(dep));
+    });
+
+    let earliestStartDateForPass = null;
+    if (requiredRootData.size > 0) {
+        let latestOfEarliestDates = new Date(0); // Start at epoch
+        const farFutureSentinelYear = 2999; // Keep as a safety check for any future issues
+        let hasAtLeastOneValidDate = false; 
+
+        requiredRootData.forEach(dep => {
+            const earliestDateForDep = earliestDates[dep];
+            
+            // This check is now just a safeguard
+            if (earliestDateForDep && earliestDateForDep.getUTCFullYear() < farFutureSentinelYear) {
+                if (earliestDateForDep > latestOfEarliestDates) {
+                    latestOfEarliestDates = earliestDateForDep;
+                }
+                hasAtLeastOneValidDate = true; 
+            } else if (earliestDateForDep) {
+                logger.log('INFO', `[PassRunner] Dependency '${dep}' is not available (date: ${earliestDateForDep.toISOString().slice(0, 10)}). Calcs requiring it will be skipped.`);
+            } else {
+                 logger.log('WARN', `[PassRunner] Dependency '${dep}' has no earliest date defined in hardcoded map.`);
+            }
+        });
+        
+        if (hasAtLeastOneValidDate) {
+             earliestStartDateForPass = latestOfEarliestDates;
+        }
+    }
+
+    // Use the pass-specific date. Fall back to absolute earliest, then config.
+    const firstDate = earliestStartDateForPass || earliestDates.absoluteEarliest;
+    const startDateUTC = firstDate 
+        ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate())) 
+        : new Date(config.earliestComputationDate+'T00:00:00Z');
+    
+    logger.log('INFO', `[PassRunner] Pass ${passToRun} requires data: [${Array.from(requiredRootData).join(', ')}].`);
+    logger.log('INFO', `[PassRunner] Determined start date for this pass: ${startDateUTC.toISOString().slice(0, 10)}`);
+    // --- END OF LOGIC BLOCK ---
+
+    const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
+    const firstDayOfBackfill = allExpectedDates.length > 0 ? allExpectedDates[0] : null; // --- MOVED FROM ABOVE ---
+    
     const standardCalcs = calcsInThisPass.filter(c => c.type==='standard'); 
     const metaCalcs = calcsInThisPass.filter(c => c.type==='meta');
     const processDate = async (dateStr) => {
@@ -39,7 +108,7 @@ async function runComputationPass(config, dependencies, computationManifest) {
             const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
             if (!rootData) { logger.log('WARN', `[PassRunner] Skipping ${dateStr} for Pass ${passToRun}: No root data.`);return;}
             const existingResults = await fetchExistingResults(dateStr, calcsInThisPass, computationManifest, config, dependencies);
-            const { standardCalcsToRun, metaCalcsToRun } = filterCalculations(standardCalcs, metaCalcs, rootData.status, existingResults, passToRun, dateStr, logger);
+            const { standardCalcsToRun, metaCalcsToRun } = filterCalculations(standardCalcs, metaCalcs, rootData.status, existingResults, passToRun, dateStr, logger,dateStr === firstDayOfBackfill );            
             if (standardCalcsToRun.length === 0 && metaCalcsToRun.length === 0) {logger.log('INFO', `[PassRunner] All calcs for ${dateStr} Pass ${passToRun} are already complete. Skipping.`);return;}
             if (standardCalcsToRun.length) await runStandardComputationPass(dateToProcess, standardCalcsToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData);
             if (metaCalcsToRun.length) await runMetaComputationPass(dateToProcess, metaCalcsToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, rootData);

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -72,20 +72,26 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
 
 /** --- MODIFIED: Stage 5: Filter calculations to skip completed work ---
  */
-function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, logger) {
+function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, logger, isFirstDayOfBackfill = false) { 
     const skipped = new Set();
-    // Filter Standard Calcs
     const standardCalcsToRun = standardCalcs.filter(c => {
         if (existingResults[c.name]) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Result already exists.`);return false;}
-        const { canRun, missing } = checkRootDependencies(c, rootDataStatus);
-        if (canRun) return true;
-        logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Missing root data: [${missing.join(', ')}]`);
-        skipped.add(c.name);
-        return false;
+        if (isFirstDayOfBackfill && c.isHistorical) {
+            logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Historical calc on the first day of backfill (no yesterday).`);
+            skipped.add(c.name);
+            return false;
+        }
     });
     // Filter Meta Calcs
     const metaCalcsToRun = metaCalcs.filter(c => {
         if (existingResults[c.name]) {logger.log('TRACE', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Result already exists.`);skipped.add(c.name);return false;}
+        // --- START: RECOMMENDED ADDITION ---
+        if (isFirstDayOfBackfill && c.isHistorical) {
+            logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Historical calc on the first day of backfill (no yesterday).`);
+            skipped.add(c.name);
+            return false;
+        }
+        // --- END: RECOMMENDED ADDITION ---
         // 1. Check root data
         const { canRun, missing: missingRoot } = checkRootDependencies(c, rootDataStatus);
         if (!canRun) {logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing root data: [${missingRoot.join(', ')}]`);skipped.add(c.name);return false;}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.137",
+  "version": "1.0.139",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [