bulltrackers-module 1.0.653 → 1.0.654

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -5,16 +5,14 @@
  * UPDATED: Enforces Strict One-Shot Policy (Standard -> HighMem -> Dead Letter).
  * UPDATED: Generates Google Cloud Trace Context (traceId/spanId) for end-to-end monitoring.
  * UPDATED: Added Schedule Awareness (Daily, Weekly, Monthly) to filter tasks by date.
- * UPDATED: Force Run now validates Root Data Availability before dispatching.
  */
 
 const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
 const { groupByPass, analyzeDateExecution } = require('../WorkflowOrchestrator.js');
 const { PubSubUtils } = require('../../core/utils/pubsub_utils');
 const { fetchComputationStatus } = require('../persistence/StatusRepository');
-// [UPDATED] Imported getAvailabilityWindow and checkRootDependencies
-const { checkRootDataAvailability, getAvailabilityWindow, checkRootDependencies } = require('../data/AvailabilityChecker');
-const { runFinalSweepCheck } = require('../tools/FinalSweepReporter'); 
+const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
+const { runFinalSweepCheck } = require('../tools/FinalSweepReporter'); // [NEW]
 const crypto = require('crypto');
 
 const OOM_THRESHOLD_MB = 1500; // Unused
@@ -232,71 +230,92 @@ async function handleForceRun(config, dependencies, computationManifest, reqBody
     }
 
     // 2. Determine Target Dates
-    let targetDates = [];
+    let candidateDates = [];
     if (dateInput) {
         // Single Date Mode
-        // We still perform the check, but for a single item array
-        if (Array.isArray(dateInput)) {
-            targetDates = dateInput;
-        } else {
-            targetDates = [dateInput];
-        }
+        candidateDates = [dateInput];
     } else {
         // All Dates Mode (Backfill)
         logger.log('INFO', `[ForceRun] No date provided. Calculating date range for ${computationName}...`);
         const earliestDates = await getEarliestDataDates(config, dependencies);
         // Calculate from system start until today
-        targetDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date());
+        candidateDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date());
     }
     
-    // [NEW] 3. Filter Impossible Dates (Availability Check)
-    logger.log('INFO', `[ForceRun] 🔍 Validating data availability for ${targetDates.length} candidate dates...`);
-    
-    // Sort dates to get efficient min/max for range query
-    targetDates.sort();
-    const startDate = targetDates[0];
-    const endDate = targetDates[targetDates.length - 1];
+    logger.log('INFO', `[ForceRun] Checking ${candidateDates.length} candidate dates for runnability...`);
 
-    const availabilityMap = await getAvailabilityWindow(dependencies, startDate, endDate);
-    const validDates = [];
-    const skippedStats = {};
-
-    for (const date of targetDates) {
-        const status = availabilityMap.get(date);
+    // 3. Filter to only runnable dates using analyzeDateExecution
+    const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
+    const calcsInPass = groupByPass(computationManifest, manifestItem.pass || "1");
+    const targetComputationNormalized = normalizeName(computationName);
+    
+    // Filter to only the target computation
+    const targetCalcs = calcsInPass.filter(c => normalizeName(c.name) === targetComputationNormalized);
+    
+    if (targetCalcs.length === 0) {
+        throw new Error(`Computation '${computationName}' not found in pass ${manifestItem.pass || "1"}`);
+    }
+    
+    const runnableDates = [];
+    const skippedDates = [];
+    
+    for (const dateStr of candidateDates) {
+        // Check root data availability
+        const rootDataStatus = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
+        
+        // Get computation status for this date
+        const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
         
-        // If no index exists, we pass an empty object (AvailabilityChecker will fail all checks)
-        // This effectively filters out dates where we have NO knowledge of data
-        const effectiveStatus = status || {}; 
+        // Check previous day status if needed
+        let prevDailyStatus = null;
+        if (targetCalcs.some(c => c.isHistorical)) {
+            const prevDate = new Date(dateStr + 'T00:00:00Z');
+            prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+            prevDailyStatus = await fetchComputationStatus(prevDate.toISOString().slice(0, 10), config, dependencies);
+        }
+        
+        // Analyze if this computation can run on this date
+        const report = analyzeDateExecution(dateStr, targetCalcs, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus);
         
-        const check = checkRootDependencies(manifestItem, effectiveStatus);
+        // Check if the target computation is runnable, needs re-run, or has failed dependencies
+        const isRunnable = report.runnable.some(t => normalizeName(t.name) === targetComputationNormalized);
+        const needsReRun = report.reRuns.some(t => normalizeName(t.name) === targetComputationNormalized);
+        const hasFailedDep = report.failedDependency.some(t => normalizeName(t.name) === targetComputationNormalized);
+        const isImpossible = report.impossible.some(t => normalizeName(t.name) === targetComputationNormalized);
+        const isBlocked = report.blocked.some(t => normalizeName(t.name) === targetComputationNormalized);
         
-        if (check.canRun) {
-            validDates.push(date);
+        if (isRunnable || needsReRun || hasFailedDep) {
+            runnableDates.push(dateStr);
+        } else if (isImpossible) {
+            skippedDates.push({ date: dateStr, reason: report.impossible.find(t => normalizeName(t.name) === targetComputationNormalized)?.reason || 'Impossible' });
+        } else if (isBlocked) {
+            skippedDates.push({ date: dateStr, reason: report.blocked.find(t => normalizeName(t.name) === targetComputationNormalized)?.reason || 'Blocked' });
         } else {
-            const reason = check.missing.length > 0 ? `Missing: ${check.missing.join(', ')}` : 'Dependencies not met';
-            skippedStats[reason] = (skippedStats[reason] || 0) + 1;
+            skippedDates.push({ date: dateStr, reason: 'Not runnable (unknown reason)' });
         }
     }
-
-    if (validDates.length === 0) {
-        logger.log('WARN', `[ForceRun] 🛑 ABORTING: No valid dates found for ${computationName} out of ${targetDates.length} requested.`);
+    
+    logger.log('INFO', `[ForceRun] ✅ Found ${runnableDates.length} runnable dates out of ${candidateDates.length} candidates`);
+    if (skippedDates.length > 0) {
+        logger.log('INFO', `[ForceRun] ⏭️ Skipped ${skippedDates.length} dates: ${skippedDates.slice(0, 5).map(s => `${s.date} (${s.reason})`).join(', ')}${skippedDates.length > 5 ? '...' : ''}`);
+    }
+    
+    if (runnableDates.length === 0) {
         return {
-            status: 'ABORTED',
+            status: 'NO_RUNNABLE_DATES',
             computation: computationName,
-            reason: 'NO_DATA_AVAILABLE',
-            skippedDetails: skippedStats
+            mode: dateInput ? 'SINGLE_DATE' : 'ALL_DATES',
+            datesChecked: candidateDates.length,
+            datesRunnable: 0,
+            skippedReasons: skippedDates.slice(0, 10)
         };
     }
+    
+    logger.log('WARN', `[ForceRun] 🚨 MANUALLY Triggering ${computationName} for ${runnableDates.length} runnable dates. Pass: ${manifestItem.pass}`);
 
-    if (validDates.length < targetDates.length) {
-        logger.log('INFO', `[ForceRun] ⚠️ Filtered impossible dates: ${targetDates.length} requested -> ${validDates.length} valid.`, { skippedStats });
-    }
-
-    logger.log('WARN', `[ForceRun] 🚨 MANUALLY Triggering ${computationName} for ${validDates.length} VALID dates. Pass: ${manifestItem.pass}`);
-
-    // 4. Construct Tasks
+    // 4. Construct Tasks (only for runnable dates)
     const dispatchId = crypto.randomUUID();
-    const tasks = validDates.map(date => {
+    const tasks = runnableDates.map(date => {
         const traceId = crypto.randomBytes(16).toString('hex');
         const spanId = crypto.randomBytes(8).toString('hex');
         return {
@@ -312,7 +331,7 @@ async function handleForceRun(config, dependencies, computationManifest, reqBody
         };
     });
 
-    // 5. Batch Publish (Chunked to stay under Pub/Sub limits)
+    // 4. Batch Publish (Chunked to stay under Pub/Sub limits)
     const CHUNK_SIZE = 250; // Safe batch size
     const topic = (reqBody.resources === 'high-mem') 
         ? (config.computationTopicHighMem || 'computation-tasks-highmem')
@@ -337,10 +356,11 @@ async function handleForceRun(config, dependencies, computationManifest, reqBody
     return { 
         status: 'FORCED',
         computation: computationName,
-        mode: dateInput ? 'SINGLE/ARRAY_DATE' : 'ALL_DATES',
-        datesRequested: targetDates.length,
+        mode: dateInput ? 'SINGLE_DATE' : 'ALL_DATES',
+        datesChecked: candidateDates.length,
+        datesRunnable: runnableDates.length,
         datesTriggered: dispatchedCount,
-        skippedImpossible: targetDates.length - dispatchedCount,
+        skippedCount: skippedDates.length,
         targetTopic: topic
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.653",
+  "version": "1.0.654",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [