bulltrackers-module 1.0.342 → 1.0.343

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,8 +1,8 @@
 /**
  * FILENAME: computation-system/helpers/computation_dispatcher.js
  * PURPOSE: Sequential Cursor-Based Dispatcher.
+ * UPDATED: Implemented "Fast-Forward" Scanning Loop to skip empty dates efficiently.
  * UPDATED: Enforces Strict One-Shot Policy (Standard -> HighMem -> Dead Letter).
- * UPDATED: Prevents infinite loops by permanently ignoring deterministic failures.
  * UPDATED: Generates Google Cloud Trace Context (traceId/spanId) for end-to-end monitoring.
  */
 
@@ -12,7 +12,7 @@ const { PubSubUtils } = require('../../core/utils/pubsub_utils');
 const { fetchComputationStatus } = require('../persistence/StatusRepository');
 const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
 const crypto = require('crypto');
-const monConfig = require('../config/monitoring_config'); // [NEW]
+const monConfig = require('../config/monitoring_config');
 
 const OOM_THRESHOLD_MB = 1500;
 const BASE_SECONDS_PER_WEIGHT_UNIT = 3;
@@ -281,7 +281,6 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
     const currentDispatchId = crypto.randomUUID();
     
     const tasksPayload = validTasks.map(t => {
-        // [NEW] Generate Eyeball Trace
         const traceId = crypto.randomBytes(16).toString('hex');
         const spanId = crypto.randomBytes(8).toString('hex');
 
@@ -294,7 +293,6 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
             dispatchId: currentDispatchId,
             triggerReason: 'SWEEP_RECOVERY',
             resources: 'high-mem', // FORCE
-            // [NEW] Attach the eyeball
             traceContext: {
                 traceId: traceId,
                 spanId: spanId,
@@ -315,7 +313,7 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
 }
 
 // =============================================================================
-// LOGIC: Standard Dispatch (Original)
+// LOGIC: Standard Dispatch (Fast-Forward Enabled)
 // =============================================================================
 async function handleStandardDispatch(config, dependencies, computationManifest, reqBody) {
     const { logger, db } = dependencies;
@@ -336,16 +334,29 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
     const sessionDates = await getStableDateSession(config, dependencies, passToRun, dateLimitStr, forceRebuild);
     if (!sessionDates || sessionDates.length === 0) return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
 
-    let selectedDate = null;
+    // --- Fast-Forward Loop Configuration ---
+    // Scans up to 50 dates or 40 seconds to find work, avoiding empty "wait loops"
+    const MAX_SCAN_DEPTH = 50;
+    const TIME_LIMIT_MS = 40000; 
+    const startTime = Date.now();
+
+    let currentCursor = targetCursorN;
     let selectedTasks = [];
+    let selectedDate = null;
+    let datesScanned = 0;
 
-    if (targetCursorN <= sessionDates.length) {
-        selectedDate = sessionDates[targetCursorN - 1];
-    } else {
-        return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
-    }
+    // Loop until work is found, end is reached, or safety limits hit
+    while (currentCursor <= sessionDates.length) {
+        datesScanned++;
+        selectedDate = sessionDates[currentCursor - 1]; // 0-indexed array
+
+        // 1. Safety Break (Prevent Timeout)
+        if ((Date.now() - startTime) > TIME_LIMIT_MS || datesScanned > MAX_SCAN_DEPTH) {
+            logger.log('INFO', `[Dispatcher] ⏩ Fast-forward paused at ${selectedDate} after scanning ${datesScanned} dates.`);
+            break;
+        }
 
-    if (selectedDate) {
+        // 2. Analyze Date
         const earliestDates = await getEarliestDataDates(config, dependencies);
         let prevDailyStatusPromise = Promise.resolve(null);
         if (calcsInThisPass.some(c => c.isHistorical)) {
@@ -362,48 +373,70 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
             checkRootDataAvailability(selectedDate, config, dependencies, DEFINITIVE_EARLIEST_DATES)
         ]);
 
-        if (availability && availability.status) { // Just check availability exists
+        if (availability && availability.status) {
             const report = analyzeDateExecution(selectedDate, calcsInThisPass, availability.status, dailyStatus, manifestMap, prevDailyStatus);
             let rawTasks = [...report.runnable, ...report.reRuns];
 
             if (rawTasks.length > 0) {
                 rawTasks = await attemptSimHashResolution(dependencies, selectedDate, rawTasks, dailyStatus, manifestMap);
-                selectedTasks = await filterActiveTasks(db, selectedDate, passToRun, rawTasks, logger);
-            }
-            
-            if (selectedTasks.length > 0) {
-                // Split Logic: Moves OOMs to High-Mem, drops dead letters
-                const { standard, highMem } = await splitRoutes(db, selectedDate, passToRun, selectedTasks, logger);
-                selectedTasks = [...standard, ...highMem];
+                const activeTasks = await filterActiveTasks(db, selectedDate, passToRun, rawTasks, logger);
+                
+                if (activeTasks.length > 0) {
+                    const { standard, highMem } = await splitRoutes(db, selectedDate, passToRun, activeTasks, logger);
+                    selectedTasks = [...standard, ...highMem];
+                    
+                    if (selectedTasks.length > 0) {
+                        // Found work! Break loop to dispatch.
+                        break; 
+                    }
+                }
             }
         }
+
+        // No work found for this date. Fast-forward to next.
+        currentCursor++;
     }
 
+    // --- Result Handling ---
+
+    // Case 1: Satiated (Scanned to end of session with no work)
+    if (currentCursor > sessionDates.length && selectedTasks.length === 0) {
+        return {
+            status: 'CONTINUE_PASS',
+            dateProcessed: selectedDate,
+            dispatched: 0,
+            n_cursor_ignored: false,
+            remainingDates: 0,
+            nextCursor: currentCursor // Matches length + 1
+        };
+    }
+
+    // Case 2: Paused by Limit (No work found yet, but more dates remain)
     if (selectedTasks.length === 0) {
         return {
             status: 'CONTINUE_PASS',
             dateProcessed: selectedDate,
             dispatched: 0,
             n_cursor_ignored: false,
-            etaSeconds: 0,
-            remainingDates: sessionDates.length - targetCursorN
+            remainingDates: sessionDates.length - currentCursor + 1,
+            nextCursor: currentCursor // Resume from here
         };
     }
 
+    // Case 3: Work Found (Dispatching)
     const totalweight = selectedTasks.reduce((sum, t) => sum + (manifestWeightMap.get(normalizeName(t.name)) || 1.0), 0);
     const currentDispatchId = crypto.randomUUID();
     const etaSeconds = Math.max(20, Math.ceil(totalweight * BASE_SECONDS_PER_WEIGHT_UNIT));
 
-    const taskDetails = selectedTasks.map(t => `${t.name} (${t.reason})`);
-    logger.log('INFO', `[Dispatcher] ✅ Dispatching ${selectedTasks.length} tasks for ${selectedDate}.`, {
-        date: selectedDate, pass: passToRun, dispatchedCount: selectedTasks.length, etaSeconds, dispatchId: currentDispatchId, details: taskDetails
-    });
+    if (datesScanned > 1) {
+        logger.log('INFO', `[Dispatcher] ⏩ Fast-forwarded ${datesScanned - 1} empty dates. Dispatching ${selectedTasks.length} tasks for ${selectedDate}.`);
+    } else {
+        logger.log('INFO', `[Dispatcher] ✅ Dispatching ${selectedTasks.length} tasks for ${selectedDate}.`);
+    }
 
     const mapToTaskPayload = (t) => {
-        // [NEW] Generate Eyeball Trace
         const traceId = crypto.randomBytes(16).toString('hex');
         const spanId = crypto.randomBytes(8).toString('hex');
-
         return {
             ...t, 
             action: 'RUN_COMPUTATION_DATE', 
@@ -413,7 +446,6 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
             dispatchId: currentDispatchId, 
             triggerReason: t.reason, 
             resources: t.resources || 'standard',
-            // [NEW] Attach the eyeball
             traceContext: {
                 traceId: traceId,
                 spanId: spanId,
@@ -448,7 +480,8 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
         dispatched: selectedTasks.length,
         n_cursor_ignored: false,
         etaSeconds: etaSeconds,
-        remainingDates: sessionDates.length - targetCursorN
+        remainingDates: sessionDates.length - targetCursorN,
+        nextCursor: currentCursor + 1 // Start next scan AFTER this date
     };
 }
 
@@ -465,32 +498,25 @@ async function splitRoutes(db, date, pass, tasks, logger) {
         const doc = await db.doc(ledgerPath).get();
         
         if (!doc.exists) {
-            // New task -> Standard
             standard.push(task);
             continue;
         }
 
         const data = doc.data();
         
-        // If it FAILED, we check if we can escalate it.
         if (data.status === 'FAILED') {
             const stage = data.error?.stage;
 
-            // 1. QUALITY / LOGIC FAIL: Dead Letter (Drop it)
             if (['QUALITY_CIRCUIT_BREAKER', 'SEMANTIC_GATE'].includes(stage)) {
                 logger.log('WARN', `[Dispatcher] 🛑 Dropping ${name} - Deterministic Failure (${stage}).`);
                 continue; 
             }
 
-            // 2. PREVIOUSLY HIGH MEM FAIL: Dead Letter (Drop it)
             if (data.resourceTier === 'high-mem') {
                 logger.log('WARN', `[Dispatcher] 🛑 Dropping ${name} - Failed on High-Mem already.`);
                 continue;
             }
 
-            // 3. STANDARD FAIL (Crash/OOM): Promote to High Mem (Retry)
-            // If it failed standard, we give it ONE shot on high-mem.
-            // Note: Even if it was an "Unknown" error, we promote to High-Mem to cover OOMs that looked like crashes.
             highMem.push({ 
                 ...task, 
                 resources: 'high-mem', 
@@ -498,7 +524,6 @@ async function splitRoutes(db, date, pass, tasks, logger) {
             });
 
         } else {
-            // If status is not FAILED (e.g. was Zombie and filterActiveTasks passed it), retry Standard.
             standard.push(task);
         }
     }

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,6 +1,6 @@
 # Cloud Workflows: Precision Cursor-Based Orchestrator
-# SIMPLE MODE: Dispatch -> Wait ETA -> Next Date
-# UPGRADED: Added "Sweep & Verify" Step for High-Mem Recovery
+# FAST-FORWARD MODE: Dispatch -> (Internal Loop) -> Wait ETA -> Jump to Next Valid Cursor
+# UPGRADED: Supports 'nextCursor' to skip empty date ranges instantly.
 
 main:
   params: [input]
@@ -39,12 +39,12 @@ main:
 
                       - evaluate_dispatch:
                           switch:
-                            # 1. End of Session (Dispatcher reached end of date list)
+                            # 1. End of Session (Dispatcher reached absolute end of date list)
                             - condition: '${dispatch_res.body.status == "MOVE_TO_NEXT_PASS"}'
                               assign:
                                 - pass_complete: true
                             
-                            # 2. Satiation Check (Specific to date/logic)
+                            # 2. Satiation Check (Dispatcher scanned to end, found nothing)
                             - condition: '${dispatch_res.body.status == "CONTINUE_PASS" and dispatch_res.body.remainingDates == 0 and dispatch_res.body.dispatched == 0}'
                               steps:
                                 - log_satiation:
@@ -55,7 +55,7 @@ main:
                                     assign:
                                       - pass_complete: true
 
-                            # 3. Work Dispatched: Wait ETA -> Move Next (Ignored flag is FALSE)
+                            # 3. Work Dispatched: Wait ETA -> Jump to 'nextCursor'
                             - condition: '${dispatch_res.body.dispatched > 0}'
                               steps:
                                 - wait_for_completion:
@@ -64,25 +64,26 @@ main:
                                       seconds: '${int(dispatch_res.body.etaSeconds)}'
                                 - update_cursor:
                                     assign:
-                                      - n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
+                                      # CRITICAL CHANGE: Use dispatcher's specific jump target
+                                      - n_cursor: '${default(dispatch_res.body.nextCursor, n_cursor + 1)}'
                                 - next_loop_work:
                                     next: sequential_date_loop
 
-                            # 4. No Work (Clean or Busy): Move Next Immediately
+                            # 4. No Work (Fast-Forward limit reached or Single Check): Jump to 'nextCursor'
                             - condition: '${dispatch_res.body.dispatched == 0}'
                               steps:
                                 - wait_short:
                                     call: sys.sleep
                                     args:
-                                      seconds: 2 # Tiny debounce
+                                      seconds: 2 # Tiny debounce for API limits
                                 - update_cursor_retry:
                                     assign:
-                                        # Dispatcher sends n_cursor_ignored=false, so we increment.
-                                        - n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
+                                        # CRITICAL CHANGE: Resume exactly where the Fast-Forward loop stopped
+                                        - n_cursor: '${default(dispatch_res.body.nextCursor, n_cursor + 1)}'
                                 - next_loop_retry:
                                     next: sequential_date_loop
             
-            # --- NEW STEP: VERIFICATION & SWEEP ---
+            # --- VERIFICATION & SWEEP (No changes needed here) ---
             - verify_pass_completion:
                 call: http.post
                 args:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.342",
+  "version": "1.0.343",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [