bulltrackers-module 1.0.342 → 1.0.344

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,8 +1,8 @@
 /**
  * FILENAME: computation-system/helpers/computation_dispatcher.js
  * PURPOSE: Sequential Cursor-Based Dispatcher.
+ * UPDATED: Implemented "Fast-Forward" Scanning Loop to skip empty dates efficiently.
  * UPDATED: Enforces Strict One-Shot Policy (Standard -> HighMem -> Dead Letter).
- * UPDATED: Prevents infinite loops by permanently ignoring deterministic failures.
  * UPDATED: Generates Google Cloud Trace Context (traceId/spanId) for end-to-end monitoring.
  */
 
@@ -12,9 +12,9 @@ const { PubSubUtils } = require('../../core/utils/pubsub_utils');
 const { fetchComputationStatus } = require('../persistence/StatusRepository');
 const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
 const crypto = require('crypto');
-const monConfig = require('../config/monitoring_config'); // [NEW]
+const monConfig = require('../config/monitoring_config');
 
-const OOM_THRESHOLD_MB = 1500;
+const OOM_THRESHOLD_MB = 1500; // Unused
 const BASE_SECONDS_PER_WEIGHT_UNIT = 3;
 const SESSION_CACHE_DURATION_MS = 1000 * 60 * 30; // 30 Minutes
 const STALE_LOCK_THRESHOLD_MS = 1000 * 60 * 15; 
@@ -281,7 +281,6 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
     const currentDispatchId = crypto.randomUUID();
     
     const tasksPayload = validTasks.map(t => {
-        // [NEW] Generate Eyeball Trace
         const traceId = crypto.randomBytes(16).toString('hex');
         const spanId = crypto.randomBytes(8).toString('hex');
 
@@ -294,7 +293,6 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
             dispatchId: currentDispatchId,
             triggerReason: 'SWEEP_RECOVERY',
             resources: 'high-mem', // FORCE
-            // [NEW] Attach the eyeball
             traceContext: {
                 traceId: traceId,
                 spanId: spanId,
@@ -315,7 +313,7 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
 }
 
 // =============================================================================
-// LOGIC: Standard Dispatch (Original)
+// LOGIC: Standard Dispatch (Fast-Forward Enabled)
 // =============================================================================
 async function handleStandardDispatch(config, dependencies, computationManifest, reqBody) {
     const { logger, db } = dependencies;
@@ -336,16 +334,29 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
     const sessionDates = await getStableDateSession(config, dependencies, passToRun, dateLimitStr, forceRebuild);
     if (!sessionDates || sessionDates.length === 0) return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
 
-    let selectedDate = null;
+    // --- Fast-Forward Loop Configuration ---
+    // Scans up to 50 dates or 40 seconds to find work, avoiding empty "wait loops"
+    const MAX_SCAN_DEPTH = 50;
+    const TIME_LIMIT_MS = 40000; 
+    const startTime = Date.now();
+
+    let currentCursor = targetCursorN;
     let selectedTasks = [];
+    let selectedDate = null;
+    let datesScanned = 0;
 
-    if (targetCursorN <= sessionDates.length) {
-        selectedDate = sessionDates[targetCursorN - 1];
-    } else {
-        return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
-    }
+    // Loop until work is found, end is reached, or safety limits hit
+    while (currentCursor <= sessionDates.length) {
+        datesScanned++;
+        selectedDate = sessionDates[currentCursor - 1]; // 0-indexed array
+
+        // 1. Safety Break (Prevent Timeout)
+        if ((Date.now() - startTime) > TIME_LIMIT_MS || datesScanned > MAX_SCAN_DEPTH) {
+            logger.log('INFO', `[Dispatcher] ⏩ Fast-forward paused at ${selectedDate} after scanning ${datesScanned} dates.`);
+            break;
+        }
 
-    if (selectedDate) {
+        // 2. Analyze Date
         const earliestDates = await getEarliestDataDates(config, dependencies);
         let prevDailyStatusPromise = Promise.resolve(null);
         if (calcsInThisPass.some(c => c.isHistorical)) {
@@ -362,48 +373,70 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
             checkRootDataAvailability(selectedDate, config, dependencies, DEFINITIVE_EARLIEST_DATES)
         ]);
 
-        if (availability && availability.status) { // Just check availability exists
+        if (availability && availability.status) {
             const report = analyzeDateExecution(selectedDate, calcsInThisPass, availability.status, dailyStatus, manifestMap, prevDailyStatus);
             let rawTasks = [...report.runnable, ...report.reRuns];
 
             if (rawTasks.length > 0) {
                 rawTasks = await attemptSimHashResolution(dependencies, selectedDate, rawTasks, dailyStatus, manifestMap);
-                selectedTasks = await filterActiveTasks(db, selectedDate, passToRun, rawTasks, logger);
-            }
-            
-            if (selectedTasks.length > 0) {
-                // Split Logic: Moves OOMs to High-Mem, drops dead letters
-                const { standard, highMem } = await splitRoutes(db, selectedDate, passToRun, selectedTasks, logger);
-                selectedTasks = [...standard, ...highMem];
+                const activeTasks = await filterActiveTasks(db, selectedDate, passToRun, rawTasks, logger);
+                
+                if (activeTasks.length > 0) {
+                    const { standard, highMem } = await splitRoutes(db, selectedDate, passToRun, activeTasks, logger);
+                    selectedTasks = [...standard, ...highMem];
+                    
+                    if (selectedTasks.length > 0) {
+                        // Found work! Break loop to dispatch.
+                        break; 
+                    }
+                }
             }
         }
+
+        // No work found for this date. Fast-forward to next.
+        currentCursor++;
     }
 
+    // --- Result Handling ---
+
+    // Case 1: Satiated (Scanned to end of session with no work)
+    if (currentCursor > sessionDates.length && selectedTasks.length === 0) {
+        return {
+            status: 'CONTINUE_PASS',
+            dateProcessed: selectedDate,
+            dispatched: 0,
+            n_cursor_ignored: false,
+            remainingDates: 0,
+            nextCursor: currentCursor // Matches length + 1
+        };
+    }
+
+    // Case 2: Paused by Limit (No work found yet, but more dates remain)
     if (selectedTasks.length === 0) {
         return {
             status: 'CONTINUE_PASS',
             dateProcessed: selectedDate,
             dispatched: 0,
             n_cursor_ignored: false,
-            etaSeconds: 0,
-            remainingDates: sessionDates.length - targetCursorN
+            remainingDates: sessionDates.length - currentCursor + 1,
+            nextCursor: currentCursor // Resume from here
         };
     }
 
+    // Case 3: Work Found (Dispatching)
     const totalweight = selectedTasks.reduce((sum, t) => sum + (manifestWeightMap.get(normalizeName(t.name)) || 1.0), 0);
     const currentDispatchId = crypto.randomUUID();
     const etaSeconds = Math.max(20, Math.ceil(totalweight * BASE_SECONDS_PER_WEIGHT_UNIT));
 
-    const taskDetails = selectedTasks.map(t => `${t.name} (${t.reason})`);
-    logger.log('INFO', `[Dispatcher] ✅ Dispatching ${selectedTasks.length} tasks for ${selectedDate}.`, {
-        date: selectedDate, pass: passToRun, dispatchedCount: selectedTasks.length, etaSeconds, dispatchId: currentDispatchId, details: taskDetails
-    });
+    if (datesScanned > 1) {
+        logger.log('INFO', `[Dispatcher] ⏩ Fast-forwarded ${datesScanned - 1} empty dates. Dispatching ${selectedTasks.length} tasks for ${selectedDate}.`);
+    } else {
+        logger.log('INFO', `[Dispatcher] ✅ Dispatching ${selectedTasks.length} tasks for ${selectedDate}.`);
+    }
 
     const mapToTaskPayload = (t) => {
-        // [NEW] Generate Eyeball Trace
         const traceId = crypto.randomBytes(16).toString('hex');
         const spanId = crypto.randomBytes(8).toString('hex');
-
         return {
             ...t, 
             action: 'RUN_COMPUTATION_DATE', 
@@ -413,7 +446,6 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
             dispatchId: currentDispatchId, 
             triggerReason: t.reason, 
             resources: t.resources || 'standard',
-            // [NEW] Attach the eyeball
             traceContext: {
                 traceId: traceId,
                 spanId: spanId,
@@ -448,13 +480,15 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
         dispatched: selectedTasks.length,
         n_cursor_ignored: false,
         etaSeconds: etaSeconds,
-        remainingDates: sessionDates.length - targetCursorN
+        remainingDates: sessionDates.length - targetCursorN,
+        nextCursor: currentCursor + 1 // Start next scan AFTER this date
     };
 }
 
 // =============================================================================
 // HELPER: Route Splitting (One-Shot Enforcement)
 // =============================================================================
+// [UPDATED] Route Splitting with Version-Aware Dead Lettering
 async function splitRoutes(db, date, pass, tasks, logger) {
     const standard = [];
     const highMem = [];
@@ -465,32 +499,51 @@ async function splitRoutes(db, date, pass, tasks, logger) {
         const doc = await db.doc(ledgerPath).get();
         
         if (!doc.exists) {
-            // New task -> Standard
             standard.push(task);
             continue;
         }
 
         const data = doc.data();
         
-        // If it FAILED, we check if we can escalate it.
+        // CHECK: Is the task currently running or pending?
+        if (['PENDING', 'IN_PROGRESS'].includes(data.status)) {
+            // It's active, don't dispatch again
+            continue; 
+        }
+        
         if (data.status === 'FAILED') {
             const stage = data.error?.stage;
 
-            // 1. QUALITY / LOGIC FAIL: Dead Letter (Drop it)
-            if (['QUALITY_CIRCUIT_BREAKER', 'SEMANTIC_GATE'].includes(stage)) {
-                logger.log('WARN', `[Dispatcher] 🛑 Dropping ${name} - Deterministic Failure (${stage}).`);
-                continue; 
+            // 1. DETERMINISTIC FAILURES (Never Retry)
+            if (['QUALITY_CIRCUIT_BREAKER', 'SEMANTIC_GATE', 'SHARDING_LIMIT_EXCEEDED'].includes(stage)) {
+                 logger.log('WARN', `[Dispatcher] 🛑 Dropping ${name} - Deterministic Failure (${stage}).`);
+                 continue; 
             }
 
-            // 2. PREVIOUSLY HIGH MEM FAIL: Dead Letter (Drop it)
+            // 2. HIGH MEMORY FAILURE HANDLING (The New Logic)
             if (data.resourceTier === 'high-mem') {
-                logger.log('WARN', `[Dispatcher] 🛑 Dropping ${name} - Failed on High-Mem already.`);
-                continue;
+                const failedHash = data.hash || data.composition?.code; // Support legacy or new structure
+                const currentHash = task.hash;
+
+                // A. EXACT CODE MATCH: It failed High-Mem with THIS code.
+                if (failedHash === currentHash) {
+                    logger.log('WARN', `[Dispatcher] 💀 Dead End: ${name} failed High-Mem on this version (${currentHash.slice(0,6)}). Waiting for code fix.`);
+                    continue; // STOP. Do not retry.
+                } 
+                
+                // B. CODE MISMATCH: The code has changed since the High-Mem failure.
+                // We reset it to 'standard' to see if the fix optimized memory usage.
+                else {
+                    logger.log('INFO', `[Dispatcher] 🔄 Code Updated for ${name}. Resetting High-Mem failure to Standard retry.`);
+                    standard.push({
+                        ...task,
+                        reason: 'Retry: Code Version Changed'
+                    });
+                    continue;
+                }
             }
 
-            // 3. STANDARD FAIL (Crash/OOM): Promote to High Mem (Retry)
-            // If it failed standard, we give it ONE shot on high-mem.
-            // Note: Even if it was an "Unknown" error, we promote to High-Mem to cover OOMs that looked like crashes.
+            // 3. STANDARD FAILURE -> PROMOTE TO HIGH MEM
             highMem.push({ 
                 ...task, 
                 resources: 'high-mem', 
@@ -498,7 +551,8 @@ async function splitRoutes(db, date, pass, tasks, logger) {
             });
 
         } else {
-            // If status is not FAILED (e.g. was Zombie and filterActiveTasks passed it), retry Standard.
+            // Status is likely COMPLETED (but filterActiveTasks should have caught this)
+            // or some other state. Default to standard if we are here.
             standard.push(task);
         }
     }

package/functions/computation-system/helpers/computation_worker.js

@@ -2,6 +2,7 @@
  * FILENAME: computation-system/helpers/computation_worker.js
  * UPDATED: Implements Google Cloud Monitoring Heartbeats.
  * UPDATED: Implements Structured Logging with Trace Context.
+ * FIXED: 'manifest' scope issue (declared outside try/catch).
  */
 
 const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
@@ -143,8 +144,6 @@ async function handleComputationTask(message, config, dependencies) {
     const workerId = process.env.K_REVISION || os.hostname();
 
     // 2. Initialize Trace-Aware Logger (The "Eyeball")
-    // If traceContext is present (from dispatcher), we inject it into globalMetadata
-    // so ALL logs from this execution line up in Google Trace.
     const globalMetadata = {};
     if (traceContext && monConfig.enabled) {
         globalMetadata['logging.googleapis.com/trace'] = `projects/${monConfig.project}/traces/${traceContext.traceId}`;
@@ -155,7 +154,7 @@ async function handleComputationTask(message, config, dependencies) {
     const logger = new StructuredLogger({ 
         minLevel: config.minLevel || 'INFO', 
         enableStructured: true, 
-        globalMetadata, // [NEW] Inject Trace
+        globalMetadata, 
         ...config 
     });
     
@@ -172,11 +171,13 @@ async function handleComputationTask(message, config, dependencies) {
     logger.log('INFO', `[Worker] 📥 Task: ${computation} (${date}) [Tier: ${resourceTier}] [ID: ${dispatchId}]`);
 
     // --- STEP 2: START DUAL HEARTBEATS ---
-    // Starts both Firestore writes (legacy) AND Google Metrics (new)
     const heartbeats = startMemoryHeartbeat(db, ledgerPath, workerId, computation, traceContext?.traceId);
 
+    // [FIX] Declare manifest here so it is accessible in both try and catch blocks
+    let manifest;
+
     try {
-        const manifest = getManifest(config.activeProductLines || [], calculations, runDeps);
+        manifest = getManifest(config.activeProductLines || [], calculations, runDeps);
         const startTime = Date.now();
         
         const result = await executeDispatchTask(
@@ -207,7 +208,6 @@ async function handleComputationTask(message, config, dependencies) {
         };
 
         await db.doc(ledgerPath).update({ status: 'COMPLETED', completedAt: new Date() });
-        // Use the new recorder which also prints JSON for logs
         await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, metrics, triggerReason, resourceTier);
 
     } catch (err) { 
@@ -215,10 +215,25 @@ async function handleComputationTask(message, config, dependencies) {
         
         const isDeterministic = ['SHARDING_LIMIT_EXCEEDED', 'QUALITY_CIRCUIT_BREAKER', 'SEMANTIC_GATE'].includes(err.stage);
         
+        // [FIX] Now we can safely access 'manifest' to find the current hash
+        let currentHash = 'unknown';
+        if (typeof manifest !== 'undefined' && Array.isArray(manifest)) {
+             const item = manifest.find(c => normalizeName(c.name) === normalizeName(computation));
+             if (item) currentHash = item.hash;
+        }
+
         if (isDeterministic || (message.deliveryAttempt || 1) >= MAX_RETRIES) {
             const errorPayload = { message: err.message, stage: err.stage || 'FATAL' };
 
-            await db.doc(ledgerPath).set({ status: 'FAILED', error: errorPayload, failedAt: new Date() }, { merge: true });
+            // [UPDATED] We now save 'hash' and 'resourceTier' to the ledger on failure
+            await db.doc(ledgerPath).set({ 
+                status: 'FAILED', 
+                error: errorPayload, 
+                failedAt: new Date(),
+                hash: currentHash, // <--- CRITICAL FOR DISPATCHER DEAD-LETTER CHECK
+                resourceTier: resourceTier 
+            }, { merge: true });
+
             await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'FATAL' }, { peakMemoryMB: heartbeats.getPeak() }, triggerReason, resourceTier);
             return;
         }

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,6 +1,6 @@
 # Cloud Workflows: Precision Cursor-Based Orchestrator
-# SIMPLE MODE: Dispatch -> Wait ETA -> Next Date
-# UPGRADED: Added "Sweep & Verify" Step for High-Mem Recovery
+# FAST-FORWARD MODE: Dispatch -> (Internal Loop) -> Wait ETA -> Jump to Next Valid Cursor
+# UPGRADED: Supports 'nextCursor' to skip empty date ranges instantly.
 
 main:
   params: [input]
@@ -39,12 +39,12 @@ main:
 
                       - evaluate_dispatch:
                           switch:
-                            # 1. End of Session (Dispatcher reached end of date list)
+                            # 1. End of Session (Dispatcher reached absolute end of date list)
                             - condition: '${dispatch_res.body.status == "MOVE_TO_NEXT_PASS"}'
                               assign:
                                 - pass_complete: true
                             
-                            # 2. Satiation Check (Specific to date/logic)
+                            # 2. Satiation Check (Dispatcher scanned to end, found nothing)
                             - condition: '${dispatch_res.body.status == "CONTINUE_PASS" and dispatch_res.body.remainingDates == 0 and dispatch_res.body.dispatched == 0}'
                               steps:
                                 - log_satiation:
@@ -55,7 +55,7 @@ main:
                                     assign:
                                       - pass_complete: true
 
-                            # 3. Work Dispatched: Wait ETA -> Move Next (Ignored flag is FALSE)
+                            # 3. Work Dispatched: Wait ETA -> Jump to 'nextCursor'
                             - condition: '${dispatch_res.body.dispatched > 0}'
                               steps:
                                 - wait_for_completion:
@@ -64,25 +64,26 @@ main:
                                       seconds: '${int(dispatch_res.body.etaSeconds)}'
                                 - update_cursor:
                                     assign:
-                                      - n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
+                                      # CRITICAL CHANGE: Use dispatcher's specific jump target
+                                      - n_cursor: '${default(dispatch_res.body.nextCursor, n_cursor + 1)}'
                                 - next_loop_work:
                                     next: sequential_date_loop
 
-                            # 4. No Work (Clean or Busy): Move Next Immediately
+                            # 4. No Work (Fast-Forward limit reached or Single Check): Jump to 'nextCursor'
                             - condition: '${dispatch_res.body.dispatched == 0}'
                               steps:
                                 - wait_short:
                                     call: sys.sleep
                                     args:
-                                      seconds: 2 # Tiny debounce
+                                      seconds: 2 # Tiny debounce for API limits
                                 - update_cursor_retry:
                                     assign:
-                                        # Dispatcher sends n_cursor_ignored=false, so we increment.
-                                        - n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
+                                        # CRITICAL CHANGE: Resume exactly where the Fast-Forward loop stopped
+                                        - n_cursor: '${default(dispatch_res.body.nextCursor, n_cursor + 1)}'
                                 - next_loop_retry:
                                     next: sequential_date_loop
             
-            # --- NEW STEP: VERIFICATION & SWEEP ---
+            # --- VERIFICATION & SWEEP (No changes needed here) ---
             - verify_pass_completion:
                 call: http.post
                 args:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.342",
+  "version": "1.0.344",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [