bulltrackers-module 1.0.309 → 1.0.311

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,6 +1,7 @@
 /**
  * FILENAME: computation-system/helpers/computation_dispatcher.js
  * PURPOSE: Sequential Cursor-Based Dispatcher with Hyper-Verbose Telemetry.
+ * FIX: Added dispatchId generation and triggerReason mapping.
  */
 
 const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
@@ -8,6 +9,7 @@ const { groupByPass, analyzeDateExecution } = require('../WorkflowOrchestrator.j
 const { PubSubUtils } = require('../../core/utils/pubsub_utils');
 const { fetchComputationStatus } = require('../persistence/StatusRepository');
 const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
+const crypto = require('crypto'); // [NEW] Required for dispatchId
 
 const OOM_THRESHOLD_MB = 1500; 
 const SECONDS_PER_CALC_MARGIN = 25; 
@@ -37,7 +39,6 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     const { logger, db } = dependencies;
     const pubsubUtils    = new PubSubUtils(dependencies);
     
-    // 1. Capture Inputs
     const passToRun     = String(reqBody.pass || config.COMPUTATION_PASS_TO_RUN || "1");
     const targetCursorN = parseInt(reqBody.cursorIndex || 1);
     const dateLimitStr  = reqBody.date || config.date || "2025-01-01"; 
@@ -53,55 +54,34 @@ async function dispatchComputationPass(config, dependencies, computationManifest
         return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
     }
 
-    // 2. Discover Discovery Boundaries
     const earliestDates = await getEarliestDataDates(config, dependencies);
-    logger.log('INFO', `[Dispatcher] Discovery Boundaries: Earliest=${earliestDates.absoluteEarliest.toISOString().slice(0,10)}, Limit=${dateLimitStr}`);
-
     const allDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date(dateLimitStr + 'T00:00:00Z'));
     
     if (allDates.length === 0) {
-        logger.log('ERROR', `[Dispatcher] ❌ Date range is empty. Check if dateLimit is before earliest data.`);
+        logger.log('ERROR', `[Dispatcher] ❌ Date range is empty.`);
         return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
     }
 
-    // 3. Date Scanning Loop
     const dirtyDates = [];
-    let blockedCount = 0;
-    let upToDateCount = 0;
-
-    logger.log('INFO', `[Dispatcher] Scanning ${allDates.length} dates for work...`);
-
     for (const d of allDates) {
         const dailyStatus  = await fetchComputationStatus(d, config, dependencies);
         const availability = await checkRootDataAvailability(d, config, dependencies, DEFINITIVE_EARLIEST_DATES);
         
-        // Detailed check on availability status
-        if (!availability || !availability.status.hasPrices) {
-             // Log every 30 days to avoid log spam if data is missing for long periods
-             if (allDates.indexOf(d) % 30 === 0) logger.log('DEBUG', `[Dispatcher] ${d}: Root Data Index Missing or Price=false.`);
-             blockedCount++;
-             continue;
-        }
+        if (!availability || !availability.status.hasPrices) continue;
 
         const report = analyzeDateExecution(d, calcsInThisPass, availability.status, dailyStatus, manifestMap, null);
         const tasks  = [...report.runnable, ...report.reRuns];
         
         if (tasks.length > 0) {
-            logger.log('INFO', `[Dispatcher] ✨ Found Dirty Date: ${d} (${tasks.length} tasks)`);
             dirtyDates.push({ date: d, tasks });
-        } else {
-            upToDateCount++;
         }
     }
 
-    logger.log('INFO', `[Dispatcher] Scan Complete: ${dirtyDates.length} dirty, ${upToDateCount} up-to-date, ${blockedCount} blocked/missing data.`);
-
     let selectedDate  = null;
     let selectedTasks = [];
     let isReroute     = false;
     let isSweep       = false;
 
-    // 4. Cursor Logic
     if (targetCursorN > 1 && (targetCursorN - 2) < dirtyDates.length) {
         const prevEntry = dirtyDates[targetCursorN - 2];
         const reroutes  = await getHighMemReroutes(db, prevEntry.date, passToRun, prevEntry.tasks);
@@ -110,7 +90,6 @@ async function dispatchComputationPass(config, dependencies, computationManifest
             selectedDate  = prevEntry.date;
             selectedTasks = reroutes;
             isReroute     = true;
-            logger.log('INFO', `[Dispatcher] 🔄 Reroute detected for ${selectedDate}. Retrying same cursor position with High-Mem.`);
         }
     }
 
@@ -119,26 +98,32 @@ async function dispatchComputationPass(config, dependencies, computationManifest
             const entry   = dirtyDates[targetCursorN - 1];
             selectedDate  = entry.date;
             selectedTasks = entry.tasks;
-            logger.log('INFO', `[Dispatcher] Selecting Dirty Date #${targetCursorN}: ${selectedDate}`);
-        } else {
-            if (dirtyDates.length > 0) {
-                isSweep       = true;
-                selectedDate  = dirtyDates[0].date;
-                selectedTasks = dirtyDates[0].tasks;
-                logger.log('INFO', `[Dispatcher] 🧹 Satiation Sweep: Checking earliest dirty date ${selectedDate}`);
-            }
+        } else if (dirtyDates.length > 0) {
+            isSweep       = true;
+            selectedDate  = dirtyDates[0].date;
+            selectedTasks = dirtyDates[0].tasks;
         }
     }
 
-    // 5. Termination Check
     if (!selectedDate) {
-        logger.log('SUCCESS', `[Dispatcher] ✅ Pass ${passToRun} is fully satiated. Signalling MOVE_TO_NEXT_PASS.`);
         return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0, etaSeconds: 0 };
     }
 
-    // 6. Pub/Sub Dispatch
-    const standardTasks = selectedTasks.filter(t => t.resources !== 'high-mem').map(t => ({ ...t, action: 'RUN_COMPUTATION_DATE', computation: t.name,  date: selectedDate, pass: passToRun }));
-    const highMemTasks  = selectedTasks.filter(t => t.resources === 'high-mem').map(t => ({ ...t, action: 'RUN_COMPUTATION_DATE', computation: t.name,  date: selectedDate, pass: passToRun }));
+    // [FIX] Generate unique Dispatch ID for this wave and map triggerReason
+    const currentDispatchId = crypto.randomUUID();
+
+    const mapToTaskPayload = (t) => ({
+        ...t,
+        action: 'RUN_COMPUTATION_DATE',
+        computation: t.name,
+        date: selectedDate,
+        pass: passToRun,
+        dispatchId: currentDispatchId, // [NEW] Added for Worker Audit Lease
+        triggerReason: t.reason        // [NEW] Map 'reason' from Orchestrator to 'triggerReason'
+    });
+
+    const standardTasks = selectedTasks.filter(t => t.resources !== 'high-mem').map(mapToTaskPayload);
+    const highMemTasks  = selectedTasks.filter(t => t.resources === 'high-mem').map(mapToTaskPayload);
 
     const pubPromises = [];
     if (standardTasks.length > 0) {
@@ -159,8 +144,6 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
     const etaSeconds = Math.max(20, selectedTasks.length * SECONDS_PER_CALC_MARGIN);
     
-    logger.log('INFO', `[Dispatcher] 🛰️ DISPATCHED ${selectedTasks.length} tasks for ${selectedDate}. ETA ${etaSeconds}s.`);
-
     return {
         status          : isSweep ? 'RECOVERY' : 'CONTINUE_PASS',
         dateProcessed   : selectedDate,

package/functions/computation-system/helpers/computation_worker.js

@@ -1,7 +1,6 @@
 /**
  * FILENAME: computation-system/helpers/computation_worker.js
- * UPDATED: Removed redundant Callback and Sentinel logic.
- * The system now relies on Dispatcher cursor satiation.
+ * UPDATED: Fixed Firestore 'undefined' field error for dispatchId.
  */
 
 const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
@@ -17,7 +16,6 @@ const calculations = calculationPackage.calculations;
 
 const MAX_RETRIES = 3;
 
-/** Black Box Recorder for Peak Memory. */
 function startMemoryHeartbeat(db, ledgerPath, intervalMs = 2000) {
     let peakRss = 0;
     const timer = setInterval(async () => {
@@ -48,13 +46,15 @@ async function handleComputationTask(message, config, dependencies) {
 
     logger.log('INFO', `[Worker] 📥 Task: ${computation} (${date}) [Tier: ${resourceTier}]`);
 
-    // 1. Audit Lease (Lease on life for the monitor to see)
-    await db.doc(ledgerPath).set({
+    // [FIX] Build document object and only add dispatchId if it is defined
+    const leaseData = {
         status: 'IN_PROGRESS',
         workerId: process.env.K_REVISION || os.hostname(),
-        startedAt: new Date(),
-        dispatchId
-    }, { merge: true });
+        startedAt: new Date()
+    };
+    if (dispatchId) leaseData.dispatchId = dispatchId;
+
+    await db.doc(ledgerPath).set(leaseData, { merge: true });
 
     const heartbeat = startMemoryHeartbeat(db, ledgerPath);
 
@@ -84,7 +84,6 @@ async function handleComputationTask(message, config, dependencies) {
             composition: calcUpdate.composition
         };
 
-        // Mark ledger as completed
         await db.doc(ledgerPath).update({ status: 'COMPLETED', completedAt: new Date() });
         await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, metrics, triggerReason, resourceTier);
 
@@ -95,9 +94,9 @@ async function handleComputationTask(message, config, dependencies) {
         if (isDeterministic || (message.deliveryAttempt || 1) >= MAX_RETRIES) {
             await db.doc(ledgerPath).set({ status: 'FAILED', error: err.message, failedAt: new Date() }, { merge: true });
             await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'FATAL' }, { peakMemoryMB: heartbeat.getPeak() }, triggerReason, resourceTier);
-            return; // Exit without throwing to prevent endless Pub/Sub retries for dead logic
+            return;
         }
-        throw err; // Trigger Pub/Sub retry for non-deterministic transient errors
+        throw err;
     }
 }
 

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,5 +1,6 @@
 # Cloud Workflows: Precision Cursor-Based Orchestrator
 # PURPOSE: Orchestrates 5 passes with dynamic date detection and cursor logic.
+# UPDATED: Added Short-Circuit logic to break infinite loops on empty dispatches.
 
 main:
   params: [input]
@@ -26,6 +27,7 @@ main:
                 assign:
                   - n_cursor: 1
                   - pass_complete: false
+                  - consecutive_empty_dispatches: 0 # Track consecutive "duds" to prevent infinite loops
 
             - sequential_date_loop:
                 switch:
@@ -49,9 +51,12 @@ main:
                               assign:
                                 - pass_complete: true
                             
-                            # State 2: Tasks were dispatched
+                            # State 2: Tasks were dispatched (Healthy State)
                             - condition: '${dispatch_res.body.dispatched > 0}'
                               steps:
+                                - reset_retry_counter:
+                                    assign:
+                                      - consecutive_empty_dispatches: 0 # Reset counter because progress was made
                                 - log_dispatch:
                                     call: sys.log
                                     args:
@@ -64,8 +69,45 @@ main:
                                     assign:
                                       # If n_cursor_ignored is true, stay on same N to retry (e.g. for high-mem)
                                       - n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
-                                - next_loop:
+                                - next_loop_work:
                                     next: sequential_date_loop
 
+                            # State 3: No tasks dispatched (Potential Infinite Loop Scenario)
+                            # The Dispatcher is "Continuing" but found nothing runnable on the target date.
+                            - condition: '${dispatch_res.body.dispatched == 0}'
+                              steps:
+                                - increment_retry:
+                                    assign:
+                                      - consecutive_empty_dispatches: '${consecutive_empty_dispatches + 1}'
+                                - check_break_condition:
+                                    switch:
+                                      # If we have tried 3 times in a row with 0 results, assume the date is "stuck"
+                                      - condition: '${consecutive_empty_dispatches >= 3}'
+                                        steps:
+                                          - log_break:
+                                              call: sys.log
+                                              args:
+                                                text: '${"Pass " + pass_id + " - 🛑 FORCE BREAK: 3 consecutive empty dispatches. Moving to next pass to prevent infinite loop."}'
+                                          - force_complete:
+                                              assign:
+                                                - pass_complete: true
+                                      # Otherwise, wait briefly and retry (or move cursor depending on dispatcher logic)
+                                      - condition: '${true}'
+                                        steps:
+                                          - log_retry:
+                                              call: sys.log
+                                              args:
+                                                text: '${"Pass " + pass_id + " - Empty dispatch (" + string(consecutive_empty_dispatches) + "/3). Retrying..."}'
+                                          - wait_short:
+                                              call: sys.sleep
+                                              args:
+                                                seconds: 5
+                                          - update_cursor_retry:
+                                              assign:
+                                                 # Still advance cursor if it wasn't a strict reroute, to try next date
+                                                 - n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
+                                          - next_loop_retry:
+                                              next: sequential_date_loop
+
     - finish:
         return: "Pipeline Execution Satiated and Complete"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.309",
+  "version": "1.0.311",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [