bulltrackers-module 1.0.304 → 1.0.306

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,7 +1,6 @@
 /**
  * FILENAME: computation-system/helpers/computation_dispatcher.js
- * PURPOSE: Sequential Cursor-Based Dispatcher.
- * IMPLEMENTS: Dirty-Date Discovery, Forensics Rerouting, and Satiation Sweeps.
+ * PURPOSE: Sequential Cursor-Based Dispatcher with Hyper-Verbose Telemetry.
  */
 
 const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
@@ -11,12 +10,8 @@ const { fetchComputationStatus } = require('../persistence/StatusRepository');
 const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
 
 const OOM_THRESHOLD_MB = 1500; 
-const SECONDS_PER_CALC_MARGIN = 25; // 20s base + safety margin
+const SECONDS_PER_CALC_MARGIN = 25; 
 
-/**
- * Checks if specific tasks on a date need a high-memory reroute.
- * Returns only tasks that failed on 'standard' and haven't been tried on 'high-mem'.
- */
 async function getHighMemReroutes(db, date, pass, tasks) {
     const reroutes = [];
     for (const task of tasks) {
@@ -26,7 +21,6 @@ async function getHighMemReroutes(db, date, pass, tasks) {
         
         if (doc.exists) {
             const data = doc.data();
-            // Check if it failed due to memory and hasn't been attempted on high-mem yet
             const isOOM = (data.status === 'FAILED' || data.status === 'CRASH') && 
                           (data.resourceTier !== 'high-mem') &&
                           ((data.peakMemoryMB > OOM_THRESHOLD_MB) || (data.error && /memory/i.test(data.error.message)));
@@ -43,42 +37,71 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     const { logger, db } = dependencies;
     const pubsubUtils    = new PubSubUtils(dependencies);
     
-    // Inputs from Workflow Cursor
-    const passToRun     = String(reqBody.pass || config.COMPUTATION_PASS_TO_RUN);
+    // 1. Capture Inputs
+    const passToRun     = String(reqBody.pass || config.COMPUTATION_PASS_TO_RUN || "1");
     const targetCursorN = parseInt(reqBody.cursorIndex || 1);
-    const dateLimitStr  = reqBody.date || config.date; 
+    const dateLimitStr  = reqBody.date || config.date || "2025-01-01"; 
+
+    logger.log('INFO', `[Dispatcher] 🚀 STARTING DISPATCH: Pass ${passToRun}, Cursor ${targetCursorN}, Limit ${dateLimitStr}`);
 
     const manifestMap     = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
     const passes          = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
 
     if (!calcsInThisPass.length) {
+        logger.log('WARN', `[Dispatcher] 🛑 No calculations found for Pass ${passToRun}. Moving to next pass.`);
         return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
     }
 
-    // 1. Discover all "Dirty" Dates (Dates needing work)
+    // 2. Discover Discovery Boundaries
     const earliestDates = await getEarliestDataDates(config, dependencies);
-    const allDates      = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date(dateLimitStr + 'T00:00:00Z'));
+    logger.log('INFO', `[Dispatcher] Discovery Boundaries: Earliest=${earliestDates.absoluteEarliest.toISOString().slice(0,10)}, Limit=${dateLimitStr}`);
+
+    const allDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date(dateLimitStr + 'T00:00:00Z'));
     
+    if (allDates.length === 0) {
+        logger.log('ERROR', `[Dispatcher] ❌ Date range is empty. Check if dateLimit is before earliest data.`);
+        return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
+    }
+
+    // 3. Date Scanning Loop
     const dirtyDates = [];
+    let blockedCount = 0;
+    let upToDateCount = 0;
+
+    logger.log('INFO', `[Dispatcher] Scanning ${allDates.length} dates for work...`);
+
     for (const d of allDates) {
         const dailyStatus  = await fetchComputationStatus(d, config, dependencies);
         const availability = await checkRootDataAvailability(d, config, dependencies, DEFINITIVE_EARLIEST_DATES);
         
+        // Detailed check on availability status
+        if (!availability || !availability.status.hasPrices) {
+             // Log every 30 days to avoid log spam if data is missing for long periods
+             if (allDates.indexOf(d) % 30 === 0) logger.log('DEBUG', `[Dispatcher] ${d}: Root Data Index Missing or Price=false.`);
+             blockedCount++;
+             continue;
+        }
+
         const report = analyzeDateExecution(d, calcsInThisPass, availability.status, dailyStatus, manifestMap, null);
         const tasks  = [...report.runnable, ...report.reRuns];
         
         if (tasks.length > 0) {
+            logger.log('INFO', `[Dispatcher] ✨ Found Dirty Date: ${d} (${tasks.length} tasks)`);
             dirtyDates.push({ date: d, tasks });
+        } else {
+            upToDateCount++;
         }
     }
 
+    logger.log('INFO', `[Dispatcher] Scan Complete: ${dirtyDates.length} dirty, ${upToDateCount} up-to-date, ${blockedCount} blocked/missing data.`);
+
     let selectedDate  = null;
     let selectedTasks = [];
     let isReroute     = false;
     let isSweep       = false;
 
-    // 2. Logic: Prioritize Reroute for N-1
+    // 4. Cursor Logic
     if (targetCursorN > 1 && (targetCursorN - 2) < dirtyDates.length) {
         const prevEntry = dirtyDates[targetCursorN - 2];
         const reroutes  = await getHighMemReroutes(db, prevEntry.date, passToRun, prevEntry.tasks);
@@ -87,34 +110,35 @@ async function dispatchComputationPass(config, dependencies, computationManifest
             selectedDate  = prevEntry.date;
             selectedTasks = reroutes;
             isReroute     = true;
-            logger.log('INFO', `[Dispatcher] Reroute detected for ${selectedDate}. Pausing N increment.`);
+            logger.log('INFO', `[Dispatcher] 🔄 Reroute detected for ${selectedDate}. Retrying same cursor position with High-Mem.`);
         }
     }
 
-    // 3. Logic: N-th Dirty Date or Final Sweep
     if (!selectedDate) {
         if (targetCursorN <= dirtyDates.length) {
             const entry   = dirtyDates[targetCursorN - 1];
             selectedDate  = entry.date;
             selectedTasks = entry.tasks;
+            logger.log('INFO', `[Dispatcher] Selecting Dirty Date #${targetCursorN}: ${selectedDate}`);
         } else {
-            // Final Satiation Sweep: Check if anything was missed (recovery)
             if (dirtyDates.length > 0) {
                 isSweep       = true;
                 selectedDate  = dirtyDates[0].date;
                 selectedTasks = dirtyDates[0].tasks;
+                logger.log('INFO', `[Dispatcher] 🧹 Satiation Sweep: Checking earliest dirty date ${selectedDate}`);
             }
         }
     }
 
-    // 4. Signal Pass Completion
+    // 5. Termination Check
     if (!selectedDate) {
+        logger.log('SUCCESS', `[Dispatcher] ✅ Pass ${passToRun} is fully satiated. Signalling MOVE_TO_NEXT_PASS.`);
         return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0, etaSeconds: 0 };
     }
 
-    // 5. Dispatch to PubSub (Standard vs. High-Mem)
-    const standardTasks = selectedTasks.filter(t => t.resources !== 'high-mem').map(t => ({ ...t, date: selectedDate, pass: passToRun }));
-    const highMemTasks  = selectedTasks.filter(t => t.resources === 'high-mem').map(t => ({ ...t, date: selectedDate, pass: passToRun }));
+    // 6. Pub/Sub Dispatch
+    const standardTasks = selectedTasks.filter(t => t.resources !== 'high-mem').map(t => ({ ...t, action: 'RUN_COMPUTATION_DATE', computation: t.name,  date: selectedDate, pass: passToRun }));
+    const highMemTasks  = selectedTasks.filter(t => t.resources === 'high-mem').map(t => ({ ...t, action: 'RUN_COMPUTATION_DATE', computation: t.name,  date: selectedDate, pass: passToRun }));
 
     const pubPromises = [];
     if (standardTasks.length > 0) {
@@ -135,13 +159,13 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
     const etaSeconds = Math.max(20, selectedTasks.length * SECONDS_PER_CALC_MARGIN);
     
-    logger.log('INFO', `[Dispatcher] ${isReroute ? 'Reroute' : (isSweep ? 'Sweep' : 'Standard')} Run: ${selectedDate}. Tasks: ${selectedTasks.length}. ETA: ${etaSeconds}s`);
+    logger.log('INFO', `[Dispatcher] 🛰️ DISPATCHED ${selectedTasks.length} tasks for ${selectedDate}. ETA ${etaSeconds}s.`);
 
     return {
         status          : isSweep ? 'RECOVERY' : 'CONTINUE_PASS',
         dateProcessed   : selectedDate,
         dispatched      : selectedTasks.length,
-        n_cursor_ignored: isReroute, // Tell workflow to stay on same N
+        n_cursor_ignored: isReroute,
         etaSeconds      : etaSeconds
     };
 }

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,5 +1,5 @@
 # Cloud Workflows: Precision Cursor-Based Orchestrator
-# PURPOSE: Orchestrates 5 passes with deterministic pacing and syntax fixes.
+# PURPOSE: Orchestrates 5 passes with dynamic date detection and cursor logic.
 
 main:
   params: [input]
@@ -8,7 +8,14 @@ main:
         assign:
           - project: '${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}'
           - passes: ["1", "2", "3", "4", "5"]
-          - date_to_run: '${default(map.get(input, "date"), "2025-01-01")}'
+          # Dynamically calculate today's date (YYYY-MM-DD) if no date is provided in input
+          - current_date: '${text.split(time.format(sys.now()), "T")[0]}'
+          - date_to_run: '${default(map.get(input, "date"), current_date)}'
+
+    - log_start:
+        call: sys.log
+        args:
+          text: '${"Starting Pipeline execution. Target Date Limit: " + date_to_run}'
 
     - run_sequential_passes:
         for:
@@ -37,12 +44,12 @@ main:
 
                       - evaluate_dispatch:
                           switch:
-                            # State 1: Pass exhausted and Satiation Sweep complete
+                            # State 1: Dispatcher signal to move to the next pass
                             - condition: '${dispatch_res.body.status == "MOVE_TO_NEXT_PASS"}'
                               assign:
                                 - pass_complete: true
                             
-                            # State 2: Tasks dispatched (Standard, Reroute, or Recovery Sweep)
+                            # State 2: Tasks were dispatched
                             - condition: '${dispatch_res.body.dispatched > 0}'
                               steps:
                                 - log_dispatch:
@@ -55,7 +62,7 @@ main:
                                       seconds: '${int(dispatch_res.body.etaSeconds)}'
                                 - update_cursor:
                                     assign:
-                                      # If n_cursor_ignored is true (Reroute or Sweep Recovery), we stay on same N
+                                      # If n_cursor_ignored is true, stay on same N to retry (e.g. for high-mem)
                                       - n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
                                 - next_loop:
                                     next: sequential_date_loop

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.304",
+  "version": "1.0.306",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [