bulltrackers-module 1.0.303 → 1.0.305

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,7 +1,6 @@
 /**
  * FILENAME: computation-system/helpers/computation_dispatcher.js
- * PURPOSE: Sequential Cursor-Based Dispatcher.
- * IMPLEMENTS: Dirty-Date Discovery, Forensics Rerouting, and Satiation Sweeps.
+ * PURPOSE: Sequential Cursor-Based Dispatcher with Hyper-Verbose Telemetry.
  */
 
 const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
@@ -11,22 +10,17 @@ const { fetchComputationStatus } = require('../persistence/StatusRepository');
 const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
 
 const OOM_THRESHOLD_MB = 1500; 
-const SECONDS_PER_CALC_MARGIN = 25; // 20s base + safety margin
+const SECONDS_PER_CALC_MARGIN = 25; 
 
-/**
- * Checks if specific tasks on a date need a high-memory reroute.
- * Returns only tasks that failed on 'standard' and haven't been tried on 'high-mem'.
- */
 async function getHighMemReroutes(db, date, pass, tasks) {
     const reroutes = [];
     for (const task of tasks) {
-        const name = normalizeName(task.name);
+        const name       = normalizeName(task.name);
         const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${name}`;
-        const doc = await db.doc(ledgerPath).get();
+        const doc        = await db.doc(ledgerPath).get();
         
         if (doc.exists) {
             const data = doc.data();
-            // Check if it failed due to memory and hasn't been attempted on high-mem yet
             const isOOM = (data.status === 'FAILED' || data.status === 'CRASH') && 
                           (data.resourceTier !== 'high-mem') &&
                           ((data.peakMemoryMB > OOM_THRESHOLD_MB) || (data.error && /memory/i.test(data.error.message)));
@@ -41,108 +35,138 @@ async function getHighMemReroutes(db, date, pass, tasks) {
 
 async function dispatchComputationPass(config, dependencies, computationManifest, reqBody = {}) {
     const { logger, db } = dependencies;
-    const pubsubUtils = new PubSubUtils(dependencies);
+    const pubsubUtils    = new PubSubUtils(dependencies);
     
-    // Inputs from Workflow Cursor
-    const passToRun = String(reqBody.pass || config.COMPUTATION_PASS_TO_RUN);
+    // 1. Capture Inputs
+    const passToRun     = String(reqBody.pass || config.COMPUTATION_PASS_TO_RUN || "1");
     const targetCursorN = parseInt(reqBody.cursorIndex || 1);
-    const dateLimitStr = reqBody.date || config.date; 
+    const dateLimitStr  = reqBody.date || config.date || "2025-01-01"; 
 
-    const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
-    const passes = groupByPass(computationManifest);
+    logger.log('INFO', `[Dispatcher] 🚀 STARTING DISPATCH: Pass ${passToRun}, Cursor ${targetCursorN}, Limit ${dateLimitStr}`);
+
+    const manifestMap     = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
+    const passes          = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
 
     if (!calcsInThisPass.length) {
+        logger.log('WARN', `[Dispatcher] 🛑 No calculations found for Pass ${passToRun}. Moving to next pass.`);
         return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
     }
 
-    // 1. Discover all "Dirty" Dates (Dates needing work)
+    // 2. Discover Discovery Boundaries
     const earliestDates = await getEarliestDataDates(config, dependencies);
+    logger.log('INFO', `[Dispatcher] Discovery Boundaries: Earliest=${earliestDates.absoluteEarliest.toISOString().slice(0,10)}, Limit=${dateLimitStr}`);
+
     const allDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date(dateLimitStr + 'T00:00:00Z'));
     
+    if (allDates.length === 0) {
+        logger.log('ERROR', `[Dispatcher] ❌ Date range is empty. Check if dateLimit is before earliest data.`);
+        return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
+    }
+
+    // 3. Date Scanning Loop
     const dirtyDates = [];
+    let blockedCount = 0;
+    let upToDateCount = 0;
+
+    logger.log('INFO', `[Dispatcher] Scanning ${allDates.length} dates for work...`);
+
     for (const d of allDates) {
-        const dailyStatus = await fetchComputationStatus(d, config, dependencies);
+        const dailyStatus  = await fetchComputationStatus(d, config, dependencies);
         const availability = await checkRootDataAvailability(d, config, dependencies, DEFINITIVE_EARLIEST_DATES);
         
+        // Detailed check on availability status
+        if (!availability || !availability.status.hasPrices) {
+             // Log every 30 days to avoid log spam if data is missing for long periods
+             if (allDates.indexOf(d) % 30 === 0) logger.log('DEBUG', `[Dispatcher] ${d}: Root Data Index Missing or Price=false.`);
+             blockedCount++;
+             continue;
+        }
+
         const report = analyzeDateExecution(d, calcsInThisPass, availability.status, dailyStatus, manifestMap, null);
-        const tasks = [...report.runnable, ...report.reRuns];
+        const tasks  = [...report.runnable, ...report.reRuns];
         
         if (tasks.length > 0) {
+            logger.log('INFO', `[Dispatcher] ✨ Found Dirty Date: ${d} (${tasks.length} tasks)`);
             dirtyDates.push({ date: d, tasks });
+        } else {
+            upToDateCount++;
         }
     }
 
-    let selectedDate = null;
+    logger.log('INFO', `[Dispatcher] Scan Complete: ${dirtyDates.length} dirty, ${upToDateCount} up-to-date, ${blockedCount} blocked/missing data.`);
+
+    let selectedDate  = null;
     let selectedTasks = [];
-    let isReroute = false;
-    let isSweep = false;
+    let isReroute     = false;
+    let isSweep       = false;
 
-    // 2. Logic: Prioritize Reroute for N-1
+    // 4. Cursor Logic
     if (targetCursorN > 1 && (targetCursorN - 2) < dirtyDates.length) {
         const prevEntry = dirtyDates[targetCursorN - 2];
-        const reroutes = await getHighMemReroutes(db, prevEntry.date, passToRun, prevEntry.tasks);
+        const reroutes  = await getHighMemReroutes(db, prevEntry.date, passToRun, prevEntry.tasks);
         
         if (reroutes.length > 0) {
-            selectedDate = prevEntry.date;
+            selectedDate  = prevEntry.date;
             selectedTasks = reroutes;
-            isReroute = true;
-            logger.log('INFO', `[Dispatcher] Reroute detected for ${selectedDate}. Pausing N increment.`);
+            isReroute     = true;
+            logger.log('INFO', `[Dispatcher] 🔄 Reroute detected for ${selectedDate}. Retrying same cursor position with High-Mem.`);
         }
     }
 
-    // 3. Logic: N-th Dirty Date or Final Sweep
     if (!selectedDate) {
         if (targetCursorN <= dirtyDates.length) {
-            const entry = dirtyDates[targetCursorN - 1];
-            selectedDate = entry.date;
+            const entry   = dirtyDates[targetCursorN - 1];
+            selectedDate  = entry.date;
             selectedTasks = entry.tasks;
+            logger.log('INFO', `[Dispatcher] Selecting Dirty Date #${targetCursorN}: ${selectedDate}`);
         } else {
-            // Final Satiation Sweep: Check if anything was missed (recovery)
             if (dirtyDates.length > 0) {
-                isSweep = true;
-                selectedDate = dirtyDates[0].date;
+                isSweep       = true;
+                selectedDate  = dirtyDates[0].date;
                 selectedTasks = dirtyDates[0].tasks;
+                logger.log('INFO', `[Dispatcher] 🧹 Satiation Sweep: Checking earliest dirty date ${selectedDate}`);
             }
         }
     }
 
-    // 4. Signal Pass Completion
+    // 5. Termination Check
     if (!selectedDate) {
+        logger.log('SUCCESS', `[Dispatcher] ✅ Pass ${passToRun} is fully satiated. Signalling MOVE_TO_NEXT_PASS.`);
         return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0, etaSeconds: 0 };
     }
 
-    // 5. Dispatch to PubSub (Standard vs. High-Mem)
+    // 6. Pub/Sub Dispatch
     const standardTasks = selectedTasks.filter(t => t.resources !== 'high-mem').map(t => ({ ...t, date: selectedDate, pass: passToRun }));
-    const highMemTasks = selectedTasks.filter(t => t.resources === 'high-mem').map(t => ({ ...t, date: selectedDate, pass: passToRun }));
+    const highMemTasks  = selectedTasks.filter(t => t.resources === 'high-mem').map(t => ({ ...t, date: selectedDate, pass: passToRun }));
 
     const pubPromises = [];
     if (standardTasks.length > 0) {
         pubPromises.push(pubsubUtils.batchPublishTasks(dependencies, {
             topicName: config.computationTopicStandard || 'computation-tasks',
-            tasks: standardTasks,
-            taskType: `pass-${passToRun}-std`
+            tasks    : standardTasks,
+            taskType : `pass-${passToRun}-std`
         }));
     }
     if (highMemTasks.length > 0) {
         pubPromises.push(pubsubUtils.batchPublishTasks(dependencies, {
             topicName: config.computationTopicHighMem || 'computation-tasks-highmem',
-            tasks: highMemTasks,
-            taskType: `pass-${passToRun}-high`
+            tasks    : highMemTasks,
+            taskType : `pass-${passToRun}-high`
         }));
     }
     await Promise.all(pubPromises);
 
     const etaSeconds = Math.max(20, selectedTasks.length * SECONDS_PER_CALC_MARGIN);
     
-    logger.log('INFO', `[Dispatcher] ${isReroute ? 'Reroute' : (isSweep ? 'Sweep' : 'Standard')} Run: ${selectedDate}. Tasks: ${selectedTasks.length}. ETA: ${etaSeconds}s`);
+    logger.log('INFO', `[Dispatcher] 🛰️ DISPATCHED ${selectedTasks.length} tasks for ${selectedDate}. ETA ${etaSeconds}s.`);
 
     return {
-        status: isSweep ? 'RECOVERY' : 'CONTINUE_PASS',
-        dateProcessed: selectedDate,
-        dispatched: selectedTasks.length,
-        n_cursor_ignored: isReroute, // Tell workflow to stay on same N
-        etaSeconds: etaSeconds
+        status          : isSweep ? 'RECOVERY' : 'CONTINUE_PASS',
+        dateProcessed   : selectedDate,
+        dispatched      : selectedTasks.length,
+        n_cursor_ignored: isReroute,
+        etaSeconds      : etaSeconds
     };
 }
 

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,12 +1,14 @@
 # Cloud Workflows: Precision Cursor-Based Orchestrator
+# PURPOSE: Orchestrates 5 passes with deterministic pacing and syntax fixes.
+
 main:
   params: [input]
   steps:
     - init:
         assign:
-          - project: ${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}
+          - project: '${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}'
           - passes: ["1", "2", "3", "4", "5"]
-          - date_to_run: ${default(map.get(input, "date"), "2025-01-01")}
+          - date_to_run: '${default(map.get(input, "date"), "2025-01-01")}'
 
     - run_sequential_passes:
         for:
@@ -25,29 +27,38 @@ main:
                       - call_dispatcher:
                           call: http.post
                           args:
-                            url: ${"https://europe-west1-" + project + ".cloudfunctions.net/dispatch-pass-" + pass_id}
+                            url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-pass-" + pass_id}'
                             body:
-                              pass: ${pass_id}
-                              cursorIndex: ${n_cursor}
-                              date: ${date_to_run}
+                              pass: '${pass_id}'
+                              cursorIndex: '${n_cursor}'
+                              date: '${date_to_run}'
                             auth: { type: OIDC }
                           result: dispatch_res
 
                       - evaluate_dispatch:
                           switch:
-                            # State 1: Dispatcher signals pass is fully completed
-                            - condition: ${dispatch_res.body.status == "MOVE_TO_NEXT_PASS"}
-                              assign: [pass_complete: true]
+                            # State 1: Pass exhausted and Satiation Sweep complete
+                            - condition: '${dispatch_res.body.status == "MOVE_TO_NEXT_PASS"}'
+                              assign:
+                                - pass_complete: true
                             
-                            # State 2: Tasks dispatched (Standard, Reroute, or Recovery)
-                            - condition: ${dispatch_res.body.dispatched > 0}
+                            # State 2: Tasks dispatched (Standard, Reroute, or Recovery Sweep)
+                            - condition: '${dispatch_res.body.dispatched > 0}'
                               steps:
-                                - wait_for_eta:
+                                - log_dispatch:
+                                    call: sys.log
+                                    args:
+                                      text: '${"Pass " + pass_id + " - Dispatching " + string(dispatch_res.body.dispatched) + " tasks for " + dispatch_res.body.dateProcessed + ". ETA: " + string(dispatch_res.body.etaSeconds) + "s."}'
+                                - wait_for_completion:
                                     call: sys.sleep
-                                    args: { seconds: ${dispatch_res.body.etaSeconds} }
+                                    args:
+                                      seconds: '${int(dispatch_res.body.etaSeconds)}'
                                 - update_cursor:
                                     assign:
-                                      # If it was a reroute, stay on the same N to retry the sequence
-                                      - n_cursor: ${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}
-                                - next_step:
-                                    next: sequential_date_loop
+                                      # If n_cursor_ignored is true (Reroute or Sweep Recovery), we stay on same N
+                                      - n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
+                                - next_loop:
+                                    next: sequential_date_loop
+
+    - finish:
+        return: "Pipeline Execution Satiated and Complete"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.303",
+  "version": "1.0.305",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [