bulltrackers-module 1.0.310 → 1.0.312

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,6 +1,7 @@
 /**
  * FILENAME: computation-system/helpers/computation_dispatcher.js
  * PURPOSE: Sequential Cursor-Based Dispatcher with Hyper-Verbose Telemetry.
+ * FIX: Added dispatchId, triggerReason, and detailed logging requirements.
  */
 
 const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
@@ -8,6 +9,7 @@ const { groupByPass, analyzeDateExecution } = require('../WorkflowOrchestrator.j
 const { PubSubUtils } = require('../../core/utils/pubsub_utils');
 const { fetchComputationStatus } = require('../persistence/StatusRepository');
 const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
+const crypto = require('crypto'); 
 
 const OOM_THRESHOLD_MB = 1500; 
 const SECONDS_PER_CALC_MARGIN = 25; 
@@ -37,108 +39,107 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     const { logger, db } = dependencies;
     const pubsubUtils    = new PubSubUtils(dependencies);
     
-    // 1. Capture Inputs
     const passToRun     = String(reqBody.pass || config.COMPUTATION_PASS_TO_RUN || "1");
     const targetCursorN = parseInt(reqBody.cursorIndex || 1);
     const dateLimitStr  = reqBody.date || config.date || "2025-01-01"; 
 
-    logger.log('INFO', `[Dispatcher] 🚀 STARTING DISPATCH: Pass ${passToRun}, Cursor ${targetCursorN}, Limit ${dateLimitStr}`);
-
     const manifestMap     = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
     const passes          = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
 
     if (!calcsInThisPass.length) {
-        logger.log('WARN', `[Dispatcher] 🛑 No calculations found for Pass ${passToRun}. Moving to next pass.`);
+        logger.log('WARN', `[Dispatcher] 🛑 No calculations found for Pass ${passToRun}.`);
         return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
     }
 
-    // 2. Discover Discovery Boundaries
     const earliestDates = await getEarliestDataDates(config, dependencies);
-    logger.log('INFO', `[Dispatcher] Discovery Boundaries: Earliest=${earliestDates.absoluteEarliest.toISOString().slice(0,10)}, Limit=${dateLimitStr}`);
-
     const allDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date(dateLimitStr + 'T00:00:00Z'));
     
     if (allDates.length === 0) {
-        logger.log('ERROR', `[Dispatcher] ❌ Date range is empty. Check if dateLimit is before earliest data.`);
+        logger.log('ERROR', `[Dispatcher] ❌ Date range is empty.`);
         return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
     }
 
-    // 3. Date Scanning Loop
+    // 1. Identify all "Dirty" dates (dates that actually have work to do)
     const dirtyDates = [];
-    let blockedCount = 0;
-    let upToDateCount = 0;
-
-    logger.log('INFO', `[Dispatcher] Scanning ${allDates.length} dates for work...`);
-
     for (const d of allDates) {
         const dailyStatus  = await fetchComputationStatus(d, config, dependencies);
         const availability = await checkRootDataAvailability(d, config, dependencies, DEFINITIVE_EARLIEST_DATES);
         
-        // Detailed check on availability status
-        if (!availability || !availability.status.hasPrices) {
-             // Log every 30 days to avoid log spam if data is missing for long periods
-             if (allDates.indexOf(d) % 30 === 0) logger.log('DEBUG', `[Dispatcher] ${d}: Root Data Index Missing or Price=false.`);
-             blockedCount++;
-             continue;
-        }
+        if (!availability || !availability.status.hasPrices) continue;
 
         const report = analyzeDateExecution(d, calcsInThisPass, availability.status, dailyStatus, manifestMap, null);
         const tasks  = [...report.runnable, ...report.reRuns];
         
         if (tasks.length > 0) {
-            logger.log('INFO', `[Dispatcher] ✨ Found Dirty Date: ${d} (${tasks.length} tasks)`);
             dirtyDates.push({ date: d, tasks });
-        } else {
-            upToDateCount++;
         }
     }
 
-    logger.log('INFO', `[Dispatcher] Scan Complete: ${dirtyDates.length} dirty, ${upToDateCount} up-to-date, ${blockedCount} blocked/missing data.`);
-
     let selectedDate  = null;
     let selectedTasks = [];
     let isReroute     = false;
     let isSweep       = false;
 
-    // 4. Cursor Logic
+    // Logic for Reroutes (OOM handling)
     if (targetCursorN > 1 && (targetCursorN - 2) < dirtyDates.length) {
         const prevEntry = dirtyDates[targetCursorN - 2];
         const reroutes  = await getHighMemReroutes(db, prevEntry.date, passToRun, prevEntry.tasks);
-        
         if (reroutes.length > 0) {
             selectedDate  = prevEntry.date;
             selectedTasks = reroutes;
             isReroute     = true;
-            logger.log('INFO', `[Dispatcher] 🔄 Reroute detected for ${selectedDate}. Retrying same cursor position with High-Mem.`);
         }
     }
 
+    // Logic for standard cursor progression
     if (!selectedDate) {
         if (targetCursorN <= dirtyDates.length) {
             const entry   = dirtyDates[targetCursorN - 1];
             selectedDate  = entry.date;
             selectedTasks = entry.tasks;
-            logger.log('INFO', `[Dispatcher] Selecting Dirty Date #${targetCursorN}: ${selectedDate}`);
-        } else {
-            if (dirtyDates.length > 0) {
-                isSweep       = true;
-                selectedDate  = dirtyDates[0].date;
-                selectedTasks = dirtyDates[0].tasks;
-                logger.log('INFO', `[Dispatcher] 🧹 Satiation Sweep: Checking earliest dirty date ${selectedDate}`);
-            }
+        } else if (dirtyDates.length > 0) {
+            isSweep       = true;
+            selectedDate  = dirtyDates[0].date;
+            selectedTasks = dirtyDates[0].tasks;
         }
     }
 
-    // 5. Termination Check
     if (!selectedDate) {
-        logger.log('SUCCESS', `[Dispatcher] ✅ Pass ${passToRun} is fully satiated. Signalling MOVE_TO_NEXT_PASS.`);
+        logger.log('INFO', `[Dispatcher] 🏁 Pass ${passToRun} is fully satiated. No work remaining.`);
         return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0, etaSeconds: 0 };
     }
 
-    // 6. Pub/Sub Dispatch
-    const standardTasks = selectedTasks.filter(t => t.resources !== 'high-mem').map(t => ({ ...t, action: 'RUN_COMPUTATION_DATE', computation: t.name,  date: selectedDate, pass: passToRun }));
-    const highMemTasks  = selectedTasks.filter(t => t.resources === 'high-mem').map(t => ({ ...t, action: 'RUN_COMPUTATION_DATE', computation: t.name,  date: selectedDate, pass: passToRun }));
+    // 2. Prepare Payload and Telemetry
+    const currentDispatchId = crypto.randomUUID();
+    const etaSeconds = Math.max(20, selectedTasks.length * SECONDS_PER_CALC_MARGIN);
+    const remainingDatesCount = Math.max(0, dirtyDates.length - targetCursorN);
+
+    // requirement: condense computations into a log payload
+    const computationNames = selectedTasks.map(t => t.name);
+
+    logger.log('INFO', `[Dispatcher] ✅ Dispatching ${selectedTasks.length} tasks for ${selectedDate}. ETA: ${etaSeconds}s.`, {
+        date: selectedDate,
+        pass: passToRun,
+        dispatchedCount: selectedTasks.length,
+        remainingCursorDates: remainingDatesCount,
+        etaSeconds: etaSeconds,
+        dispatchId: currentDispatchId,
+        tasks: computationNames // Condensed into JSON payload
+    });
+
+    const mapToTaskPayload = (t) => ({
+        ...t,
+        action: 'RUN_COMPUTATION_DATE',
+        computation: t.name,
+        date: selectedDate,
+        pass: passToRun,
+        dispatchId: currentDispatchId,
+        triggerReason: t.reason 
+    });
+
+    const standardTasks = selectedTasks.filter(t => t.resources !== 'high-mem').map(mapToTaskPayload);
+    const highMemTasks  = selectedTasks.filter(t => t.resources === 'high-mem').map(mapToTaskPayload);
 
     const pubPromises = [];
     if (standardTasks.length > 0) {
@@ -157,16 +158,13 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     }
     await Promise.all(pubPromises);
 
-    const etaSeconds = Math.max(20, selectedTasks.length * SECONDS_PER_CALC_MARGIN);
-    
-    logger.log('INFO', `[Dispatcher] 🛰️ DISPATCHED ${selectedTasks.length} tasks for ${selectedDate}. ETA ${etaSeconds}s.`);
-
     return {
         status          : isSweep ? 'RECOVERY' : 'CONTINUE_PASS',
         dateProcessed   : selectedDate,
         dispatched      : selectedTasks.length,
         n_cursor_ignored: isReroute,
-        etaSeconds      : etaSeconds
+        etaSeconds      : etaSeconds,
+        remainingDates  : remainingDatesCount // For Workflow consumption
     };
 }
 

package/functions/computation-system/helpers/computation_worker.js

@@ -1,7 +1,6 @@
 /**
  * FILENAME: computation-system/helpers/computation_worker.js
- * UPDATED: Removed redundant Callback and Sentinel logic.
- * The system now relies on Dispatcher cursor satiation.
+ * UPDATED: Fixed Firestore 'undefined' field error for dispatchId.
  */
 
 const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
@@ -17,7 +16,6 @@ const calculations = calculationPackage.calculations;
 
 const MAX_RETRIES = 3;
 
-/** Black Box Recorder for Peak Memory. */
 function startMemoryHeartbeat(db, ledgerPath, intervalMs = 2000) {
     let peakRss = 0;
     const timer = setInterval(async () => {
@@ -48,13 +46,15 @@ async function handleComputationTask(message, config, dependencies) {
 
     logger.log('INFO', `[Worker] 📥 Task: ${computation} (${date}) [Tier: ${resourceTier}]`);
 
-    // 1. Audit Lease (Lease on life for the monitor to see)
-    await db.doc(ledgerPath).set({
+    // [FIX] Build document object and only add dispatchId if it is defined
+    const leaseData = {
         status: 'IN_PROGRESS',
         workerId: process.env.K_REVISION || os.hostname(),
-        startedAt: new Date(),
-        dispatchId
-    }, { merge: true });
+        startedAt: new Date()
+    };
+    if (dispatchId) leaseData.dispatchId = dispatchId;
+
+    await db.doc(ledgerPath).set(leaseData, { merge: true });
 
     const heartbeat = startMemoryHeartbeat(db, ledgerPath);
 
@@ -84,7 +84,6 @@ async function handleComputationTask(message, config, dependencies) {
             composition: calcUpdate.composition
         };
 
-        // Mark ledger as completed
         await db.doc(ledgerPath).update({ status: 'COMPLETED', completedAt: new Date() });
         await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, metrics, triggerReason, resourceTier);
 
@@ -95,9 +94,9 @@ async function handleComputationTask(message, config, dependencies) {
         if (isDeterministic || (message.deliveryAttempt || 1) >= MAX_RETRIES) {
             await db.doc(ledgerPath).set({ status: 'FAILED', error: err.message, failedAt: new Date() }, { merge: true });
             await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'FATAL' }, { peakMemoryMB: heartbeat.getPeak() }, triggerReason, resourceTier);
-            return; // Exit without throwing to prevent endless Pub/Sub retries for dead logic
+            return;
         }
-        throw err; // Trigger Pub/Sub retry for non-deterministic transient errors
+        throw err;
     }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.310",
+  "version": "1.0.312",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [