bulltrackers-module 1.0.323 → 1.0.325

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,7 +1,7 @@
 /**
  * FILENAME: computation-system/helpers/computation_dispatcher.js
- * PURPOSE: Sequential Cursor-Based Dispatcher with Hyper-Verbose Telemetry.
- * UPDATED: Sweep mode now auto-upgrades missed tasks to high-mem due to potential silent OOMs.
+ * PURPOSE: Sequential Cursor-Based Dispatcher.
+ * BEHAVIOR: Dispatch -> Wait ETA -> Next Date.
  */
 
 const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
@@ -9,138 +9,222 @@ const { groupByPass, analyzeDateExecution } = require('../WorkflowOrchestrator.j
 const { PubSubUtils } = require('../../core/utils/pubsub_utils');
 const { fetchComputationStatus } = require('../persistence/StatusRepository');
 const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
-const crypto = require('crypto'); 
+const crypto = require('crypto');
 
-const OOM_THRESHOLD_MB = 1500; 
+const OOM_THRESHOLD_MB = 1500;
 const BASE_SECONDS_PER_WEIGHT_UNIT = 3;
+const SESSION_CACHE_DURATION_MS = 1000 * 60 * 30; // 30 Minutes
 
-async function getHighMemReroutes(db, date, pass, tasks) {
-    const reroutes = [];
-    for (const task of tasks) {
-        const name       = normalizeName(task.name);
-        const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${name}`;
-        const doc        = await db.doc(ledgerPath).get();
+// =============================================================================
+// HELPER: Ledger Awareness (Prevents Race Conditions)
+// =============================================================================
+async function filterActiveTasks(db, date, pass, tasks) {
+    if (!tasks || tasks.length === 0) return [];
+    
+    const checkPromises = tasks.map(async (t) => {
+        const taskName = normalizeName(t.name);
+        const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${taskName}`;
+        const snap = await db.doc(ledgerPath).get();
         
-        if (doc.exists) {
-            const data = doc.data();
-            const isOOM = (data.status === 'FAILED' || data.status === 'CRASH') && 
-                          (data.resourceTier !== 'high-mem') &&
-                          ((data.peakMemoryMB > OOM_THRESHOLD_MB) || (data.error && /memory/i.test(data.error.message)));
+        if (snap.exists) {
+            const data = snap.data();
+            // Check PENDING, IN_PROGRESS, or "Ghost" (Completed < 1 min ago)
+            const isActive = ['PENDING', 'IN_PROGRESS'].includes(data.status);
+            const isJustFinished = data.status === 'COMPLETED' && 
+                                   data.completedAt && 
+                                   (Date.now() - new Date(data.completedAt).getTime() < 60 * 1000);
             
-            if (isOOM) {
-                reroutes.push({ ...task, resources: 'high-mem' });
+            if (isActive || isJustFinished) return null; // Filter out
+        }
+        return t;
+    });
+    
+    const results = await Promise.all(checkPromises);
+    return results.filter(t => t !== null);
+}
+
+// =============================================================================
+// HELPER: SimHash Stability (Solves Live Analysis Disconnect)
+// =============================================================================
+async function attemptSimHashResolution(dependencies, date, tasks, dailyStatus, manifestMap) {
+    const { db, logger } = dependencies;
+    const resolvedTasks = [];
+    const remainingTasks = [];
+    const simHashCache = new Map();
+
+    for (const task of tasks) {
+        const currentStatus = dailyStatus ? dailyStatus[task.name] : null;
+        const manifestItem = manifestMap.get(normalizeName(task.name));
+
+        if (currentStatus && currentStatus.simHash && manifestItem) {
+            let newSimHash = simHashCache.get(manifestItem.hash);
+            if (!newSimHash) {
+                const simDoc = await db.collection('system_simhash_registry').doc(manifestItem.hash).get();
+                if (simDoc.exists) {
+                    newSimHash = simDoc.data().simHash;
+                    simHashCache.set(manifestItem.hash, newSimHash);
+                }
+            }
+
+            if (newSimHash && newSimHash === currentStatus.simHash) {
+                resolvedTasks.push({
+                    name: task.name,
+                    hash: manifestItem.hash,
+                    simHash: newSimHash,
+                    prevStatus: currentStatus
+                });
+                continue; 
             }
         }
+        remainingTasks.push(task);
     }
-    return reroutes;
+
+    if (resolvedTasks.length > 0) {
+        const updatePayload = {};
+        resolvedTasks.forEach(t => {
+            updatePayload[t.name] = {
+                ...(t.prevStatus || {}),
+                hash: t.hash,
+                simHash: t.simHash,
+                reason: 'SimHash Stable (Auto-Resolved)',
+                lastUpdated: new Date().toISOString()
+            };
+        });
+        await db.collection('computation_status').doc(date).set(updatePayload, { merge: true });
+        logger.log('INFO', `[SimHash] ⏩ Fast-forwarded ${resolvedTasks.length} tasks for ${date} (Logic Unchanged).`);
+    }
+
+    return remainingTasks;
+}
+
+// =============================================================================
+// HELPER: Stable Session Management (Solves Cursor Shifting)
+// =============================================================================
+async function getStableDateSession(config, dependencies, passToRun, dateLimitStr, forceRebuild) {
+    const { db, logger } = dependencies;
+    const sessionId = `pass_${passToRun}_${dateLimitStr.replace(/-/g, '')}`;
+    const sessionRef = db.collection('dispatcher_sessions').doc(sessionId);
+
+    if (!forceRebuild) {
+        const sessionSnap = await sessionRef.get();
+        if (sessionSnap.exists) {
+            const data = sessionSnap.data();
+            if ((Date.now() - new Date(data.createdAt).getTime()) < SESSION_CACHE_DURATION_MS) {
+                // logger.log('INFO', `[Session] 📂 Loaded stable session for Pass ${passToRun}.`);
+                return data.dates;
+            }
+        }
+    }
+
+    logger.log('INFO', `[Session] 🔄 Rebuilding dispatch session for Pass ${passToRun}...`);
+    const earliestDates = await getEarliestDataDates(config, dependencies);
+    const allDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date(dateLimitStr + 'T00:00:00Z'));
+    
+    await sessionRef.set({ dates: allDates, createdAt: new Date().toISOString(), configHash: dateLimitStr });
+    return allDates;
 }
 
+// =============================================================================
+// MAIN DISPATCHER
+// =============================================================================
 async function dispatchComputationPass(config, dependencies, computationManifest, reqBody = {}) {
     const { logger, db } = dependencies;
     const pubsubUtils    = new PubSubUtils(dependencies);
     
-    const passToRun     = String(reqBody.pass || config.COMPUTATION_PASS_TO_RUN || "1");
+    const passToRun     = String(reqBody.pass || "1");
     const targetCursorN = parseInt(reqBody.cursorIndex || 1);
-    const dateLimitStr  = reqBody.date || config.date || "2025-01-01"; 
+    const dateLimitStr  = reqBody.date || "2025-01-01";
+    const forceRebuild  = reqBody.forceRebuild === true;
 
     const manifestMap     = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
     const passes          = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
-
     const manifestWeightMap = new Map(computationManifest.map(c => [normalizeName(c.name), c.weight || 1.0]));
 
-    if (!calcsInThisPass.length) {
-        logger.log('WARN', `[Dispatcher] 🛑 No calculations found for Pass ${passToRun}.`);
-        return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
-    }
+    if (!calcsInThisPass.length) return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
 
-    const earliestDates = await getEarliestDataDates(config, dependencies);
-    const allDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date(dateLimitStr + 'T00:00:00Z'));
-    
-    if (allDates.length === 0) {
-        logger.log('ERROR', `[Dispatcher] ❌ Date range is empty.`);
+    // 1. Get Stable Date List
+    const sessionDates = await getStableDateSession(config, dependencies, passToRun, dateLimitStr, forceRebuild);
+    if (!sessionDates || sessionDates.length === 0) return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
+
+    // 2. Select Date
+    let selectedDate = null;
+    let selectedTasks = [];
+
+    if (targetCursorN <= sessionDates.length) {
+        selectedDate = sessionDates[targetCursorN - 1];
+    } else {
         return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
     }
 
-    // 1. Identify all "Dirty" dates (dates that actually have work to do)
-    const dirtyDates = [];
-    for (const d of allDates) {
-        const dailyStatus  = await fetchComputationStatus(d, config, dependencies);
-        const availability = await checkRootDataAvailability(d, config, dependencies, DEFINITIVE_EARLIEST_DATES);
+    // 3. Analyze SPECIFIC Date
+    if (selectedDate) {
+        const earliestDates = await getEarliestDataDates(config, dependencies);
+        const needsHistory = calcsInThisPass.some(c => c.isHistorical);
         
-        if (!availability || !availability.status.hasPrices) continue;
-
-        const report = analyzeDateExecution(d, calcsInThisPass, availability.status, dailyStatus, manifestMap, null);
-        const tasks  = [...report.runnable, ...report.reRuns];
-        
-        if (tasks.length > 0) {
-            dirtyDates.push({ date: d, tasks });
+        let prevDailyStatusPromise = Promise.resolve(null);
+        if (needsHistory) {
+            const prevD = new Date(selectedDate + 'T00:00:00Z');
+            prevD.setUTCDate(prevD.getUTCDate() - 1);
+            if (prevD >= earliestDates.absoluteEarliest) {
+                prevDailyStatusPromise = fetchComputationStatus(prevD.toISOString().slice(0, 10), config, dependencies);
+            }
         }
-    }
 
-    let selectedDate  = null;
-    let selectedTasks = [];
-    let isReroute     = false;
-    let isSweep       = false;
-
-    // Logic for Reroutes (Known OOM handling)
-    if (targetCursorN > 1 && (targetCursorN - 2) < dirtyDates.length) {
-        const prevEntry = dirtyDates[targetCursorN - 2];
-        const reroutes  = await getHighMemReroutes(db, prevEntry.date, passToRun, prevEntry.tasks);
-        if (reroutes.length > 0) {
-            selectedDate  = prevEntry.date;
-            selectedTasks = reroutes;
-            isReroute     = true;
-        }
-    }
+        const [dailyStatus, prevDailyStatus, availability] = await Promise.all([
+            fetchComputationStatus(selectedDate, config, dependencies),
+            prevDailyStatusPromise,
+            checkRootDataAvailability(selectedDate, config, dependencies, DEFINITIVE_EARLIEST_DATES)
+        ]);
+
+        if (availability && availability.status.hasPrices) {
+            const report = analyzeDateExecution(selectedDate, calcsInThisPass, availability.status, dailyStatus, manifestMap, prevDailyStatus);
+            let rawTasks = [...report.runnable, ...report.reRuns];
 
-    // Logic for standard cursor progression or Sweep (Recovery) mode
-    if (!selectedDate) {
-        if (targetCursorN <= dirtyDates.length) {
-            const entry   = dirtyDates[targetCursorN - 1];
-            selectedDate  = entry.date;
-            selectedTasks = entry.tasks;
-        } else if (dirtyDates.length > 0) {
-            // RECOVERY/SWEEP MODE:
-            // The cursor has passed the number of dirty dates, but tasks still remain.
-            // These tasks are automatically routed to 'high-mem' with recovery reasoning.
-            isSweep       = true;
-            selectedDate  = dirtyDates[0].date;
-            selectedTasks = dirtyDates[0].tasks.map(t => ({
-                ...t,
-                resources: 'high-mem',
-                reason: `${t.reason || 'Missed Computation'} [RECOVERY_AUTO_UPGRADE: Potential Silent OOM]`
-            }));
+            if (rawTasks.length > 0) {
+                rawTasks = await attemptSimHashResolution(dependencies, selectedDate, rawTasks, dailyStatus, manifestMap);
+                
+                // Ledger Filter: Removes tasks that are already running
+                selectedTasks = await filterActiveTasks(db, selectedDate, passToRun, rawTasks);
+            }
+            
+            // OOM / High-Mem Reroute Check
+            if (selectedTasks.length > 0) {
+                const reroutes = await getHighMemReroutes(db, selectedDate, passToRun, selectedTasks);
+                if (reroutes.length > 0) {
+                    selectedTasks = reroutes;
+                }
+            }
         }
     }
 
-    if (!selectedDate) {
-        logger.log('INFO', `[Dispatcher] 🏁 Pass ${passToRun} is fully satiated. No work remaining.`);
-        return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0, etaSeconds: 0 };
+    // 4. Dispatch Logic
+    if (selectedTasks.length === 0) {
+        // Return 0 dispatched, FALSE cursor ignored -> Move to NEXT date immediately.
+        return {
+            status: 'CONTINUE_PASS',
+            dateProcessed: selectedDate,
+            dispatched: 0,
+            n_cursor_ignored: false,
+            etaSeconds: 0,
+            remainingDates: sessionDates.length - targetCursorN
+        };
     }
 
-    const totalweight = selectedTasks.reduce((sum, t) => {
-        const weight = manifestWeightMap.get(normalizeName(t.name)) || 1.0;
-        return sum + weight;
-    }, 0);
-
-    // 2. Prepare Payload and Telemetry
+    // 5. Send Tasks
+    const totalweight = selectedTasks.reduce((sum, t) => sum + (manifestWeightMap.get(normalizeName(t.name)) || 1.0), 0);
     const currentDispatchId = crypto.randomUUID();
     const etaSeconds = Math.max(20, Math.ceil(totalweight * BASE_SECONDS_PER_WEIGHT_UNIT));
-    const remainingDatesCount = Math.max(0, dirtyDates.length - targetCursorN);
 
-    // [UPDATED] Capture both name and reason for transparency
     const taskDetails = selectedTasks.map(t => `${t.name} (${t.reason})`);
-
-    logger.log('INFO', `[Dispatcher] ✅ Dispatching ${selectedTasks.length} tasks for ${selectedDate}. ETA: ${etaSeconds}s. [Mode: ${isSweep ? 'RECOVERY' : 'NORMAL'}]`, {
-        date:                 selectedDate,
-        pass:                 passToRun,
-        dispatchedCount:      selectedTasks.length,
-        remainingCursorDates: remainingDatesCount,
-        totalweight:          totalweight,
-        etaSeconds:           etaSeconds,
-        dispatchId:           currentDispatchId,
-        tasks:                taskDetails // [UPDATED] Now logs "calc-name (Reason)"
+    logger.log('INFO', `[Dispatcher] ✅ Dispatching ${selectedTasks.length} tasks for ${selectedDate}.`, {
+        date: selectedDate,
+        pass: passToRun,
+        dispatchedCount: selectedTasks.length,
+        cursor: targetCursorN,
+        etaSeconds: etaSeconds,
+        dispatchId: currentDispatchId,
+        tasks: taskDetails 
     });
 
     const mapToTaskPayload = (t) => ({
@@ -150,7 +234,7 @@ async function dispatchComputationPass(config, dependencies, computationManifest
         date: selectedDate,
         pass: passToRun,
         dispatchId: currentDispatchId,
-        triggerReason: t.reason, // Already passed to worker
+        triggerReason: t.reason, 
         resources: t.resources || 'standard'
     });
 
@@ -161,27 +245,50 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     if (standardTasks.length > 0) {
         pubPromises.push(pubsubUtils.batchPublishTasks(dependencies, {
             topicName: config.computationTopicStandard || 'computation-tasks',
-            tasks    : standardTasks,
-            taskType : `pass-${passToRun}-std`
+            tasks: standardTasks,
+            taskType: `pass-${passToRun}-std`
         }));
     }
     if (highMemTasks.length > 0) {
         pubPromises.push(pubsubUtils.batchPublishTasks(dependencies, {
             topicName: config.computationTopicHighMem || 'computation-tasks-highmem',
-            tasks    : highMemTasks,
-            taskType : `pass-${passToRun}-high`
+            tasks: highMemTasks,
+            taskType: `pass-${passToRun}-high`
         }));
     }
     await Promise.all(pubPromises);
 
+    // CRITICAL: We dispatched work. 
+    // We return n_cursor_ignored: FALSE.
+    // This tells the workflow to Wait ETA -> Increment Cursor -> Move to Next Date.
     return {
-        status          : isSweep ? 'RECOVERY' : 'CONTINUE_PASS',
-        dateProcessed   : selectedDate,
-        dispatched      : selectedTasks.length,
-        n_cursor_ignored: isReroute,
-        etaSeconds      : etaSeconds,
-        remainingDates  : remainingDatesCount 
+        status: 'CONTINUE_PASS',
+        dateProcessed: selectedDate,
+        dispatched: selectedTasks.length,
+        n_cursor_ignored: false, // FORCE NEXT DATE
+        etaSeconds: etaSeconds,
+        remainingDates: sessionDates.length - targetCursorN
     };
 }
 
+async function getHighMemReroutes(db, date, pass, tasks) {
+    const reroutes = [];
+    for (const task of tasks) {
+        const name = normalizeName(task.name);
+        const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${name}`;
+        const doc = await db.doc(ledgerPath).get();
+        
+        if (doc.exists) {
+            const data = doc.data();
+            const isOOM = (data.status === 'FAILED' || data.status === 'CRASH') && 
+                          (data.resourceTier !== 'high-mem') &&
+                          ((data.peakMemoryMB > OOM_THRESHOLD_MB) || (data.error && /memory/i.test(data.error.message)));
+            if (isOOM) {
+                reroutes.push({ ...task, resources: 'high-mem' });
+            }
+        }
+    }
+    return reroutes;
+}
+
 module.exports = { dispatchComputationPass };

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,5 +1,5 @@
 # Cloud Workflows: Precision Cursor-Based Orchestrator
-# UPDATED: Added satiation detection to break early on 0 remaining dates.
+# SIMPLE MODE: Dispatch -> Wait ETA -> Next Date
 
 main:
   params: [input]
@@ -20,7 +20,6 @@ main:
                 assign:
                   - n_cursor: 1
                   - pass_complete: false
-                  - consecutive_empty_dispatches: 0
 
             - sequential_date_loop:
                 switch:
@@ -39,26 +38,25 @@ main:
 
                       - evaluate_dispatch:
                           switch:
+                            # 1. End of Session (Dispatcher reached end of date list)
                             - condition: '${dispatch_res.body.status == "MOVE_TO_NEXT_PASS"}'
                               assign:
                                 - pass_complete: true
                             
-                            # NEW: Explicit Satiation Check
-                            - condition: '${dispatch_res.body.status == "CONTINUE_PASS" and dispatch_res.body.remainingDates == 0}'
+                            # 2. Satiation Check (Specific to date/logic)
+                            - condition: '${dispatch_res.body.status == "CONTINUE_PASS" and dispatch_res.body.remainingDates == 0 and dispatch_res.body.dispatched == 0}'
                               steps:
                                 - log_satiation:
                                     call: sys.log
                                     args:
-                                      text: '${"Pass " + pass_id + " - ✅ Pass satiated (0 remaining dates). Moving to next pass."}'
+                                      text: '${"Pass " + pass_id + " - ✅ Pass satiated (0 remaining). Next pass."}'
                                 - mark_complete:
                                     assign:
                                       - pass_complete: true
 
+                            # 3. Work Dispatched: Wait ETA -> Move Next (Ignored flag is FALSE)
                             - condition: '${dispatch_res.body.dispatched > 0}'
                               steps:
-                                - reset_retry_counter:
-                                    assign:
-                                      - consecutive_empty_dispatches: 0
                                 - wait_for_completion:
                                     call: sys.sleep
                                     args:
@@ -69,26 +67,18 @@ main:
                                 - next_loop_work:
                                     next: sequential_date_loop
 
+                            # 4. No Work (Clean or Busy): Move Next Immediately
                             - condition: '${dispatch_res.body.dispatched == 0}'
                               steps:
-                                - increment_retry:
+                                - wait_short:
+                                    call: sys.sleep
+                                    args:
+                                      seconds: 2 # Tiny debounce
+                                - update_cursor_retry:
                                     assign:
-                                      - consecutive_empty_dispatches: '${consecutive_empty_dispatches + 1}'
-                                - check_break_condition:
-                                    switch:
-                                      - condition: '${consecutive_empty_dispatches >= 3}'
-                                        assign:
-                                          - pass_complete: true
-                                      - condition: '${true}'
-                                        steps:
-                                          - wait_short:
-                                              call: sys.sleep
-                                              args:
-                                                seconds: 5
-                                          - update_cursor_retry:
-                                              assign:
-                                                 - n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
-                                          - next_loop_retry:
-                                              next: sequential_date_loop
+                                        # Dispatcher sends n_cursor_ignored=false, so we increment.
+                                        - n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
+                                - next_loop_retry:
+                                    next: sequential_date_loop
     - finish:
         return: "Pipeline Execution Satiated and Complete"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.323",
+  "version": "1.0.325",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [