bulltrackers-module 1.0.324 → 1.0.325

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,7 +1,7 @@
 /**
  * FILENAME: computation-system/helpers/computation_dispatcher.js
- * PURPOSE: Sequential Cursor-Based Dispatcher with Ledger Awareness, SimHash Stability, and Session Caching.
- * UPDATED: Fixed Ledger Blindness, Cursor Shifting, and Live Analysis Disconnect.
+ * PURPOSE: Sequential Cursor-Based Dispatcher.
+ * BEHAVIOR: Dispatch -> Wait ETA -> Next Date.
  */
 
 const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
@@ -34,7 +34,7 @@ async function filterActiveTasks(db, date, pass, tasks) {
                                    data.completedAt && 
                                    (Date.now() - new Date(data.completedAt).getTime() < 60 * 1000);
             
-            if (isActive || isJustFinished) return null;
+            if (isActive || isJustFinished) return null; // Filter out
         }
         return t;
     });
@@ -50,18 +50,13 @@ async function attemptSimHashResolution(dependencies, date, tasks, dailyStatus,
     const { db, logger } = dependencies;
     const resolvedTasks = [];
     const remainingTasks = [];
-
-    // Cache for SimHashes to avoid redundant DB lookups in loop
     const simHashCache = new Map();
 
     for (const task of tasks) {
-        // Only apply to Re-Runs (Hash Mismatches), not fresh runs (Missing Data)
         const currentStatus = dailyStatus ? dailyStatus[task.name] : null;
         const manifestItem = manifestMap.get(normalizeName(task.name));
 
         if (currentStatus && currentStatus.simHash && manifestItem) {
-            
-            // 1. Get the SimHash for the NEW code (from Registry)
             let newSimHash = simHashCache.get(manifestItem.hash);
             if (!newSimHash) {
                 const simDoc = await db.collection('system_simhash_registry').doc(manifestItem.hash).get();
@@ -71,13 +66,12 @@ async function attemptSimHashResolution(dependencies, date, tasks, dailyStatus,
                 }
             }
 
-            // 2. Compare
             if (newSimHash && newSimHash === currentStatus.simHash) {
                 resolvedTasks.push({
                     name: task.name,
                     hash: manifestItem.hash,
                     simHash: newSimHash,
-                    prevStatus: currentStatus // Pass previous status to preserve other fields
+                    prevStatus: currentStatus
                 });
                 continue; 
             }
@@ -85,30 +79,24 @@ async function attemptSimHashResolution(dependencies, date, tasks, dailyStatus,
         remainingTasks.push(task);
     }
 
-    // 3. Apply Updates for Stable Tasks
     if (resolvedTasks.length > 0) {
         const updatePayload = {};
-        
         resolvedTasks.forEach(t => {
-            // [FIXED] Construct full nested object to avoid dot-notation issues with .set()
-            // We merge existing data (like resultHash) so we don't lose the valid calculation output
             updatePayload[t.name] = {
-                ...(t.prevStatus || {}), // Keep existing resultHash, output, etc.
-                hash: t.hash,            // Update to new code hash
-                simHash: t.simHash,      // Confirmed stable simHash
+                ...(t.prevStatus || {}),
+                hash: t.hash,
+                simHash: t.simHash,
                 reason: 'SimHash Stable (Auto-Resolved)',
                 lastUpdated: new Date().toISOString()
             };
         });
-        
-        // Use set with merge: true. Now that keys are "clean" (no dots), 
-        // objects will merge correctly into the document structure.
         await db.collection('computation_status').doc(date).set(updatePayload, { merge: true });
         logger.log('INFO', `[SimHash] ⏩ Fast-forwarded ${resolvedTasks.length} tasks for ${date} (Logic Unchanged).`);
     }
 
     return remainingTasks;
 }
+
 // =============================================================================
 // HELPER: Stable Session Management (Solves Cursor Shifting)
 // =============================================================================
@@ -117,36 +105,22 @@ async function getStableDateSession(config, dependencies, passToRun, dateLimitSt
     const sessionId = `pass_${passToRun}_${dateLimitStr.replace(/-/g, '')}`;
     const sessionRef = db.collection('dispatcher_sessions').doc(sessionId);
 
-    // 1. Try to Load Session
     if (!forceRebuild) {
         const sessionSnap = await sessionRef.get();
         if (sessionSnap.exists) {
             const data = sessionSnap.data();
-            const age = Date.now() - new Date(data.createdAt).getTime();
-            if (age < SESSION_CACHE_DURATION_MS) {
-                logger.log('INFO', `[Session] 📂 Loaded stable session for Pass ${passToRun} (${data.dates.length} dates).`);
+            if ((Date.now() - new Date(data.createdAt).getTime()) < SESSION_CACHE_DURATION_MS) {
+                // logger.log('INFO', `[Session] 📂 Loaded stable session for Pass ${passToRun}.`);
                 return data.dates;
             }
         }
     }
 
-    // 2. Rebuild Session (Expensive Scan)
     logger.log('INFO', `[Session] 🔄 Rebuilding dispatch session for Pass ${passToRun}...`);
     const earliestDates = await getEarliestDataDates(config, dependencies);
     const allDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date(dateLimitStr + 'T00:00:00Z'));
     
-    // We only want dates that *might* be dirty.
-    // Optimization: We add ALL dates to the list. The dispatcher checks them individually.
-    // Why? Because if we pre-filter here, we repeat the work of the dispatcher.
-    // Better: Store the plain list of dates sorted descending (newest first usually better for backfills, ascending for standard).
-    // Let's stick to Ascending (oldest first) as standard.
-    
-    await sessionRef.set({
-        dates: allDates,
-        createdAt: new Date().toISOString(),
-        configHash: dateLimitStr // Simple versioning
-    });
-
+    await sessionRef.set({ dates: allDates, createdAt: new Date().toISOString(), configHash: dateLimitStr });
     return allDates;
 }
 
@@ -157,9 +131,9 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     const { logger, db } = dependencies;
     const pubsubUtils    = new PubSubUtils(dependencies);
     
-    const passToRun     = String(reqBody.pass || config.COMPUTATION_PASS_TO_RUN || "1");
+    const passToRun     = String(reqBody.pass || "1");
     const targetCursorN = parseInt(reqBody.cursorIndex || 1);
-    const dateLimitStr  = reqBody.date || config.date || "2025-01-01";
+    const dateLimitStr  = reqBody.date || "2025-01-01";
     const forceRebuild  = reqBody.forceRebuild === true;
 
     const manifestMap     = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
@@ -167,38 +141,26 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     const calcsInThisPass = passes[passToRun] || [];
     const manifestWeightMap = new Map(computationManifest.map(c => [normalizeName(c.name), c.weight || 1.0]));
 
-    if (!calcsInThisPass.length) {
-        return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
-    }
+    if (!calcsInThisPass.length) return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
 
-    // 1. Get Stable Date List (Solves Shifting Cursor)
+    // 1. Get Stable Date List
     const sessionDates = await getStableDateSession(config, dependencies, passToRun, dateLimitStr, forceRebuild);
-    
-    if (!sessionDates || sessionDates.length === 0) {
-        return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
-    }
+    if (!sessionDates || sessionDates.length === 0) return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
 
-    // 2. Select Date based on Cursor
+    // 2. Select Date
     let selectedDate = null;
     let selectedTasks = [];
-    let isReroute = false;
-    let isSweep = false;
 
-    // Check bounds
     if (targetCursorN <= sessionDates.length) {
-        // Normal Operation
         selectedDate = sessionDates[targetCursorN - 1];
     } else {
-        // End of list
         return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
     }
 
-    // 3. Analyze SPECIFIC Date (Live Analysis)
-    // We only fetch status for the ONE date we are looking at + context
+    // 3. Analyze SPECIFIC Date
     if (selectedDate) {
-        // A. Fetch Context
-        const needsHistory = calcsInThisPass.some(c => c.isHistorical);
         const earliestDates = await getEarliestDataDates(config, dependencies);
+        const needsHistory = calcsInThisPass.some(c => c.isHistorical);
         
         let prevDailyStatusPromise = Promise.resolve(null);
         if (needsHistory) {
@@ -219,51 +181,42 @@ async function dispatchComputationPass(config, dependencies, computationManifest
             const report = analyzeDateExecution(selectedDate, calcsInThisPass, availability.status, dailyStatus, manifestMap, prevDailyStatus);
             let rawTasks = [...report.runnable, ...report.reRuns];
 
-            // B. Apply SimHash Resolution (Problem #1)
             if (rawTasks.length > 0) {
                 rawTasks = await attemptSimHashResolution(dependencies, selectedDate, rawTasks, dailyStatus, manifestMap);
-            }
-
-            // C. Apply Ledger Filter (Problem #2)
-            if (rawTasks.length > 0) {
+                
+                // Ledger Filter: Removes tasks that are already running
                 selectedTasks = await filterActiveTasks(db, selectedDate, passToRun, rawTasks);
             }
             
-            // D. Check for High-Mem Reroutes (OOM handling)
+            // OOM / High-Mem Reroute Check
             if (selectedTasks.length > 0) {
                 const reroutes = await getHighMemReroutes(db, selectedDate, passToRun, selectedTasks);
                 if (reroutes.length > 0) {
                     selectedTasks = reroutes;
-                    isReroute = true;
                 }
             }
-        } else {
-            logger.log('WARN', `[Dispatcher] Date ${selectedDate} skipped (Data Unavailable).`);
         }
     }
 
     // 4. Dispatch Logic
     if (selectedTasks.length === 0) {
-        // Nothing to do for this date.
-        // CRITICAL: We return dispatched: 0, but n_cursor_ignored: FALSE.
-        // This tells workflow to increment cursor and check the next date in the Stable Session.
+        // Return 0 dispatched, FALSE cursor ignored -> Move to NEXT date immediately.
         return {
             status: 'CONTINUE_PASS',
             dateProcessed: selectedDate,
             dispatched: 0,
-            n_cursor_ignored: false, // Proceed to next date
+            n_cursor_ignored: false,
             etaSeconds: 0,
             remainingDates: sessionDates.length - targetCursorN
         };
     }
 
-    // 5. Publish Tasks
+    // 5. Send Tasks
     const totalweight = selectedTasks.reduce((sum, t) => sum + (manifestWeightMap.get(normalizeName(t.name)) || 1.0), 0);
     const currentDispatchId = crypto.randomUUID();
     const etaSeconds = Math.max(20, Math.ceil(totalweight * BASE_SECONDS_PER_WEIGHT_UNIT));
 
     const taskDetails = selectedTasks.map(t => `${t.name} (${t.reason})`);
-
     logger.log('INFO', `[Dispatcher] ✅ Dispatching ${selectedTasks.length} tasks for ${selectedDate}.`, {
         date: selectedDate,
         pass: passToRun,
@@ -305,13 +258,14 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     }
     await Promise.all(pubPromises);
 
-    // CRITICAL: We dispatched work. We want to check THIS date again next time 
-    // to ensure tasks completed. So we IGNORE cursor increment.
+    // CRITICAL: We dispatched work. 
+    // We return n_cursor_ignored: FALSE.
+    // This tells the workflow to Wait ETA -> Increment Cursor -> Move to Next Date.
     return {
         status: 'CONTINUE_PASS',
         dateProcessed: selectedDate,
         dispatched: selectedTasks.length,
-        n_cursor_ignored: true, // Hold cursor until this date is clean
+        n_cursor_ignored: false, // FORCE NEXT DATE
         etaSeconds: etaSeconds,
         remainingDates: sessionDates.length - targetCursorN
     };

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,5 +1,5 @@
 # Cloud Workflows: Precision Cursor-Based Orchestrator
-# UPDATED: Added satiation detection to break early on 0 remaining dates.
+# SIMPLE MODE: Dispatch -> Wait ETA -> Next Date
 
 main:
   params: [input]
@@ -20,7 +20,6 @@ main:
                 assign:
                   - n_cursor: 1
                   - pass_complete: false
-                  - consecutive_empty_dispatches: 0
 
             - sequential_date_loop:
                 switch:
@@ -39,26 +38,25 @@ main:
 
                       - evaluate_dispatch:
                           switch:
+                            # 1. End of Session (Dispatcher reached end of date list)
                             - condition: '${dispatch_res.body.status == "MOVE_TO_NEXT_PASS"}'
                               assign:
                                 - pass_complete: true
                             
-                            # NEW: Explicit Satiation Check
-                            - condition: '${dispatch_res.body.status == "CONTINUE_PASS" and dispatch_res.body.remainingDates == 0}'
+                            # 2. Satiation Check (Specific to date/logic)
+                            - condition: '${dispatch_res.body.status == "CONTINUE_PASS" and dispatch_res.body.remainingDates == 0 and dispatch_res.body.dispatched == 0}'
                               steps:
                                 - log_satiation:
                                     call: sys.log
                                     args:
-                                      text: '${"Pass " + pass_id + " - ✅ Pass satiated (0 remaining dates). Moving to next pass."}'
+                                      text: '${"Pass " + pass_id + " - ✅ Pass satiated (0 remaining). Next pass."}'
                                 - mark_complete:
                                     assign:
                                       - pass_complete: true
 
+                            # 3. Work Dispatched: Wait ETA -> Move Next (Ignored flag is FALSE)
                             - condition: '${dispatch_res.body.dispatched > 0}'
                               steps:
-                                - reset_retry_counter:
-                                    assign:
-                                      - consecutive_empty_dispatches: 0
                                 - wait_for_completion:
                                     call: sys.sleep
                                     args:
@@ -69,26 +67,18 @@ main:
                                 - next_loop_work:
                                     next: sequential_date_loop
 
+                            # 4. No Work (Clean or Busy): Move Next Immediately
                             - condition: '${dispatch_res.body.dispatched == 0}'
                               steps:
-                                - increment_retry:
+                                - wait_short:
+                                    call: sys.sleep
+                                    args:
+                                      seconds: 2 # Tiny debounce
+                                - update_cursor_retry:
                                     assign:
-                                      - consecutive_empty_dispatches: '${consecutive_empty_dispatches + 1}'
-                                - check_break_condition:
-                                    switch:
-                                      - condition: '${consecutive_empty_dispatches >= 3}'
-                                        assign:
-                                          - pass_complete: true
-                                      - condition: '${true}'
-                                        steps:
-                                          - wait_short:
-                                              call: sys.sleep
-                                              args:
-                                                seconds: 5
-                                          - update_cursor_retry:
-                                              assign:
-                                                 - n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
-                                          - next_loop_retry:
-                                              next: sequential_date_loop
+                                        # Dispatcher sends n_cursor_ignored=false, so we increment.
+                                        - n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
+                                - next_loop_retry:
+                                    next: sequential_date_loop
     - finish:
         return: "Pipeline Execution Satiated and Complete"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.324",
+  "version": "1.0.325",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [