bulltrackers-module 1.0.328 → 1.0.330

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -2,7 +2,7 @@
  * FILENAME: computation-system/helpers/computation_dispatcher.js
  * PURPOSE: Sequential Cursor-Based Dispatcher.
  * BEHAVIOR: Dispatch -> Wait ETA -> Next Date.
- * UPDATED: Added "Zombie Protocol" to auto-recover stale locks.
+ * UPDATED: Added "Sweep" Protocol for OOM recovery & High-Mem Verification.
  */
 
 const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
@@ -15,16 +15,14 @@ const crypto = require('crypto');
 const OOM_THRESHOLD_MB = 1500;
 const BASE_SECONDS_PER_WEIGHT_UNIT = 3;
 const SESSION_CACHE_DURATION_MS = 1000 * 60 * 30; // 30 Minutes
-
-// [NEW] Zombie Timeout: Max Cloud Function run is 9m (540s). 
-// If no heartbeat/start within 15m, it's definitely dead.
 const STALE_LOCK_THRESHOLD_MS = 1000 * 60 * 15; 
 
 // =============================================================================
-// HELPER: Ledger Awareness (Prevents Race Conditions & Clears Zombies)
+// HELPER: Ledger Awareness
 // =============================================================================
-async function filterActiveTasks(db, date, pass, tasks, logger) {
+async function filterActiveTasks(db, date, pass, tasks, logger, forceRun = false) {
     if (!tasks || tasks.length === 0) return [];
+    if (forceRun) return tasks; // Bypass check for Sweep Mode
     
     const checkPromises = tasks.map(async (t) => {
         const taskName = normalizeName(t.name);
@@ -37,7 +35,6 @@ async function filterActiveTasks(db, date, pass, tasks, logger) {
 
             // 1. ZOMBIE CHECK (Recover Stale Locks)
             if (isActive) {
-                // Prefer heartbeat, fall back to start time
                 const lastActivityTime = data.telemetry?.lastHeartbeat 
                     ? new Date(data.telemetry.lastHeartbeat).getTime() 
                     : (data.startedAt ? new Date(data.startedAt).getTime() : 0);
@@ -45,25 +42,18 @@ async function filterActiveTasks(db, date, pass, tasks, logger) {
                 const timeSinceActive = Date.now() - lastActivityTime;
 
                 if (timeSinceActive > STALE_LOCK_THRESHOLD_MS) {
-                    if (logger) {
-                        logger.log('WARN', `[Dispatcher] 🧟 Breaking stale lock for ${taskName}. Inactive for ${(timeSinceActive/60000).toFixed(1)} mins.`);
-                    }
-                    // Return task (Re-dispatching it will overwrite the old lock in Firestore)
+                    if (logger) logger.log('WARN', `[Dispatcher] 🧟 Breaking stale lock for ${taskName}.`);
                     return t; 
                 }
-                
-                // If distinct and recent, filter it out (let it run)
                 return null; 
             }
 
             // 2. GHOST CHECK (Debounce immediate re-runs)
-            // If it finished less than 1 minute ago, don't re-dispatch immediately 
-            // (prevents double-tap if latency is high)
             const isJustFinished = data.status === 'COMPLETED' && 
                                    data.completedAt && 
                                    (Date.now() - new Date(data.completedAt).getTime() < 60 * 1000);
             
-            if (isJustFinished) return null; // Filter out
+            if (isJustFinished) return null; 
         }
         return t;
     });
@@ -73,7 +63,7 @@ async function filterActiveTasks(db, date, pass, tasks, logger) {
 }
 
 // =============================================================================
-// HELPER: SimHash Stability (Solves Live Analysis Disconnect)
+// HELPER: SimHash Stability
 // =============================================================================
 async function attemptSimHashResolution(dependencies, date, tasks, dailyStatus, manifestMap) {
     const { db, logger } = dependencies;
@@ -120,15 +110,12 @@ async function attemptSimHashResolution(dependencies, date, tasks, dailyStatus,
             };
         });
         await db.collection('computation_status').doc(date).set(updatePayload, { merge: true });
-        logger.log('INFO', `[SimHash] ⏩ Fast-forwarded ${resolvedTasks.length} tasks for ${date} (Logic Unchanged).`);
+        logger.log('INFO', `[SimHash] ⏩ Fast-forwarded ${resolvedTasks.length} tasks for ${date}.`);
     }
 
     return remainingTasks;
 }
 
-// =============================================================================
-// HELPER: Stable Session Management (Solves Cursor Shifting)
-// =============================================================================
 async function getStableDateSession(config, dependencies, passToRun, dateLimitStr, forceRebuild) {
     const { db, logger } = dependencies;
     const sessionId = `pass_${passToRun}_${dateLimitStr.replace(/-/g, '')}`;
@@ -153,29 +140,164 @@ async function getStableDateSession(config, dependencies, passToRun, dateLimitSt
 }
 
 // =============================================================================
-// MAIN DISPATCHER
+// MAIN ENTRY POINT
 // =============================================================================
 async function dispatchComputationPass(config, dependencies, computationManifest, reqBody = {}) {
+    const action = reqBody.action || 'DISPATCH'; // DISPATCH, VERIFY, SWEEP
+    
+    if (action === 'VERIFY') {
+        return handlePassVerification(config, dependencies, computationManifest, reqBody);
+    } 
+    else if (action === 'SWEEP') {
+        return handleSweepDispatch(config, dependencies, computationManifest, reqBody);
+    }
+    
+    return handleStandardDispatch(config, dependencies, computationManifest, reqBody);
+}
+
+// =============================================================================
+// LOGIC: Verify Pass Completion
+// =============================================================================
+async function handlePassVerification(config, dependencies, computationManifest, reqBody) {
+    const { logger } = dependencies;
+    const passToRun = String(reqBody.pass || "1");
+    const dateLimitStr = reqBody.date || "2025-01-01";
+    
+    logger.log('INFO', `[Verify] 🧹 Sweeping Pass ${passToRun} for unfinished work...`);
+    
+    const sessionDates = await getStableDateSession(config, dependencies, passToRun, dateLimitStr, false);
+    const passes = groupByPass(computationManifest);
+    const calcsInPass = passes[passToRun] || [];
+    const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
+    const weightMap = new Map(computationManifest.map(c => [normalizeName(c.name), c.weight || 1.0]));
+
+    const missingTasks = [];
+
+    // Optimize: Batch fetch statuses if possible, but for now loop is safer for memory
+    // In production, we might want p-limit here.
+    for (const date of sessionDates) {
+        const [dailyStatus, availability] = await Promise.all([
+            fetchComputationStatus(date, config, dependencies),
+            checkRootDataAvailability(date, config, dependencies, DEFINITIVE_EARLIEST_DATES)
+        ]);
+
+        // Need previous status for historical calcs
+        let prevDailyStatus = null;
+        if (calcsInPass.some(c => c.isHistorical)) {
+             const prevD = new Date(date + 'T00:00:00Z');
+             prevD.setUTCDate(prevD.getUTCDate() - 1);
+             prevDailyStatus = await fetchComputationStatus(prevD.toISOString().slice(0, 10), config, dependencies);
+        }
+
+        const report = analyzeDateExecution(date, calcsInPass, availability ? availability.status : {}, dailyStatus, manifestMap, prevDailyStatus);
+        
+        // We only care about Runnable (New) or ReRuns (Changed/Failed)
+        // We ignore Blocked (impossible to run) and Impossible (permanent fail)
+        const pending = [...report.runnable, ...report.reRuns];
+        
+        if (pending.length > 0) {
+            // Calculate ETA
+            const totalWeight = pending.reduce((sum, t) => sum + (weightMap.get(normalizeName(t.name)) || 1.0), 0);
+            const eta = Math.max(30, Math.ceil(totalWeight * BASE_SECONDS_PER_WEIGHT_UNIT));
+            
+            missingTasks.push({
+                date: date,
+                taskCount: pending.length,
+                eta: eta
+            });
+        }
+    }
+
+    logger.log('INFO', `[Verify] Found ${missingTasks.length} dates with pending work.`);
+    return { missingTasks };
+}
+
+// =============================================================================
+// LOGIC: Sweep Dispatch (Forced High-Mem)
+// =============================================================================
+async function handleSweepDispatch(config, dependencies, computationManifest, reqBody) {
     const { logger, db } = dependencies;
-    const pubsubUtils    = new PubSubUtils(dependencies);
+    const pubsubUtils = new PubSubUtils(dependencies);
+    const passToRun = String(reqBody.pass || "1");
+    const date = reqBody.date;
+
+    if (!date) throw new Error('Sweep dispatch requires date');
+
+    const passes = groupByPass(computationManifest);
+    const calcsInPass = passes[passToRun] || [];
+    const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
+
+    // 1. Analyze specific date
+    const [dailyStatus, availability] = await Promise.all([
+        fetchComputationStatus(date, config, dependencies),
+        checkRootDataAvailability(date, config, dependencies, DEFINITIVE_EARLIEST_DATES)
+    ]);
     
-    const passToRun     = String(reqBody.pass || "1");
+    // Previous Status Fetch (simplified for brevity, assume historical dependency check works or fails safe)
+    let prevDailyStatus = null;
+    if (calcsInPass.some(c => c.isHistorical)) {
+         const prevD = new Date(date + 'T00:00:00Z');
+         prevD.setUTCDate(prevD.getUTCDate() - 1);
+         prevDailyStatus = await fetchComputationStatus(prevD.toISOString().slice(0, 10), config, dependencies);
+    }
+
+    const report = analyzeDateExecution(date, calcsInPass, availability ? availability.status : {}, dailyStatus, manifestMap, prevDailyStatus);
+    const pending = [...report.runnable, ...report.reRuns];
+
+    if (pending.length === 0) {
+        logger.log('INFO', `[Sweep] ${date} is clean. No dispatch.`);
+        return { dispatched: 0 };
+    }
+
+    // 2. FORCE High Mem & Skip Zombie Check
+    // We do NOT filterActiveTasks because we assume they might be OOM'd zombies
+    // We do NOT check resource tier from previous runs, we FORCE 'high-mem'
+    const currentDispatchId = crypto.randomUUID();
+    
+    const tasksPayload = pending.map(t => ({
+        ...t,
+        action: 'RUN_COMPUTATION_DATE',
+        computation: t.name,
+        date: date,
+        pass: passToRun,
+        dispatchId: currentDispatchId,
+        triggerReason: 'SWEEP_RECOVERY',
+        resources: 'high-mem' // FORCE
+    }));
+
+    logger.log('WARN', `[Sweep] 🧹 Forcing ${tasksPayload.length} tasks to HIGH-MEM for ${date}.`);
+
+    await pubsubUtils.batchPublishTasks(dependencies, {
+        topicName: config.computationTopicHighMem || 'computation-tasks-highmem',
+        tasks: tasksPayload,
+        taskType: `pass-${passToRun}-sweep`
+    });
+
+    return { dispatched: tasksPayload.length };
+}
+
+// =============================================================================
+// LOGIC: Standard Dispatch (Original)
+// =============================================================================
+async function handleStandardDispatch(config, dependencies, computationManifest, reqBody) {
+    const { logger, db } = dependencies;
+    const pubsubUtils = new PubSubUtils(dependencies);
+    
+    const passToRun = String(reqBody.pass || "1");
     const targetCursorN = parseInt(reqBody.cursorIndex || 1);
-    const dateLimitStr  = reqBody.date || "2025-01-01";
-    const forceRebuild  = reqBody.forceRebuild === true;
+    const dateLimitStr = reqBody.date || "2025-01-01";
+    const forceRebuild = reqBody.forceRebuild === true;
 
-    const manifestMap     = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
-    const passes          = groupByPass(computationManifest);
+    const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
+    const passes = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
     const manifestWeightMap = new Map(computationManifest.map(c => [normalizeName(c.name), c.weight || 1.0]));
 
     if (!calcsInThisPass.length) return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
 
-    // 1. Get Stable Date List
     const sessionDates = await getStableDateSession(config, dependencies, passToRun, dateLimitStr, forceRebuild);
     if (!sessionDates || sessionDates.length === 0) return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
 
-    // 2. Select Date
     let selectedDate = null;
     let selectedTasks = [];
 
@@ -185,13 +307,10 @@ async function dispatchComputationPass(config, dependencies, computationManifest
         return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
     }
 
-    // 3. Analyze SPECIFIC Date
     if (selectedDate) {
         const earliestDates = await getEarliestDataDates(config, dependencies);
-        const needsHistory = calcsInThisPass.some(c => c.isHistorical);
-        
         let prevDailyStatusPromise = Promise.resolve(null);
-        if (needsHistory) {
+        if (calcsInThisPass.some(c => c.isHistorical)) {
             const prevD = new Date(selectedDate + 'T00:00:00Z');
             prevD.setUTCDate(prevD.getUTCDate() - 1);
             if (prevD >= earliestDates.absoluteEarliest) {
@@ -211,22 +330,16 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
             if (rawTasks.length > 0) {
                 rawTasks = await attemptSimHashResolution(dependencies, selectedDate, rawTasks, dailyStatus, manifestMap);
-                
-                // [UPDATED] Pass logger to filterActiveTasks for zombie warnings
                 selectedTasks = await filterActiveTasks(db, selectedDate, passToRun, rawTasks, logger);
             }
             
-            // OOM / High-Mem Reroute Check
             if (selectedTasks.length > 0) {
                 const reroutes = await getHighMemReroutes(db, selectedDate, passToRun, selectedTasks);
-                if (reroutes.length > 0) {
-                    selectedTasks = reroutes;
-                }
+                if (reroutes.length > 0) selectedTasks = reroutes;
             }
         }
     }
 
-    // 4. Dispatch Logic
     if (selectedTasks.length === 0) {
         return {
             status: 'CONTINUE_PASS',
@@ -238,31 +351,18 @@ async function dispatchComputationPass(config, dependencies, computationManifest
         };
     }
 
-    // 5. Send Tasks
     const totalweight = selectedTasks.reduce((sum, t) => sum + (manifestWeightMap.get(normalizeName(t.name)) || 1.0), 0);
     const currentDispatchId = crypto.randomUUID();
     const etaSeconds = Math.max(20, Math.ceil(totalweight * BASE_SECONDS_PER_WEIGHT_UNIT));
 
     const taskDetails = selectedTasks.map(t => `${t.name} (${t.reason})`);
     logger.log('INFO', `[Dispatcher] ✅ Dispatching ${selectedTasks.length} tasks for ${selectedDate}.`, {
-        date: selectedDate,
-        pass: passToRun,
-        dispatchedCount: selectedTasks.length,
-        cursor: targetCursorN,
-        etaSeconds: etaSeconds,
-        dispatchId: currentDispatchId,
-        tasks: taskDetails 
+        date: selectedDate, pass: passToRun, dispatchedCount: selectedTasks.length, etaSeconds, dispatchId: currentDispatchId
     });
 
     const mapToTaskPayload = (t) => ({
-        ...t,
-        action: 'RUN_COMPUTATION_DATE',
-        computation: t.name,
-        date: selectedDate,
-        pass: passToRun,
-        dispatchId: currentDispatchId,
-        triggerReason: t.reason, 
-        resources: t.resources || 'standard'
+        ...t, action: 'RUN_COMPUTATION_DATE', computation: t.name, date: selectedDate, pass: passToRun,
+        dispatchId: currentDispatchId, triggerReason: t.reason, resources: t.resources || 'standard'
     });
 
     const standardTasks = selectedTasks.filter(t => t.resources !== 'high-mem').map(mapToTaskPayload);
@@ -301,15 +401,12 @@ async function getHighMemReroutes(db, date, pass, tasks) {
         const name = normalizeName(task.name);
         const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${name}`;
         const doc = await db.doc(ledgerPath).get();
-        
         if (doc.exists) {
             const data = doc.data();
             const isOOM = (data.status === 'FAILED' || data.status === 'CRASH') && 
                           (data.resourceTier !== 'high-mem') &&
                           ((data.peakMemoryMB > OOM_THRESHOLD_MB) || (data.error && /memory/i.test(data.error.message)));
-            if (isOOM) {
-                reroutes.push({ ...task, resources: 'high-mem' });
-            }
+            if (isOOM) reroutes.push({ ...task, resources: 'high-mem' });
         }
     }
     return reroutes;

package/functions/computation-system/reporter_epoch.js

@@ -3,4 +3,4 @@
  * PURPOSE: Master override for BuildReporter. 
  * Increment this string to force a full re-analysis regardless of code or data stability.
  */
-module.exports = "reporter-epoch-2";
+module.exports = "reporter-epoch-3";

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,5 +1,6 @@
 # Cloud Workflows: Precision Cursor-Based Orchestrator
 # SIMPLE MODE: Dispatch -> Wait ETA -> Next Date
+# UPGRADED: Added "Sweep & Verify" Step for High-Mem Recovery
 
 main:
   params: [input]
@@ -49,7 +50,7 @@ main:
                                 - log_satiation:
                                     call: sys.log
                                     args:
-                                      text: '${"Pass " + pass_id + " - ✅ Pass satiated (0 remaining). Next pass."}'
+                                      text: '${"Pass " + pass_id + " - ✅ Pass satiated (0 remaining). Starting Verification Sweep."}'
                                 - mark_complete:
                                     assign:
                                       - pass_complete: true
@@ -80,5 +81,43 @@ main:
                                         - n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
                                 - next_loop_retry:
                                     next: sequential_date_loop
+            
+            # --- NEW STEP: VERIFICATION & SWEEP ---
+            - verify_pass_completion:
+                call: http.post
+                args:
+                  url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-pass-" + pass_id}'
+                  body:
+                    action: 'VERIFY'
+                    pass: '${pass_id}'
+                    date: '${date_to_run}'
+                  auth: { type: OIDC }
+                result: verify_res
+
+            - process_sweep_tasks:
+                for:
+                  value: sweep_task
+                  in: '${verify_res.body.missingTasks}'
+                  steps:
+                    - log_sweep:
+                        call: sys.log
+                        args:
+                          text: '${"🧹 SWEEP: Disposing " + sweep_task.taskCount + " high-mem tasks for " + sweep_task.date + ". Wait: " + sweep_task.eta + "s"}'
+                    
+                    - dispatch_force_sweep:
+                        call: http.post
+                        args:
+                          url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-pass-" + pass_id}'
+                          body:
+                            action: 'SWEEP'
+                            pass: '${pass_id}'
+                            date: '${sweep_task.date}'
+                          auth: { type: OIDC }
+                    
+                    - wait_sweep_completion:
+                        call: sys.sleep
+                        args:
+                          seconds: '${int(sweep_task.eta)}'
+
     - finish:
-        return: "Pipeline Execution Satiated and Complete"
+        return: "Pipeline Execution Satiated and Verified"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.328",
+  "version": "1.0.330",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [