bulltrackers-module 1.0.324 → 1.0.326

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,7 +1,8 @@
 /**
  * FILENAME: computation-system/helpers/computation_dispatcher.js
- * PURPOSE: Sequential Cursor-Based Dispatcher with Ledger Awareness, SimHash Stability, and Session Caching.
- * UPDATED: Fixed Ledger Blindness, Cursor Shifting, and Live Analysis Disconnect.
+ * PURPOSE: Sequential Cursor-Based Dispatcher.
+ * BEHAVIOR: Dispatch -> Wait ETA -> Next Date.
+ * UPDATED: Added "Zombie Protocol" to auto-recover stale locks.
  */
 
 const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
@@ -15,10 +16,14 @@ const OOM_THRESHOLD_MB = 1500;
 const BASE_SECONDS_PER_WEIGHT_UNIT = 3;
 const SESSION_CACHE_DURATION_MS = 1000 * 60 * 30; // 30 Minutes
 
+// [NEW] Zombie Timeout: Max Cloud Function run is 9m (540s). 
+// If no heartbeat/start within 15m, it's definitely dead.
+const STALE_LOCK_THRESHOLD_MS = 1000 * 60 * 15; 
+
 // =============================================================================
-// HELPER: Ledger Awareness (Prevents Race Conditions)
+// HELPER: Ledger Awareness (Prevents Race Conditions & Clears Zombies)
 // =============================================================================
-async function filterActiveTasks(db, date, pass, tasks) {
+async function filterActiveTasks(db, date, pass, tasks, logger) {
     if (!tasks || tasks.length === 0) return [];
     
     const checkPromises = tasks.map(async (t) => {
@@ -28,13 +33,37 @@ async function filterActiveTasks(db, date, pass, tasks) {
         
         if (snap.exists) {
             const data = snap.data();
-            // Check PENDING, IN_PROGRESS, or "Ghost" (Completed < 1 min ago)
             const isActive = ['PENDING', 'IN_PROGRESS'].includes(data.status);
+
+            // 1. ZOMBIE CHECK (Recover Stale Locks)
+            if (isActive) {
+                // Prefer heartbeat, fall back to start time
+                const lastActivityTime = data.telemetry?.lastHeartbeat 
+                    ? new Date(data.telemetry.lastHeartbeat).getTime() 
+                    : (data.startedAt ? new Date(data.startedAt).getTime() : 0);
+                
+                const timeSinceActive = Date.now() - lastActivityTime;
+
+                if (timeSinceActive > STALE_LOCK_THRESHOLD_MS) {
+                    if (logger) {
+                        logger.log('WARN', `[Dispatcher] 🧟 Breaking stale lock for ${taskName}. Inactive for ${(timeSinceActive/60000).toFixed(1)} mins.`);
+                    }
+                    // Return task (Re-dispatching it will overwrite the old lock in Firestore)
+                    return t; 
+                }
+                
+                // If distinct and recent, filter it out (let it run)
+                return null; 
+            }
+
+            // 2. GHOST CHECK (Debounce immediate re-runs)
+            // If it finished less than 1 minute ago, don't re-dispatch immediately 
+            // (prevents double-tap if latency is high)
             const isJustFinished = data.status === 'COMPLETED' && 
                                    data.completedAt && 
                                    (Date.now() - new Date(data.completedAt).getTime() < 60 * 1000);
             
-            if (isActive || isJustFinished) return null;
+            if (isJustFinished) return null; // Filter out
         }
         return t;
     });
@@ -50,18 +79,13 @@ async function attemptSimHashResolution(dependencies, date, tasks, dailyStatus,
     const { db, logger } = dependencies;
     const resolvedTasks = [];
     const remainingTasks = [];
-
-    // Cache for SimHashes to avoid redundant DB lookups in loop
     const simHashCache = new Map();
 
     for (const task of tasks) {
-        // Only apply to Re-Runs (Hash Mismatches), not fresh runs (Missing Data)
         const currentStatus = dailyStatus ? dailyStatus[task.name] : null;
         const manifestItem = manifestMap.get(normalizeName(task.name));
 
         if (currentStatus && currentStatus.simHash && manifestItem) {
-            
-            // 1. Get the SimHash for the NEW code (from Registry)
             let newSimHash = simHashCache.get(manifestItem.hash);
             if (!newSimHash) {
                 const simDoc = await db.collection('system_simhash_registry').doc(manifestItem.hash).get();
@@ -71,13 +95,12 @@ async function attemptSimHashResolution(dependencies, date, tasks, dailyStatus,
                 }
             }
 
-            // 2. Compare
             if (newSimHash && newSimHash === currentStatus.simHash) {
                 resolvedTasks.push({
                     name: task.name,
                     hash: manifestItem.hash,
                     simHash: newSimHash,
-                    prevStatus: currentStatus // Pass previous status to preserve other fields
+                    prevStatus: currentStatus
                 });
                 continue; 
             }
@@ -85,30 +108,24 @@ async function attemptSimHashResolution(dependencies, date, tasks, dailyStatus,
         remainingTasks.push(task);
     }
 
-    // 3. Apply Updates for Stable Tasks
     if (resolvedTasks.length > 0) {
         const updatePayload = {};
-        
         resolvedTasks.forEach(t => {
-            // [FIXED] Construct full nested object to avoid dot-notation issues with .set()
-            // We merge existing data (like resultHash) so we don't lose the valid calculation output
             updatePayload[t.name] = {
-                ...(t.prevStatus || {}), // Keep existing resultHash, output, etc.
-                hash: t.hash,            // Update to new code hash
-                simHash: t.simHash,      // Confirmed stable simHash
+                ...(t.prevStatus || {}),
+                hash: t.hash,
+                simHash: t.simHash,
                 reason: 'SimHash Stable (Auto-Resolved)',
                 lastUpdated: new Date().toISOString()
             };
         });
-        
-        // Use set with merge: true. Now that keys are "clean" (no dots), 
-        // objects will merge correctly into the document structure.
         await db.collection('computation_status').doc(date).set(updatePayload, { merge: true });
         logger.log('INFO', `[SimHash] ⏩ Fast-forwarded ${resolvedTasks.length} tasks for ${date} (Logic Unchanged).`);
     }
 
     return remainingTasks;
 }
+
 // =============================================================================
 // HELPER: Stable Session Management (Solves Cursor Shifting)
 // =============================================================================
@@ -117,36 +134,21 @@ async function getStableDateSession(config, dependencies, passToRun, dateLimitSt
     const sessionId = `pass_${passToRun}_${dateLimitStr.replace(/-/g, '')}`;
     const sessionRef = db.collection('dispatcher_sessions').doc(sessionId);
 
-    // 1. Try to Load Session
     if (!forceRebuild) {
         const sessionSnap = await sessionRef.get();
         if (sessionSnap.exists) {
             const data = sessionSnap.data();
-            const age = Date.now() - new Date(data.createdAt).getTime();
-            if (age < SESSION_CACHE_DURATION_MS) {
-                logger.log('INFO', `[Session] 📂 Loaded stable session for Pass ${passToRun} (${data.dates.length} dates).`);
+            if ((Date.now() - new Date(data.createdAt).getTime()) < SESSION_CACHE_DURATION_MS) {
                 return data.dates;
             }
         }
     }
 
-    // 2. Rebuild Session (Expensive Scan)
     logger.log('INFO', `[Session] 🔄 Rebuilding dispatch session for Pass ${passToRun}...`);
     const earliestDates = await getEarliestDataDates(config, dependencies);
     const allDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date(dateLimitStr + 'T00:00:00Z'));
     
-    // We only want dates that *might* be dirty.
-    // Optimization: We add ALL dates to the list. The dispatcher checks them individually.
-    // Why? Because if we pre-filter here, we repeat the work of the dispatcher.
-    // Better: Store the plain list of dates sorted descending (newest first usually better for backfills, ascending for standard).
-    // Let's stick to Ascending (oldest first) as standard.
-    
-    await sessionRef.set({
-        dates: allDates,
-        createdAt: new Date().toISOString(),
-        configHash: dateLimitStr // Simple versioning
-    });
-
+    await sessionRef.set({ dates: allDates, createdAt: new Date().toISOString(), configHash: dateLimitStr });
     return allDates;
 }
 
@@ -157,9 +159,9 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     const { logger, db } = dependencies;
     const pubsubUtils    = new PubSubUtils(dependencies);
     
-    const passToRun     = String(reqBody.pass || config.COMPUTATION_PASS_TO_RUN || "1");
+    const passToRun     = String(reqBody.pass || "1");
     const targetCursorN = parseInt(reqBody.cursorIndex || 1);
-    const dateLimitStr  = reqBody.date || config.date || "2025-01-01";
+    const dateLimitStr  = reqBody.date || "2025-01-01";
     const forceRebuild  = reqBody.forceRebuild === true;
 
     const manifestMap     = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
@@ -167,38 +169,26 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     const calcsInThisPass = passes[passToRun] || [];
     const manifestWeightMap = new Map(computationManifest.map(c => [normalizeName(c.name), c.weight || 1.0]));
 
-    if (!calcsInThisPass.length) {
-        return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
-    }
+    if (!calcsInThisPass.length) return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
 
-    // 1. Get Stable Date List (Solves Shifting Cursor)
+    // 1. Get Stable Date List
     const sessionDates = await getStableDateSession(config, dependencies, passToRun, dateLimitStr, forceRebuild);
-    
-    if (!sessionDates || sessionDates.length === 0) {
-        return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
-    }
+    if (!sessionDates || sessionDates.length === 0) return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
 
-    // 2. Select Date based on Cursor
+    // 2. Select Date
     let selectedDate = null;
     let selectedTasks = [];
-    let isReroute = false;
-    let isSweep = false;
 
-    // Check bounds
     if (targetCursorN <= sessionDates.length) {
-        // Normal Operation
         selectedDate = sessionDates[targetCursorN - 1];
     } else {
-        // End of list
         return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
     }
 
-    // 3. Analyze SPECIFIC Date (Live Analysis)
-    // We only fetch status for the ONE date we are looking at + context
+    // 3. Analyze SPECIFIC Date
     if (selectedDate) {
-        // A. Fetch Context
-        const needsHistory = calcsInThisPass.some(c => c.isHistorical);
         const earliestDates = await getEarliestDataDates(config, dependencies);
+        const needsHistory = calcsInThisPass.some(c => c.isHistorical);
         
         let prevDailyStatusPromise = Promise.resolve(null);
         if (needsHistory) {
@@ -219,51 +209,41 @@ async function dispatchComputationPass(config, dependencies, computationManifest
             const report = analyzeDateExecution(selectedDate, calcsInThisPass, availability.status, dailyStatus, manifestMap, prevDailyStatus);
             let rawTasks = [...report.runnable, ...report.reRuns];
 
-            // B. Apply SimHash Resolution (Problem #1)
             if (rawTasks.length > 0) {
                 rawTasks = await attemptSimHashResolution(dependencies, selectedDate, rawTasks, dailyStatus, manifestMap);
-            }
-
-            // C. Apply Ledger Filter (Problem #2)
-            if (rawTasks.length > 0) {
-                selectedTasks = await filterActiveTasks(db, selectedDate, passToRun, rawTasks);
+                
+                // [UPDATED] Pass logger to filterActiveTasks for zombie warnings
+                selectedTasks = await filterActiveTasks(db, selectedDate, passToRun, rawTasks, logger);
             }
             
-            // D. Check for High-Mem Reroutes (OOM handling)
+            // OOM / High-Mem Reroute Check
             if (selectedTasks.length > 0) {
                 const reroutes = await getHighMemReroutes(db, selectedDate, passToRun, selectedTasks);
                 if (reroutes.length > 0) {
                     selectedTasks = reroutes;
-                    isReroute = true;
                 }
             }
-        } else {
-            logger.log('WARN', `[Dispatcher] Date ${selectedDate} skipped (Data Unavailable).`);
         }
     }
 
     // 4. Dispatch Logic
     if (selectedTasks.length === 0) {
-        // Nothing to do for this date.
-        // CRITICAL: We return dispatched: 0, but n_cursor_ignored: FALSE.
-        // This tells workflow to increment cursor and check the next date in the Stable Session.
         return {
             status: 'CONTINUE_PASS',
             dateProcessed: selectedDate,
             dispatched: 0,
-            n_cursor_ignored: false, // Proceed to next date
+            n_cursor_ignored: false,
             etaSeconds: 0,
             remainingDates: sessionDates.length - targetCursorN
         };
     }
 
-    // 5. Publish Tasks
+    // 5. Send Tasks
     const totalweight = selectedTasks.reduce((sum, t) => sum + (manifestWeightMap.get(normalizeName(t.name)) || 1.0), 0);
     const currentDispatchId = crypto.randomUUID();
     const etaSeconds = Math.max(20, Math.ceil(totalweight * BASE_SECONDS_PER_WEIGHT_UNIT));
 
     const taskDetails = selectedTasks.map(t => `${t.name} (${t.reason})`);
-
     logger.log('INFO', `[Dispatcher] ✅ Dispatching ${selectedTasks.length} tasks for ${selectedDate}.`, {
         date: selectedDate,
         pass: passToRun,
@@ -305,13 +285,11 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     }
     await Promise.all(pubPromises);
 
-    // CRITICAL: We dispatched work. We want to check THIS date again next time 
-    // to ensure tasks completed. So we IGNORE cursor increment.
     return {
         status: 'CONTINUE_PASS',
         dateProcessed: selectedDate,
         dispatched: selectedTasks.length,
-        n_cursor_ignored: true, // Hold cursor until this date is clean
+        n_cursor_ignored: false,
         etaSeconds: etaSeconds,
         remainingDates: sessionDates.length - targetCursorN
     };

package/functions/computation-system/tools/BuildReporter.js

@@ -2,6 +2,7 @@
  * FILENAME: bulltrackers-module/functions/computation-system/tools/BuildReporter.js
  * UPGRADED: Offloads heavy logic to a dedicated Cloud Function via Pub/Sub.
  * FEATURES: Patch versioning, data-drift detection (window changes), and checkpointed writes.
+ * FIX: Ensures ALL dates in the window are reported, even if analysis fails.
  */
 
 const { analyzeDateExecution } = require('../WorkflowOrchestrator');
@@ -62,7 +63,7 @@ async function handleBuildReportTrigger(message, context, config, dependencies,
 function getSystemFingerprint(manifest) {
     const sortedManifestHashes = manifest.map(c => c.hash).sort().join('|');
     return crypto.createHash('sha256')
-        .update(sortedManifestHashes + SYSTEM_EPOCH + REPORTER_EPOCH) // [UPDATED]
+        .update(sortedManifestHashes + SYSTEM_EPOCH + REPORTER_EPOCH) 
         .digest('hex');
 }
 
@@ -187,20 +188,20 @@ async function generateBuildReport(config, dependencies, manifest) {
     const lastEarliestStr = latest?.windowEarliest || 'NONE';
     const windowChanged = currentEarliestStr !== lastEarliestStr;
 
-    const epochChanged = latest?.reporterEpoch !== REPORTER_EPOCH; // [NEW]
+    const epochChanged = latest?.reporterEpoch !== REPORTER_EPOCH; 
 
     // If fingerprints match AND the window is the same, we can truly skip.
     if (latest && 
         latest.systemFingerprint === currentFingerprint && 
         !windowChanged && 
-        !epochChanged) { // [NEW]
+        !epochChanged) { 
         logger.log('INFO', `[BuildReporter] ⚡ System fingerprint, window, and reporter epoch stable. Skipping report.`);
         return { success: true, status: 'SKIPPED_IDENTICAL' };
     }
 
     // Determine primary reason for logging
     let reason = 'Code Change';
-    if (epochChanged) reason = 'Master Epoch Override'; // [NEW]
+    if (epochChanged) reason = 'Master Epoch Override'; 
     else if (windowChanged) reason = 'Data Window Drift';
 
     // Increment patch version
@@ -251,7 +252,7 @@ async function generateBuildReport(config, dependencies, manifest) {
     // Initialize the build record
     await db.collection(BUILD_RECORDS_COLLECTION).doc(buildId).set(reportHeader);
 
-    let totalRun = 0, totalReRun = 0, totalStable = 0;
+    let totalRun = 0, totalReRun = 0, totalStable = 0, totalErrors = 0;
     const limit = pLimit(10); // Concurrency for fetching statuses
 
     // Process dates in chunks of 5 for checkpointed writing
@@ -336,17 +337,33 @@ async function generateBuildReport(config, dependencies, manifest) {
                 // Write detailed date record
                 await db.collection(BUILD_RECORDS_COLLECTION).doc(buildId).collection('details').doc(dateStr).set(dateSummary);
 
-                return { run: dateSummary.run.length, rerun: dateSummary.rerun.length, stable: dateSummary.stable.length };
+                return { run: dateSummary.run.length, rerun: dateSummary.rerun.length, stable: dateSummary.stable.length, error: false };
             } catch (err) {
                 logger.log('ERROR', `[BuildReporter] Analysis failed for ${dateStr}: ${err.message}`);
-                return { run: 0, rerun: 0, stable: 0 };
+                
+                // [FIX] Write error record so the date appears in the report
+                await db.collection(BUILD_RECORDS_COLLECTION).doc(buildId).collection('details').doc(dateStr).set({
+                    error: err.message,
+                    status: 'ANALYSIS_FAILED',
+                    meta: { totalIncluded: 0, totalExpected: 0, match: false }
+                }).catch(e => logger.log('ERROR', `Failed to write error record for ${dateStr}: ${e.message}`));
+                
+                return { run: 0, rerun: 0, stable: 0, error: true };
             }
         })));
 
         // Accumulate stats and write a progress checkpoint
-        results.forEach(res => { totalRun += res.run; totalReRun += res.rerun; totalStable += res.stable; });
+        results.forEach(res => { 
+            if (res.error) totalErrors++;
+            else {
+                totalRun += res.run; 
+                totalReRun += res.rerun; 
+                totalStable += res.stable; 
+            }
+        });
+        
         await db.collection(BUILD_RECORDS_COLLECTION).doc(buildId).update({ 
-            checkpoint: `Processed ${i + dateBatch.length}/${datesToCheck.length} dates` 
+            checkpoint: `Processed ${Math.min(i + dateBatch.length, datesToCheck.length)}/${datesToCheck.length} dates` 
         });
     }
 
@@ -355,6 +372,7 @@ async function generateBuildReport(config, dependencies, manifest) {
         totalReRuns: totalReRun, 
         totalNew: totalRun, 
         totalStable: totalStable, 
+        totalErrors: totalErrors,
         scanRange: `${datesToCheck[0]} to ${datesToCheck[datesToCheck.length-1]}` 
     };
 
@@ -362,7 +380,7 @@ async function generateBuildReport(config, dependencies, manifest) {
     await db.collection(BUILD_RECORDS_COLLECTION).doc(buildId).set(reportHeader);
     await db.collection(BUILD_RECORDS_COLLECTION).doc('latest').set({ ...reportHeader, note: "Latest completed build report." });
     
-    logger.log('SUCCESS', `[BuildReporter] Build ${buildId} completed. Re-runs: ${totalReRun}, Stable: ${totalStable}, New: ${totalRun}.`);
+    logger.log('SUCCESS', `[BuildReporter] Build ${buildId} completed. Re-runs: ${totalReRun}, Stable: ${totalStable}, New: ${totalRun}, Errors: ${totalErrors}.`);
     
     return { success: true, buildId, summary: reportHeader.summary };
 }

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,5 +1,5 @@
 # Cloud Workflows: Precision Cursor-Based Orchestrator
-# UPDATED: Added satiation detection to break early on 0 remaining dates.
+# SIMPLE MODE: Dispatch -> Wait ETA -> Next Date
 
 main:
   params: [input]
@@ -20,7 +20,6 @@ main:
                 assign:
                   - n_cursor: 1
                   - pass_complete: false
-                  - consecutive_empty_dispatches: 0
 
             - sequential_date_loop:
                 switch:
@@ -39,26 +38,25 @@ main:
 
                       - evaluate_dispatch:
                           switch:
+                            # 1. End of Session (Dispatcher reached end of date list)
                             - condition: '${dispatch_res.body.status == "MOVE_TO_NEXT_PASS"}'
                               assign:
                                 - pass_complete: true
                             
-                            # NEW: Explicit Satiation Check
-                            - condition: '${dispatch_res.body.status == "CONTINUE_PASS" and dispatch_res.body.remainingDates == 0}'
+                            # 2. Satiation Check (Specific to date/logic)
+                            - condition: '${dispatch_res.body.status == "CONTINUE_PASS" and dispatch_res.body.remainingDates == 0 and dispatch_res.body.dispatched == 0}'
                               steps:
                                 - log_satiation:
                                     call: sys.log
                                     args:
-                                      text: '${"Pass " + pass_id + " - ✅ Pass satiated (0 remaining dates). Moving to next pass."}'
+                                      text: '${"Pass " + pass_id + " - ✅ Pass satiated (0 remaining). Next pass."}'
                                 - mark_complete:
                                     assign:
                                       - pass_complete: true
 
+                            # 3. Work Dispatched: Wait ETA -> Move Next (Ignored flag is FALSE)
                             - condition: '${dispatch_res.body.dispatched > 0}'
                               steps:
-                                - reset_retry_counter:
-                                    assign:
-                                      - consecutive_empty_dispatches: 0
                                 - wait_for_completion:
                                     call: sys.sleep
                                     args:
@@ -69,26 +67,18 @@ main:
                                 - next_loop_work:
                                     next: sequential_date_loop
 
+                            # 4. No Work (Clean or Busy): Move Next Immediately
                             - condition: '${dispatch_res.body.dispatched == 0}'
                               steps:
-                                - increment_retry:
+                                - wait_short:
+                                    call: sys.sleep
+                                    args:
+                                      seconds: 2 # Tiny debounce
+                                - update_cursor_retry:
                                     assign:
-                                      - consecutive_empty_dispatches: '${consecutive_empty_dispatches + 1}'
-                                - check_break_condition:
-                                    switch:
-                                      - condition: '${consecutive_empty_dispatches >= 3}'
-                                        assign:
-                                          - pass_complete: true
-                                      - condition: '${true}'
-                                        steps:
-                                          - wait_short:
-                                              call: sys.sleep
-                                              args:
-                                                seconds: 5
-                                          - update_cursor_retry:
-                                              assign:
-                                                 - n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
-                                          - next_loop_retry:
-                                              next: sequential_date_loop
+                                        # Dispatcher sends n_cursor_ignored=false, so we increment.
+                                        - n_cursor: '${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}'
+                                - next_loop_retry:
+                                    next: sequential_date_loop
     - finish:
         return: "Pipeline Execution Satiated and Complete"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.324",
+  "version": "1.0.326",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [