bulltrackers-module 1.0.640 → 1.0.642

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -18,7 +18,28 @@ const crypto = require('crypto');
 const OOM_THRESHOLD_MB = 1500; // Unused
 const BASE_SECONDS_PER_WEIGHT_UNIT = 3;
 const SESSION_CACHE_DURATION_MS = 1000 * 60 * 30; // 30 Minutes
-const STALE_LOCK_THRESHOLD_MS = 1000 * 60 * 15; 
+const STALE_LOCK_THRESHOLD_MS = 1000 * 60 * 15;
+
+// =============================================================================
+// HELPER: Firestore Timestamp Conversion
+// =============================================================================
+/**
+ * Converts a Firestore Timestamp or Date to milliseconds.
+ * Firestore stores Date objects as Timestamp objects, which have a .toDate() method.
+ * This function handles both cases correctly.
+ * @param {any} field - Firestore Timestamp, Date object, or string
+ * @returns {number} Milliseconds since epoch, or 0 if invalid
+ */
+function getMillis(field) {
+    if (!field) return 0;
+    // Handle Firestore Timestamp (has .toDate() method)
+    if (field.toDate && typeof field.toDate === 'function') {
+        return field.toDate().getTime();
+    }
+    // Handle standard Date object or string
+    const date = new Date(field);
+    return isNaN(date.getTime()) ? 0 : date.getTime();
+} 
 
 // =============================================================================
 // HELPER: Schedule Logic
@@ -73,9 +94,7 @@ async function filterActiveTasks(db, date, pass, tasks, logger, forceRun = false
             const isActive = ['PENDING', 'IN_PROGRESS'].includes(data.status);
 
             if (isActive) {
-                const lastActivityTime = data.telemetry?.lastHeartbeat 
-                    ? new Date(data.telemetry.lastHeartbeat).getTime() 
-                    : (data.startedAt ? new Date(data.startedAt).getTime() : 0);
+                const lastActivityTime = getMillis(data.telemetry?.lastHeartbeat) || getMillis(data.startedAt);
                 
                 if ((Date.now() - lastActivityTime) > STALE_LOCK_THRESHOLD_MS) {
                     if (logger) logger.log('WARN', `[Dispatcher] 🧟 Breaking stale lock for ${taskName}.`);
@@ -428,9 +447,7 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
             
            // 1. ACTIVE CHECK: Don't double-dispatch if already running... UNLESS IT'S A ZOMBIE
             if (['PENDING', 'IN_PROGRESS'].includes(data.status)) {
-                const lastActivity = data.telemetry?.lastHeartbeat 
-                    ? new Date(data.telemetry.lastHeartbeat).getTime() 
-                    : (data.startedAt ? new Date(data.startedAt).getTime() : 0);
+                const lastActivity = getMillis(data.telemetry?.lastHeartbeat) || getMillis(data.startedAt);
 
                 // If it's been silent for > 15 mins, it's a Zombie. Kill it and Re-run.
                 if ((Date.now() - lastActivity) > STALE_LOCK_THRESHOLD_MS) {
@@ -566,6 +583,10 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
 
         // Optimization: If nothing is scheduled for today, skip expensive DB checks
         if (scheduledComputations.length === 0) {
+            // DEBUG: Log when schedule filtering removes all tasks
+            if (calcsInThisPass.length > 0) {
+                logger.log('TRACE', `[Dispatcher] Date ${selectedDate}: ${calcsInThisPass.length} pass computations, but 0 scheduled for this date. Skipping.`);
+            }
             currentCursor++;
             continue;
         }
@@ -587,6 +608,11 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
             checkRootDataAvailability(selectedDate, config, dependencies, DEFINITIVE_EARLIEST_DATES)
         ]);
 
+        // DEBUG: Log availability check
+        if (!availability || !availability.status) {
+            logger.log('WARN', `[Dispatcher] ⚠️ Date ${selectedDate}: Availability check failed or returned null. Skipping analysis.`);
+        }
+
         if (availability && availability.status) {
             const report = analyzeDateExecution(
                 selectedDate, 
@@ -597,19 +623,37 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
                 prevDailyStatus
             );
             let rawTasks = [...report.runnable, ...report.reRuns];
+            
+            // DEBUG: Log analysis results
+            if (rawTasks.length === 0 && (report.runnable.length > 0 || report.reRuns.length > 0)) {
+                logger.log('WARN', `[Dispatcher] ⚠️ Date ${selectedDate}: analyzeDateExecution found ${report.runnable.length} runnable + ${report.reRuns.length} reRuns, but rawTasks is empty!`);
+            }
+            if (rawTasks.length > 0) {
+                logger.log('TRACE', `[Dispatcher] Date ${selectedDate}: analyzeDateExecution found ${report.runnable.length} runnable, ${report.reRuns.length} reRuns. Total: ${rawTasks.length}`);
+            }
 
             if (rawTasks.length > 0) {
                 rawTasks = await attemptSimHashResolution(dependencies, selectedDate, rawTasks, dailyStatus, manifestMap);
                 const activeTasks = await filterActiveTasks(db, selectedDate, passToRun, rawTasks, logger);
                 
                 if (activeTasks.length > 0) {
+                    // DEBUG: Log what we're about to route
+                    logger.log('INFO', `[Dispatcher] 🔍 Date ${selectedDate}: ${rawTasks.length} raw tasks → ${activeTasks.length} after filtering. Routing...`);
                     const { standard, highMem } = await splitRoutes(db, selectedDate, passToRun, activeTasks, logger);
                     selectedTasks = [...standard, ...highMem];
                     
+                    // DEBUG: Log routing results
+                    if (selectedTasks.length === 0 && activeTasks.length > 0) {
+                        logger.log('WARN', `[Dispatcher] ⚠️ Date ${selectedDate}: ${activeTasks.length} tasks filtered out by splitRoutes! Tasks: ${activeTasks.map(t => t.name).join(', ')}`);
+                    }
+                    
                     if (selectedTasks.length > 0) {
                         // Found work! Break loop to dispatch.
                         break; 
                     }
+                } else if (rawTasks.length > 0) {
+                    // DEBUG: Log if filterActiveTasks removed all tasks
+                    logger.log('WARN', `[Dispatcher] ⚠️ Date ${selectedDate}: ${rawTasks.length} raw tasks all filtered out by filterActiveTasks! Tasks: ${rawTasks.map(t => t.name).join(', ')}`);
                 }
             }
         }
@@ -734,16 +778,31 @@ async function splitRoutes(db, date, pass, tasks, logger) {
         const doc = await db.doc(ledgerPath).get();
         
         if (!doc.exists) {
+            // No ledger entry - trust analyzeDateExecution, dispatch as standard
             standard.push(task);
             continue;
         }
 
         const data = doc.data();
         
-        // CHECK: Is the task currently running or pending?
+        // CRITICAL FIX: If analyzeDateExecution says this task should run, we MUST trust it.
+        // The ledger might say COMPLETED, but if computation_status is missing/outdated,
+        // we need to re-run to repair the state. Only skip if actively running.
+        // Note: filterActiveTasks already filtered out non-stale PENDING/IN_PROGRESS,
+        // but we double-check here in case of race conditions.
         if (['PENDING', 'IN_PROGRESS'].includes(data.status)) {
-            // It's active, don't dispatch again
-            continue; 
+            // Check if it's stale (should have been caught by filterActiveTasks, but double-check)
+            const lastActivityTime = getMillis(data.telemetry?.lastHeartbeat) || getMillis(data.startedAt);
+            
+            if ((Date.now() - lastActivityTime) > STALE_LOCK_THRESHOLD_MS) {
+                // Stale lock - break it and continue
+                logger.log('WARN', `[Dispatcher] 🧟 splitRoutes: Breaking stale lock for ${name}.`);
+                // Fall through to handle as if no active lock
+            } else {
+                // Valid active lock - skip (shouldn't happen if filterActiveTasks worked correctly)
+                logger.log('TRACE', `[Dispatcher] splitRoutes: Skipping ${name} - Valid IN_PROGRESS (should have been filtered earlier).`);
+                continue;
+            }
         }
         
         if (data.status === 'FAILED') {
@@ -794,8 +853,12 @@ async function splitRoutes(db, date, pass, tasks, logger) {
             });
 
         } else {
-            // Status is likely COMPLETED (but filterActiveTasks should have caught this)
-            // or some other state. Default to standard if we are here.
+            // Status is likely COMPLETED or some other state.
+            // CRITICAL: If analyzeDateExecution says this should run, we MUST trust it.
+            // The ledger might show COMPLETED, but if computation_status is missing/outdated,
+            // we need to re-run to repair the state. This is the "ghost state fix" logic.
+            // Trust the Brain (analyzeDateExecution) over the Log (ledger).
+            logger.log('INFO', `[Dispatcher] 🔄 splitRoutes: ${name} has ledger status '${data.status}', but analyzeDateExecution says it should run. Trusting analysis and dispatching.`);
             standard.push(task);
         }
     }

package/functions/computation-system/tools/BuildReporter.js

@@ -399,35 +399,35 @@ async function generateBuildReport(config, dependencies, manifest) {
                     trueReRuns.forEach(item => processItem(dateSummary.rerun, item, "Logic Changed"));
                     stableUpdates.forEach(item => processItem(dateSummary.stable, item, "Logic Stable"));
 
-                // Auto-Heal Status if Stable
-                if (stableUpdates.length > 0) {
-                    const updatesPayload = {};
-                    for (const stable of stableUpdates) {
-                        const m = manifestMap.get(stable.name);
-                        const stored = dailyStatus[stable.name];
-                        
-                        // [FIX] Only auto-heal if we have valid result data
-                        // Otherwise, force re-run to regenerate everything properly
-                        const hasValidResults = stored?.resultHash && 
-                                            stored.resultHash !== 'empty' &&
-                                            stored.dependencyResultHashes &&
-                                            Object.keys(stored.dependencyResultHashes).length > 0;
-                        
-                        if (m && stored && hasValidResults) {
-                            updatesPayload[stable.name] = {
-                                hash: m.hash, simHash: stable.simHash, resultHash: stored.resultHash, 
-                                dependencyResultHashes: stored.dependencyResultHashes,
-                                category: m.category, composition: m.composition, lastUpdated: new Date()
-                            };
-                        } else {
-                            // No valid results - treat as true re-run
-                            processItem(dateSummary.rerun, stable, "Epoch Change - Rebuilding");
+                    // Auto-Heal Status if Stable
+                    if (stableUpdates.length > 0) {
+                        const updatesPayload = {};
+                        for (const stable of stableUpdates) {
+                            const m = manifestMap.get(stable.name);
+                            const stored = dailyStatus[stable.name];
+                            
+                            // [FIX] Only auto-heal if we have valid result data
+                            // Otherwise, force re-run to regenerate everything properly
+                            const hasValidResults = stored?.resultHash && 
+                                                stored.resultHash !== 'empty' &&
+                                                stored.dependencyResultHashes &&
+                                                Object.keys(stored.dependencyResultHashes).length > 0;
+                            
+                            if (m && stored && hasValidResults) {
+                                updatesPayload[stable.name] = {
+                                    hash: m.hash, simHash: stable.simHash, resultHash: stored.resultHash, 
+                                    dependencyResultHashes: stored.dependencyResultHashes,
+                                    category: m.category, composition: m.composition, lastUpdated: new Date()
+                                };
+                            } else {
+                                // No valid results - treat as true re-run
+                                processItem(dateSummary.rerun, stable, "Epoch Change - Rebuilding");
+                            }
                         }
-                    }
-                    if (Object.keys(updatesPayload).length > 0) {
-                        await updateComputationStatus(dateStr, updatesPayload, config, dependencies);
-                    }
-                }
+                        if (Object.keys(updatesPayload).length > 0) {
+                            await updateComputationStatus(dateStr, updatesPayload, config, dependencies);
+                        }
+                    }}
 
                 // Add skipped items to Stable count for metrics
                 analysis.skipped.forEach(item => processItem(dateSummary.stable, item, "Up To Date"));

package/functions/computation-system/tools/FinalSweepReporter.js

@@ -176,7 +176,16 @@ class FinalSweepReporter {
             forensics.ledgerState = ledgerState;
 
             if (['PENDING', 'IN_PROGRESS'].includes(data.status)) {
-                const lastHb = data.telemetry?.lastHeartbeat ? new Date(data.telemetry.lastHeartbeat).getTime() : 0;
+                // CRITICAL FIX: Handle Firestore Timestamp objects correctly
+                const getMillis = (field) => {
+                    if (!field) return 0;
+                    if (field.toDate && typeof field.toDate === 'function') {
+                        return field.toDate().getTime();
+                    }
+                    const date = new Date(field);
+                    return isNaN(date.getTime()) ? 0 : date.getTime();
+                };
+                const lastHb = getMillis(data.telemetry?.lastHeartbeat) || getMillis(data.startedAt);
                 if (Date.now() - lastHb > STALE_THRESHOLD_MS) {
                     forensics.rootCause = 'ZOMBIE_PROCESS';
                     forensics.reason = `Worker ${data.workerId} stopped heartbeating. Likely crashed/timeout.`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.640",
+  "version": "1.0.642",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [