bulltrackers-module 1.0.808 → 1.0.809

package/functions/computation-system-v3/framework/adapters/StateRepository.js

@@ -31,7 +31,6 @@ class StateRepository {
         const fullResultsTable = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${this.resultsTable}\``;
         const query = `SELECT entity_id, result_data FROM ${fullResultsTable} WHERE date = CAST(@date AS DATE) AND computation_name = @comp AND entity_id IN UNNEST(@ids)`;
         
-        // [FIX] Removed try/catch. If BQ fails, we must fail.
         const [rows] = await this.bigquery.query({ query, params: { date: dateStr, comp: computationName, ids: entityIds }, location: this.config.bigquery.location });
         const resultMap = {};
         for (const row of rows) {
@@ -46,7 +45,6 @@ class StateRepository {
         const fullResultsTable = `\`${this.config.bigquery.projectId}.${this.config.bigquery.dataset}.${this.resultsTable}\``;
         const query = `SELECT result_data FROM ${fullResultsTable} WHERE date = CAST(@date AS DATE) AND computation_name = @comp AND entity_id = '_global' LIMIT 1`;
         
-        // [FIX] Removed try/catch
         const [rows] = await this.bigquery.query({ query, params: { date: dateStr, comp: computationName }, location: this.config.bigquery.location });
         if (rows.length === 0) return null;
         let data = rows[0].result_data;
@@ -55,7 +53,11 @@ class StateRepository {
     }
 
     async getDailyStatus(dateStr) {
-        // [FIX] Removed try/catch. Do not return empty map on error.
+        // [FIX] REMOVED try/catch.
+        // If Firestore is down, we WANT this to throw.
+        // Returning an empty Map causes the Planner to believe ALL history is missing,
+        // triggering catastrophic backfills.
+        
         const snapshot = await this.firestore.collectionGroup('runs')
             .where('date', '==', dateStr)
             .get();
@@ -123,7 +125,7 @@ class StateRepository {
             lastUpdated: new Date(),
             startedAt: metadata.startedAt ? new Date(metadata.startedAt) : new Date(),
             totalBatches: metadata.totalBatches || 0,
-            finalized: false // [FIX] Initialize finalized flag for race-condition prevention
+            finalized: false 
         };
 
         await docRef.set(data, { merge: true });
@@ -155,17 +157,11 @@ class StateRepository {
 
             return { completedBatches: snapshot.data().count };
         } catch (e) {
-            // [FIX] Log the error!
             this.logger.error(`[StateRepo] getCheckpointProgress failed for ${checkpointId}: ${e.message}`);
             return { completedBatches: 0 }; 
         }
     }
 
-    /**
-     * [FIX] Atomic Transaction to Claim Finalization Rights.
-     * Prevents multiple workers from triggering finalization simultaneously.
-     * @returns {Promise<boolean>} true if claim successful, false otherwise.
-     */
     async claimFinalization(checkpointId, requiredTotalBatches) {
         const checkpointRef = this.firestore.collection(this.collections.checkpoints).doc(checkpointId);
 
@@ -176,12 +172,10 @@ class StateRepository {
 
                 const data = doc.data();
                 
-                // 1. Check if already finalized or finalizing
                 if (data.finalized || data.status === 'finalizing') {
                     return false;
                 }
 
-                // 2. Check if actually complete (Double check inside transaction)
                 const batchesSnap = await t.get(
                     this.firestore.collection(this.collections.checkpoints)
                     .doc(checkpointId)
@@ -192,13 +186,12 @@ class StateRepository {
                 const currentCount = batchesSnap.data().count;
 
                 if (currentCount < requiredTotalBatches) {
-                    return false; // Still pending
+                    return false; 
                 }
 
-                // 3. Claim it
                 t.update(checkpointRef, { 
                     status: 'finalizing',
-                    finalized: true, // Mark as claimed
+                    finalized: true, 
                     finalizationStartedAt: new Date()
                 });
                 
@@ -303,7 +296,6 @@ class StateRepository {
     async findZombies(thresholdMinutes) {
         const thresholdDate = new Date(Date.now() - (thresholdMinutes * 60000));
         try {
-            // NOTE: Requires Composite Index on (status, lastUpdated)
             const snapshot = await this.firestore.collection(this.collections.checkpoints)
                 .where('status', '==', 'running')
                 .where('lastUpdated', '<', thresholdDate)
@@ -322,7 +314,6 @@ class StateRepository {
             });
             return zombies;
         } catch (e) {
-            // [FIX] Do NOT fail silently. If the DB is down, we must know.
             this.logger.error(`[StateRepo] CRITICAL: findZombies failed: ${e.message}`);
             throw e; 
         }
@@ -333,7 +324,6 @@ class StateRepository {
         await docRef.set({
             status: 'recovering',
             lastUpdated: new Date(),
-            // Uses atomic increment to track retry count
             attempts: FieldValue.increment(1)
         }, { merge: true });
     }

package/functions/computation-system-v3/framework/core/Coordinator.js

@@ -5,6 +5,7 @@
  * FIX: Passes computationName to BQ Adapter for better logging.
  * FIX: Solved Race Condition in Finalization.
  * FIX: Solved Hardcoded Failure Thresholds.
+ * FIX: Persists 'blocked' status to prevent planner loops.
  */
 
 const { RunAnalyzer } = require('./RunAnalyzer');
@@ -48,6 +49,18 @@ class Coordinator {
 
             if (analysis.status === 'blocked') {
                 this.logger.log(`[Coordinator] Blocked: ${analysis.reason}`);
+                
+                // [FIX] Update state to 'blocked' so Planner knows it's handled.
+                if (!dryRun) {
+                    await this.stateRepo.updateStatus({
+                        computation: entry.name,
+                        date,
+                        status: 'blocked',
+                        hash: entry.hash,
+                        metadata: { blockedReason: analysis.reason }
+                    });
+                }
+
                 return { status: 'blocked', reason: analysis.reason };
             }
 
@@ -122,9 +135,9 @@ class Coordinator {
 
         if (failures.length > 0 && !dryRun && this.stateRepo.logBatchErrors) { await this.stateRepo.logBatchErrors(checkpointId || 'unknown-checkpoint', failures); }
 
-        // [FIX] Configurable Failure Thresholds (Instead of hardcoded 10%)
-        const maxFailureRate = this.config.execution?.maxFailureRate ?? 0.15; // Default 15%
-        const minFailuresForAbort = this.config.execution?.minFailuresForAbort ?? 5; // Minimum failures before aborting
+        // [FIX] Configurable Failure Thresholds
+        const maxFailureRate = this.config.execution?.maxFailureRate ?? 0.15; 
+        const minFailuresForAbort = this.config.execution?.minFailuresForAbort ?? 5;
 
         const failureRate = failures.length / entityIds.length;
         if (entityIds.length > 0 && failures.length >= minFailuresForAbort && failureRate > maxFailureRate) {
@@ -162,14 +175,12 @@ class Coordinator {
 
     async tryFinalizeComputation({ computationName, date, checkpointId, totalBatches }) {
         // [FIX] Use Atomic Claim Transaction instead of Check-Then-Act
-        // This prevents multiple workers from winning the race to finalize
         const claimed = await this.stateRepo.claimFinalization(checkpointId, totalBatches);
         
         if (claimed) {
             return await this.finalizeComputation({ computationName, date, checkpointId });
         }
         
-        // If not claimed, check if it's because it's still pending or already done
         const progress = await this.stateRepo.getCheckpointProgress(checkpointId);
         return { status: 'pending_or_finalized', progress: `${progress.completedBatches}/${totalBatches}` };
     }
@@ -178,14 +189,11 @@ class Coordinator {
         const entry = this.manifestMap.get(computationName.toLowerCase());
         this.logger.log(`[Coordinator] Finalizing ${entry.name} for ${date}...`);
 
-        const start = Date.now(); // Track finalization timing
         const dailyStatus = await this.stateRepo.getDailyStatus(date);
         const previousRun = dailyStatus.get(entry.name);
         
-        // Finalize storage (merge results)
         const { resultHash } = await this.storage.finalizeResults(date, entry) || {};
         
-        // Update State
         await this.stateRepo.updateStatus({
             computation: entry.name,
             date,
@@ -195,14 +203,11 @@ class Coordinator {
             metadata: { checkpointId, finalizedAt: new Date() }
         });
 
-        // [ADD] Discord Alert
         if (this.notifier) {
-            // Calculate total duration if we have metadata, otherwise just use current op
-            const duration = 0; // Simple placeholder, or calculate from checkpoint start time
+            const duration = 0; 
             this.notifier.reportSuccess(entry.name, date, duration, resultHash);
         }
 
-        // Trigger Cascades
         if (resultHash && (!previousRun || resultHash !== previousRun.resultHash)) {
             await this._triggerCascading(entry, date);
         }
@@ -232,7 +237,6 @@ class Coordinator {
         const checkpointId = `${entry.name}-${date}-${Date.now()}`;
         let batchCounter = 0;
 
-        // [FIX] Added hash to task name to prevent collision on re-runs
         const configHash = entry.hash ? entry.hash.substring(0, 8) : 'nohash';
 
         const createBatchTask = (batchIds) => {
@@ -365,7 +369,6 @@ class Coordinator {
         const totalBatches = batchCounter;
 
         if (remainingBatches.length === 0) {
-            // [ADD] Explicit Diagnostic Log
             this.logger.log(`[Coordinator] [Recovery] Diagnosis for ${entry.name}: All ${totalBatches} batches complete. Missing Finalizer. Triggering immediately.`);
             
             await this._triggerTryFinalizeTask(entry, date, checkpointId, 0, totalBatches, true);

package/functions/computation-system-v3/framework/scheduling/TaskScheduler.js

@@ -4,6 +4,8 @@
  * - Allows any pass to be scheduled if Hash Mismatch is detected.
  * - Enforces execution order via ETAs: Pass N runs 10 mins after Pass N-1.
  * - FIX: Enforces UTC date arithmetic.
+ * - FIX: T-1 Scheduling Logic.
+ * - FIX: Handles 'blocked' status properly.
  */
 
 const { ScheduleValidator } = require('./ScheduleValidator');
@@ -22,11 +24,9 @@ class TaskScheduler {
         this.LOOKBACK_DAYS = config.planningLookbackDays ?? 7;
         this.LOOKAHEAD_HOURS = config.planningLookaheadHours ?? 24;
         this.ZOMBIE_THRESHOLD_MINUTES = config.zombieThresholdMinutes ?? 15;
-        
-        // 10 Minute buffer per pass level to ensure upstream dependencies 
-        // have time to finish (or trigger cascades) before downstream runs.
         this.PASS_DELAY_SECONDS = config.passDelaySeconds ?? 600; 
 
+        // [FIX] Default to -1 (Yesterday) to enforce T-1 rule
         this.MAX_DATE_OFFSET = config.scheduling?.maxSchedulingDateOffset ?? -1;
     }
 
@@ -43,6 +43,7 @@ class TaskScheduler {
 
         this.logger.log(`[Planner] Reconciling: ${windowStart.toISOString()} to ${windowEnd.toISOString()}`);
 
+        // [FIX] Use Date Generator that respects MAX_DATE_OFFSET
         const targetDates = this._generateDateRange(windowStart, windowEnd);
         const tasksToSchedule = new Map();
         const stats = { checked: 0, missing: 0, mismatched: 0, scheduled: 0, exists: 0, deleted: 0 };
@@ -52,20 +53,22 @@ class TaskScheduler {
             const dailyStatus = await this.stateRepo.getDailyStatus(dateStr);
 
             for (const entry of this.manifest) {
-                // [UPDATE] Removed "entry.pass !== 1" check. 
-                // We now evaluate ALL passes for potential fixes/backfills.
-
                 const effectiveSchedule = this.validator.parseSchedule(entry.schedule);
                 if (!this.validator.shouldRunOnDate(effectiveSchedule, dateObj)) continue;
 
                 stats.checked++;
                 const statusEntry = dailyStatus instanceof Map ? dailyStatus.get(entry.name) : dailyStatus[entry.name];
+                
                 const lastRunHash = statusEntry?.hash;
                 const rawStatus = statusEntry?.status;
                 const status = rawStatus ? rawStatus.toLowerCase() : null;
 
                 let reason = null;
 
+                // [FIX] Logic to handle 'blocked' status.
+                // If it is 'blocked', we generally treat it as handled (do nothing).
+                // It will only be retried if it is truly missing or failed.
+                
                 if (!statusEntry || status === 'pending') {
                     reason = 'MISSING_RUN';
                     stats.missing++;
@@ -74,7 +77,8 @@ class TaskScheduler {
                     reason = 'RETRY_FAILED';
                     stats.missing++;
                 }
-                else if (lastRunHash !== entry.hash && status !== 'running') {
+                // Check mismatch only if not running and not blocked (blocked means we are waiting for something else)
+                else if (lastRunHash !== entry.hash && status !== 'running' && status !== 'blocked') {
                     reason = 'HASH_MISMATCH';
                     stats.mismatched++;
                 }
@@ -83,17 +87,13 @@ class TaskScheduler {
                     const taskKey = `root-${entry.name}-${dateStr}-${entry.hash}`;
                     
                     if (!tasksToSchedule.has(taskKey)) {
-                        // 1. Calculate Base Window (When it SHOULD run according to schedule)
+                        // 1. Calculate Base Window
                         const baseRunAt = this._getNextRunWindow(effectiveSchedule, dateObj);
                         
                         // 2. Calculate Topological Delay
-                        // If Pass 1 runs at T, Pass 2 runs at T + 10m, Pass 3 at T + 20m.
-                        // This allows Pass 1 to finish and trigger a natural Cascade for Pass 2 
-                        // BEFORE the scheduled Pass 2 task fires (avoiding redundancy/conflicts).
                         const passDelay = (entry.pass - 1) * this.PASS_DELAY_SECONDS;
                         
                         // 3. Determine Final Execution Time
-                        // If baseRunAt is 0 (meaning "ASAP" / window passed), we base delay on NOW.
                         const nowSeconds = Math.floor(Date.now() / 1000);
                         const effectiveBase = baseRunAt === 0 ? nowSeconds : baseRunAt;
                         const runAtSeconds = effectiveBase + passDelay;
@@ -140,7 +140,6 @@ class TaskScheduler {
 
     /**
      * Watchdog: Find and recover zombie tasks.
-     * UPDATED: Now diagnoses the cause of the zombie state.
      */
     async runWatchdog() {
         if (!this.stateRepo.findZombies) {
@@ -155,17 +154,16 @@ class TaskScheduler {
 
         this.logger.log(`[Watchdog] Found ${recoverable.length} zombies.`);
 
-        // [ADD] Diagnostic Phase: Inspect checkpoints to see WHY they are zombies
+        // [ADD] Diagnostic Phase
         const diagnostics = await Promise.all(recoverable.map(async z => {
             try {
-                // Use existing repo methods to peek at state
                 const progress = await this.stateRepo.getCheckpointProgress(z.checkpointId);
                 const checkpoint = await this.stateRepo.loadCheckpoint(z.checkpointId); 
                 
                 const total = checkpoint?.totalBatches || 0;
                 const completed = progress.completedBatches || 0;
                 let reason = 'Unknown Stuck State';
-                let type = 'warn'; // warn vs success
+                let type = 'warn'; 
 
                 if (total > 0 && completed >= total) {
                     reason = '✅ All Batches Done (Finalizer Failed)';
@@ -185,16 +183,25 @@ class TaskScheduler {
             }
         }));
 
-        // Notify Discord with enhanced info
         if (this.notifier) {
             await this.notifier.reportZombies(recoverable.length, diagnostics);
         }
 
-        // Claim zombies
         await Promise.all(recoverable.map(z => this.stateRepo.claimZombie(z.checkpointId)));
 
-        // Re-dispatch logic
         const recoveryTasks = recoverable.map(z => {
+            // [FIX] Enforce T-1 Rule on Zombies too to avoid resurrecting future tasks
+            const zDate = new Date(z.date);
+            const now = new Date();
+            const maxAllowed = new Date(now);
+            maxAllowed.setUTCDate(now.getUTCDate() + this.MAX_DATE_OFFSET);
+            maxAllowed.setUTCHours(23, 59, 59, 999);
+
+            if (zDate > maxAllowed) {
+                this.logger.warn(`[Watchdog] Skipping recovery for ${z.name}@${z.date} (Outside Scheduling Window)`);
+                return null;
+            }
+
             const entry = this.manifestMap.get(z.name); 
             if (!entry) return null;
 
@@ -219,7 +226,7 @@ class TaskScheduler {
         const dates = [];
         let cur = new Date(start);
         
-        // [FIX] Cap the scheduling window to T-1 (Yesterday)
+        // [FIX] Cap the scheduling window based on config (Default T-1)
         const now = new Date();
         const maxAllowed = new Date(now);
         maxAllowed.setUTCDate(now.getUTCDate() + this.MAX_DATE_OFFSET);

package/functions/computation-system-v3/handlers/dispatcher.js

@@ -1,61 +1,73 @@
 /**
- * V3 Dispatcher Handler (FIXED)
- * Routes tasks to the correct controller method.
+ * V3 Dispatcher Handler (FIXED & LOGGING ENHANCED)
+ * Routes tasks and ensures errors are returned to Cloud Tasks logs.
  */
 exports.dispatcher = async (req, res) => {
+    // [DEBUG] Log the incoming trigger to match Cloud Tasks timestamps
+    const taskId = req.get('X-CloudTasks-TaskName') || 'unknown-task';
+    console.log(`[Dispatcher] Received task ${taskId} type=${req.body.type || 'unknown'}`);
+
     try {
         const system = require('../index');
         await system.initialize();
 
-        const { type, computation, date, dryRun, ...params } = req.body;
+        const { type, computation, date, entityIds, ...params } = req.body;
 
-        // 1. Worker Batch
+        // 1. Route Worker Batches
         if (type === 'worker-batch') {
-            if (!params.entityIds) return res.status(400).json({ error: 'Missing entityIds' });
-            return res.status(200).json(await system.coordinatorInstance.processBatch({
-                computationName: computation,
-                date,
-                entityIds: params.entityIds,
-                dryRun,
-                checkpointId: params.checkpointId,
-                batchNum: params.batchNum,
-                totalBatches: params.totalBatches
-            }));
+            if (!entityIds) throw new Error('Missing entityIds');
+            const result = await system.coordinatorInstance.processBatch({
+                computationName: computation, date, entityIds, ...params
+            });
+            return res.status(200).json(result);
         }
 
-        // 2. Finalizer (Crucial Fix)
+        // 2. Route Finalizer
         if (type === 'try-finalize') {
-            return res.status(200).json(await system.coordinatorInstance.tryFinalizeComputation({
-                computationName: computation,
-                date,
-                checkpointId: params.checkpointId,
-                totalBatches: params.totalBatches
-            }));
+            const result = await system.coordinatorInstance.tryFinalizeComputation({
+                computationName: computation, date, ...params
+            });
+            return res.status(200).json(result);
         }
 
-        // 3. Cascade Trigger
+        // 3. Route Cascade
         if (type === 'cascade-trigger') {
-            return res.status(200).json(await system.runComputation({
-                computationName: computation,
-                date,
-                force: params.force || false
-            }));
+            const result = await system.runComputation({
+                computationName: computation, date, force: params.force || false
+            });
+            return res.status(200).json(result);
         }
 
-        // 4. Coordinator (Default Entry Point)
+        // 4. Default: Coordinator (Root Triggers)
         const compName = computation || req.body.computationName;
-        const targetDate = date || req.body.targetDate;
-
-        if (!compName) return res.status(400).json({ error: 'Missing computation name' });
+        if (!compName) throw new Error('Missing computation name');
 
-        return res.status(200).json(await system.runComputation({
+        const result = await system.runComputation({
             computationName: compName,
-            date: targetDate || new Date().toISOString().split('T')[0],
-            dryRun: dryRun || false
-        }));
+            date: date || new Date().toISOString().split('T')[0],
+            dryRun: params.dryRun || false,
+            ...params
+        });
+
+        // [LOGIC FIX] If the result is 'blocked', return 200 (OK) so Cloud Tasks STOPS retrying.
+        // If we return 500, Cloud Tasks will retry forever (or until max attempts).
+        if (result.status === 'blocked') {
+            console.log(`[Dispatcher] Task Blocked (Handling as Success to stop retry): ${result.reason}`);
+            return res.status(200).json(result);
+        }
+
+        return res.status(200).json(result);
 
     } catch (e) {
-        console.error('[V3-Dispatcher] Error:', e);
-        return res.status(500).json({ error: e.message });
+        // [CRITICAL] Log error to Cloud Logging
+        console.error(`[V3-Dispatcher] CRITICAL ERROR on ${taskId}:`, e);
+        
+        // Return 500 with the error message. 
+        // Cloud Tasks 'attemptResponseLog' usually captures the first few bytes of the body.
+        return res.status(500).json({ 
+            error: e.message, 
+            stack: e.stack ? e.stack.split('\n')[0] : 'no-stack',
+            taskId 
+        });
     }
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.808",
+  "version": "1.0.809",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [