bulltrackers-module 1.0.299 → 1.0.301

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,7 +1,7 @@
 /**
  * FILENAME: computation-system/helpers/computation_dispatcher.js
  * PURPOSE: "Smart Dispatcher" - Analyzes state, initializes Run Counters, and dispatches tasks.
- * UPDATED: Added per-date logging to visualize progress during large backfills.
+ * UPDATED: Optimized Forensics - Only runs on Retries (Attempt > 1) or Single-Day runs.
  */
 
 const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
@@ -37,7 +37,7 @@ async function checkCrashForensics(db, date, pass, computationName) {
             const lastRSS = data.telemetry.lastMemory.rssMB || 0;
             
             if (lastRSS > OOM_THRESHOLD_MB) {
-                console.log(`[Dispatcher] 🕵️‍♀️ Forensics: ${computationName} likely OOM'd at ${lastRSS}MB. Routing to HIGH-MEM.`);
+                // console.log(`[Dispatcher] 🕵️‍♀️ Forensics: ${computationName} likely OOM'd at ${lastRSS}MB. Routing to HIGH-MEM.`);
                 return 'high-mem';
             }
         }
@@ -59,7 +59,7 @@ async function checkCrashForensics(db, date, pass, computationName) {
  * @param {Object} config - System config (Injected with topics)
  * @param {Object} dependencies - { db, logger, ... }
  * @param {Array} computationManifest - List of calculations
- * @param {Object} reqBody - (Optional) HTTP Body containing 'callbackUrl' and 'date'
+ * @param {Object} reqBody - (Optional) HTTP Body containing 'callbackUrl', 'date', and 'attempt'
  */
 async function dispatchComputationPass(config, dependencies, computationManifest, reqBody = {}) {
     const { logger, db } = dependencies;
@@ -67,9 +67,10 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     const passToRun      = String(config.COMPUTATION_PASS_TO_RUN);
     
     // Extract Date and Callback from request body (pushed by Workflow)
-    // NOTE: 'dateStr' acts as the "Target Date" (Ceiling), usually T-1.
     const dateStr     = reqBody.date || config.date; 
     const callbackUrl = reqBody.callbackUrl || null;
+    // [NEW] Get Attempt Count (Default to 1 if missing)
+    const attemptCount = reqBody.attempt ? parseInt(reqBody.attempt) : 1;
 
     if (!passToRun) { return logger.log('ERROR', '[Dispatcher] No pass defined (COMPUTATION_PASS_TO_RUN). Aborting.'); }
     if (!dateStr)   { return logger.log('ERROR', '[Dispatcher] No date defined. Aborting.'); }
@@ -81,7 +82,7 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
     if (!calcsInThisPass.length) { return logger.log('WARN', `[Dispatcher] No calcs for Pass ${passToRun}. Exiting.`); }
 
-    logger.log('INFO', `🚀 [Dispatcher] Smart-Dispatching PASS ${passToRun} (Target: ${dateStr})`);
+    logger.log('INFO', `🚀 [Dispatcher] Smart-Dispatching PASS ${passToRun} (Target: ${dateStr}) [Attempt ${attemptCount}]`);
     
     // -- DATE ANALYSIS LOGIC (FIXED: RANGE SCAN) --
     
@@ -93,7 +94,7 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     // 2. Generate the full range of dates to check
     let allExpectedDates = getExpectedDateStrings(startDate, endDate);
 
-    // Safety fallback: if range is invalid or empty, default to target date only
+    // Safety fallback
     if (!allExpectedDates || allExpectedDates.length === 0) {
         logger.log('WARN', `[Dispatcher] Date range calculation returned empty (Start: ${startDate.toISOString()} -> End: ${endDate.toISOString()}). Defaulting to single target date.`);
         allExpectedDates = [dateStr];
@@ -104,10 +105,25 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
     const tasksToDispatch = [];
     
-    // Concurrency limit for analysis & forensics (Parallelize the historical scan)
-    const limit = pLimit(20);
+    // [FIX] Separate concurrency limits
+    const dateLimit      = pLimit(20); 
+    const forensicsLimit = pLimit(50); 
+
+    // [NEW] SMART FORENSICS TRIGGER
+    // 1. If scanning > 5 days (Backfill), SKIP (Too expensive).
+    // 2. If attempt == 1 (First Run), SKIP (Assume Standard).
+    // 3. Only run if Attempt > 1 AND Small Batch.
+    const isBulkBackfill = allExpectedDates.length > 5;
+    const shouldRunForensics = (attemptCount > 1) && !isBulkBackfill;
+
+    if (!shouldRunForensics) {
+        if (isBulkBackfill) logger.log('INFO', `[Dispatcher] ⏩ Bulk Backfill (${allExpectedDates.length} days). Skipping Forensics.`);
+        else logger.log('INFO', `[Dispatcher] ⏩ First Attempt. Skipping Forensics (Defaulting to Standard).`);
+    } else {
+        logger.log('WARN', `[Dispatcher] 🕵️‍♀️ Retry Detected (Attempt ${attemptCount}). Enabling Forensic Crash Analysis.`);
+    }
 
-    const analysisPromises = allExpectedDates.map(d => limit(async () => {
+    const analysisPromises = allExpectedDates.map(d => dateLimit(async () => {
         try {
             const fetchPromises = [
                 fetchComputationStatus(d, config, dependencies),
@@ -119,8 +135,6 @@ async function dispatchComputationPass(config, dependencies, computationManifest
                 const prevDate = new Date(d + 'T00:00:00Z');
                 prevDate.setUTCDate(prevDate.getUTCDate() - 1);
                 prevDateStr = prevDate.toISOString().slice(0, 10);
-                
-                // Only fetch previous status if it's within valid range
                 if (prevDate >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
                     fetchPromises.push(fetchComputationStatus(prevDateStr, config, dependencies));
                 }
@@ -137,7 +151,6 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
             const report = analyzeDateExecution(d, calcsInThisPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus);
 
-            // Handle Status Updates (Impossible / Blocked)
             const statusUpdates = {};
             report.impossible.forEach(item => {
                 if (dailyStatus[item.name]?.hash !== STATUS_IMPOSSIBLE) {
@@ -158,12 +171,14 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
             const validToRun = [...report.runnable, ...report.reRuns];
             
-            // [NEW] Parallel Forensics Check
-            await Promise.all(validToRun.map(item => limit(async () => {
+            await Promise.all(validToRun.map(item => forensicsLimit(async () => {
                 const compName = normalizeName(item.name);
                 
-                // 1. Determine Resource Requirements
-                const requiredResource = await checkCrashForensics(db, d, passToRun, compName);
+                // [UPDATED] Conditional Forensics
+                let requiredResource = 'standard';
+                if (shouldRunForensics) {
+                    requiredResource = await checkCrashForensics(db, d, passToRun, compName);
+                }
                 
                 const uniqueDispatchId = crypto.randomUUID(); 
                 tasksToDispatch.push({
@@ -177,12 +192,11 @@ async function dispatchComputationPass(config, dependencies, computationManifest
                     triggerReason: item.reason || "Unknown",
                     dependencyResultHashes: item.dependencyResultHashes || {},
                     timestamp: Date.now(),
-                    resources: requiredResource // 'standard' or 'high-mem'
+                    resources: requiredResource
                 });
             })));
 
-            // [UPDATED] Add Heartbeat Log
-            logger.log('INFO', `[Dispatcher] Analyzed ${d}: ${validToRun.length} tasks identified.`);
+            logger.log('INFO', `[Dispatcher] Analyzed ${d}: ${validToRun.length} tasks (Cumulative: ${tasksToDispatch.length})`);
 
         } catch (e) {
             logger.log('ERROR', `[Dispatcher] Failed analysis for ${d}: ${e.message}`);
@@ -195,14 +209,13 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
     if (tasksToDispatch.length > 0) {
         
-        // 1. Initialize Shared State Document (The Counter)
         const runId = crypto.randomUUID();
         const metaStatePath = `computation_runs/${runId}`;
         
         if (callbackUrl) {
             await db.doc(metaStatePath).set({
                 createdAt: new Date(),
-                date: dateStr, // Acts as the "Job Label" (target date)
+                date: dateStr, 
                 pass: passToRun,
                 totalTasks: tasksToDispatch.length,
                 remainingTasks: tasksToDispatch.length, 
@@ -212,29 +225,23 @@ async function dispatchComputationPass(config, dependencies, computationManifest
             logger.log('INFO', `[Dispatcher] 🏁 Run State Initialized: ${runId}. Tasks: ${tasksToDispatch.length}`);
         }
 
-        // 2. Attach Run Metadata
         tasksToDispatch.forEach(task => {
             task.runId = runId;
             task.metaStatePath = callbackUrl ? metaStatePath : null; 
         });
         
-        // 3. Create Audit Ledger Entries
         const finalDispatched = [];
-        const txnLimit        = pLimit(20);
+        const txnLimit        = pLimit(50);
         
         const txnPromises   = tasksToDispatch.map(task => txnLimit(async () => {
             const ledgerRef = db.collection(`computation_audit_ledger/${task.date}/passes/${task.pass}/tasks`).doc(task.computation);
-            
             try {
                 await db.runTransaction(async (t) => {
                     const doc = await t.get(ledgerRef);
-                    
                     if (doc.exists) {
                         const data = doc.data();
-                        // Strict Idempotency: If completed, don't run again.
                         if (data.status === 'COMPLETED') return false;
                     }
-                    
                     t.set(ledgerRef, {
                         status: 'PENDING',
                         dispatchId: task.dispatchId,
@@ -244,15 +251,12 @@ async function dispatchComputationPass(config, dependencies, computationManifest
                         createdAt: new Date(),
                         dispatcherHash: currentManifestHash,
                         triggerReason: task.triggerReason,
-                        resources: task.resources, // Log intended resource type
+                        resources: task.resources,
                         retries: 0
                     }, { merge: true });
-                    
                     return true;
                 });
-                
                 finalDispatched.push(task);
-
             } catch (txnErr) {
                 logger.log('WARN', `[Dispatcher] Transaction failed for ${task.computation}: ${txnErr.message}`);
             }
@@ -260,13 +264,10 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
         await Promise.all(txnPromises);
 
-        // 4. Publish to Pub/Sub (Segregated by Resources)
         if (finalDispatched.length > 0) {
-            
             const standardTasks = finalDispatched.filter(t => t.resources !== 'high-mem');
             const highMemTasks  = finalDispatched.filter(t => t.resources === 'high-mem');
 
-            // Publish Standard
             if (standardTasks.length > 0) {
                 logger.log('INFO', `[Dispatcher] ✅ Publishing ${standardTasks.length} Standard tasks...`);
                 await pubsubUtils.batchPublishTasks(dependencies, {
@@ -277,7 +278,6 @@ async function dispatchComputationPass(config, dependencies, computationManifest
                 });
             }
 
-            // Publish High-Mem
             if (highMemTasks.length > 0) {
                 logger.log('INFO', `[Dispatcher] 🏋️‍♀️ Publishing ${highMemTasks.length} tasks to HIGH-MEM infrastructure.`);
                 await pubsubUtils.batchPublishTasks(dependencies, {

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,6 +1,6 @@
 # Cloud Workflows Definition for BullTrackers Computation Pipeline
 # Orchestrates 5 sequential passes using Event-Driven Callbacks (Zero Polling).
-# FIXED: Restored 'passes' and 'max_retries' variables in init step.
+# UPDATED: Passes 'attempt' count to Dispatcher to trigger Smart Forensics on retries.
 
 main:
   params: [input]
@@ -16,7 +16,7 @@ main:
           - yesterday_str: ${text.substring(time.format(yesterday_timestamp), 0, 10)}
           - date_to_run: ${default(map.get(input, "date"), yesterday_str)}
 
-          # Configuration Variables (Restored)
+          # Configuration Variables
           - passes: ["1", "2", "3", "4", "5"]
           - max_retries: 3
 
@@ -71,6 +71,7 @@ main:
                             body:
                               date: ${date_to_run}
                               callbackUrl: ${callback_url} 
+                              attempt: ${attempt_count} # [UPDATED] Critical for Smart Forensics logic
                             auth:
                               type: OIDC
                             timeout: 1800 # 30 mins max for dispatch analysis
@@ -96,7 +97,7 @@ main:
                           call: events.await_callback
                           args:
                             callback: ${callback_details}
-                            timeout: 10800 # UPDATED: Reduced from 86400 (24h) to 10800 (3h) to detect crashes faster
+                            timeout: 10800 # Reduced to 3h to fail faster if stalled
                           result: callback_request
 
                       # 5. PROCESS SIGNAL

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.299",
+  "version": "1.0.301",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [