bulltrackers-module 1.0.300 → 1.0.302

package/functions/computation-system/executors/StandardExecutor.js

@@ -114,7 +114,7 @@ class StandardExecutor {
 
                 usersSinceLastFlush += chunkSize;
                 const heapStats = v8.getHeapStatistics();
-                if (usersSinceLastFlush >= 5000 || (heapStats.used_heap_size / heapStats.heap_size_limit) > 0.70) {
+                if (usersSinceLastFlush >= 500 || (heapStats.used_heap_size / heapStats.heap_size_limit) > 0.70) {
                     const flushResult = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, 'INTERMEDIATE', true, !hasFlushed);
                     hasFlushed = true;
                     StandardExecutor.mergeReports(aggregatedSuccess, aggregatedFailures, flushResult);

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,7 +1,7 @@
 /**
  * FILENAME: computation-system/helpers/computation_dispatcher.js
  * PURPOSE: "Smart Dispatcher" - Analyzes state, initializes Run Counters, and dispatches tasks.
- * UPDATED: FIXED DEADLOCK by separating concurrency limits for Dates vs Tasks.
+ * UPDATED: Optimized Forensics - Only runs on Retries (Attempt > 1) or Single-Day runs.
  */
 
 const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
@@ -59,7 +59,7 @@ async function checkCrashForensics(db, date, pass, computationName) {
  * @param {Object} config - System config (Injected with topics)
  * @param {Object} dependencies - { db, logger, ... }
  * @param {Array} computationManifest - List of calculations
- * @param {Object} reqBody - (Optional) HTTP Body containing 'callbackUrl' and 'date'
+ * @param {Object} reqBody - (Optional) HTTP Body containing 'callbackUrl', 'date', and 'attempt'
  */
 async function dispatchComputationPass(config, dependencies, computationManifest, reqBody = {}) {
     const { logger, db } = dependencies;
@@ -67,9 +67,10 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     const passToRun      = String(config.COMPUTATION_PASS_TO_RUN);
     
     // Extract Date and Callback from request body (pushed by Workflow)
-    // NOTE: 'dateStr' acts as the "Target Date" (Ceiling), usually T-1.
     const dateStr     = reqBody.date || config.date; 
     const callbackUrl = reqBody.callbackUrl || null;
+    // [NEW] Get Attempt Count (Default to 1 if missing)
+    const attemptCount = reqBody.attempt ? parseInt(reqBody.attempt) : 1;
 
     if (!passToRun) { return logger.log('ERROR', '[Dispatcher] No pass defined (COMPUTATION_PASS_TO_RUN). Aborting.'); }
     if (!dateStr)   { return logger.log('ERROR', '[Dispatcher] No date defined. Aborting.'); }
@@ -81,7 +82,7 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
     if (!calcsInThisPass.length) { return logger.log('WARN', `[Dispatcher] No calcs for Pass ${passToRun}. Exiting.`); }
 
-    logger.log('INFO', `🚀 [Dispatcher] Smart-Dispatching PASS ${passToRun} (Target: ${dateStr})`);
+    logger.log('INFO', `🚀 [Dispatcher] Smart-Dispatching PASS ${passToRun} (Target: ${dateStr}) [Attempt ${attemptCount}]`);
     
     // -- DATE ANALYSIS LOGIC (FIXED: RANGE SCAN) --
     
@@ -93,7 +94,7 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     // 2. Generate the full range of dates to check
     let allExpectedDates = getExpectedDateStrings(startDate, endDate);
 
-    // Safety fallback: if range is invalid or empty, default to target date only
+    // Safety fallback
     if (!allExpectedDates || allExpectedDates.length === 0) {
         logger.log('WARN', `[Dispatcher] Date range calculation returned empty (Start: ${startDate.toISOString()} -> End: ${endDate.toISOString()}). Defaulting to single target date.`);
         allExpectedDates = [dateStr];
@@ -104,9 +105,23 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
     const tasksToDispatch = [];
     
-    // [FIX] Separate concurrency limits to prevent DEADLOCK
-    const dateLimit      = pLimit(20); // Parallel Days
-    const forensicsLimit = pLimit(50); // Parallel Checks per day
+    // [FIX] Separate concurrency limits
+    const dateLimit      = pLimit(20); 
+    const forensicsLimit = pLimit(50); 
+
+    // [NEW] SMART FORENSICS TRIGGER
+    // 1. If scanning > 5 days (Backfill), SKIP (Too expensive).
+    // 2. If attempt == 1 (First Run), SKIP (Assume Standard).
+    // 3. Only run if Attempt > 1 AND Small Batch.
+    const isBulkBackfill = allExpectedDates.length > 5;
+    const shouldRunForensics = (attemptCount > 1) && !isBulkBackfill;
+
+    if (!shouldRunForensics) {
+        if (isBulkBackfill) logger.log('INFO', `[Dispatcher] ⏩ Bulk Backfill (${allExpectedDates.length} days). Skipping Forensics.`);
+        else logger.log('INFO', `[Dispatcher] ⏩ First Attempt. Skipping Forensics (Defaulting to Standard).`);
+    } else {
+        logger.log('WARN', `[Dispatcher] 🕵️‍♀️ Retry Detected (Attempt ${attemptCount}). Enabling Forensic Crash Analysis.`);
+    }
 
     const analysisPromises = allExpectedDates.map(d => dateLimit(async () => {
         try {
@@ -120,8 +135,6 @@ async function dispatchComputationPass(config, dependencies, computationManifest
                 const prevDate = new Date(d + 'T00:00:00Z');
                 prevDate.setUTCDate(prevDate.getUTCDate() - 1);
                 prevDateStr = prevDate.toISOString().slice(0, 10);
-                
-                // Only fetch previous status if it's within valid range
                 if (prevDate >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
                     fetchPromises.push(fetchComputationStatus(prevDateStr, config, dependencies));
                 }
@@ -138,7 +151,6 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
             const report = analyzeDateExecution(d, calcsInThisPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus);
 
-            // Handle Status Updates (Impossible / Blocked)
             const statusUpdates = {};
             report.impossible.forEach(item => {
                 if (dailyStatus[item.name]?.hash !== STATUS_IMPOSSIBLE) {
@@ -159,12 +171,14 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
             const validToRun = [...report.runnable, ...report.reRuns];
             
-            // [FIX] Use separate 'forensicsLimit' here to avoid deadlock with 'dateLimit'
             await Promise.all(validToRun.map(item => forensicsLimit(async () => {
                 const compName = normalizeName(item.name);
                 
-                // 1. Determine Resource Requirements
-                const requiredResource = await checkCrashForensics(db, d, passToRun, compName);
+                // [UPDATED] Conditional Forensics
+                let requiredResource = 'standard';
+                if (shouldRunForensics) {
+                    requiredResource = await checkCrashForensics(db, d, passToRun, compName);
+                }
                 
                 const uniqueDispatchId = crypto.randomUUID(); 
                 tasksToDispatch.push({
@@ -178,11 +192,10 @@ async function dispatchComputationPass(config, dependencies, computationManifest
                     triggerReason: item.reason || "Unknown",
                     dependencyResultHashes: item.dependencyResultHashes || {},
                     timestamp: Date.now(),
-                    resources: requiredResource // 'standard' or 'high-mem'
+                    resources: requiredResource
                 });
             })));
 
-            // [PROGRESS LOG] This should now fire correctly
             logger.log('INFO', `[Dispatcher] Analyzed ${d}: ${validToRun.length} tasks (Cumulative: ${tasksToDispatch.length})`);
 
         } catch (e) {
@@ -196,14 +209,13 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
     if (tasksToDispatch.length > 0) {
         
-        // 1. Initialize Shared State Document (The Counter)
         const runId = crypto.randomUUID();
         const metaStatePath = `computation_runs/${runId}`;
         
         if (callbackUrl) {
             await db.doc(metaStatePath).set({
                 createdAt: new Date(),
-                date: dateStr, // Acts as the "Job Label" (target date)
+                date: dateStr, 
                 pass: passToRun,
                 totalTasks: tasksToDispatch.length,
                 remainingTasks: tasksToDispatch.length, 
@@ -213,29 +225,23 @@ async function dispatchComputationPass(config, dependencies, computationManifest
             logger.log('INFO', `[Dispatcher] 🏁 Run State Initialized: ${runId}. Tasks: ${tasksToDispatch.length}`);
         }
 
-        // 2. Attach Run Metadata
         tasksToDispatch.forEach(task => {
             task.runId = runId;
             task.metaStatePath = callbackUrl ? metaStatePath : null; 
         });
         
-        // 3. Create Audit Ledger Entries
         const finalDispatched = [];
-        const txnLimit        = pLimit(50); // Increased txn throughput
+        const txnLimit        = pLimit(50);
         
         const txnPromises   = tasksToDispatch.map(task => txnLimit(async () => {
             const ledgerRef = db.collection(`computation_audit_ledger/${task.date}/passes/${task.pass}/tasks`).doc(task.computation);
-            
             try {
                 await db.runTransaction(async (t) => {
                     const doc = await t.get(ledgerRef);
-                    
                     if (doc.exists) {
                         const data = doc.data();
-                        // Strict Idempotency: If completed, don't run again.
                         if (data.status === 'COMPLETED') return false;
                     }
-                    
                     t.set(ledgerRef, {
                         status: 'PENDING',
                         dispatchId: task.dispatchId,
@@ -245,15 +251,12 @@ async function dispatchComputationPass(config, dependencies, computationManifest
                         createdAt: new Date(),
                         dispatcherHash: currentManifestHash,
                         triggerReason: task.triggerReason,
-                        resources: task.resources, // Log intended resource type
+                        resources: task.resources,
                         retries: 0
                     }, { merge: true });
-                    
                     return true;
                 });
-                
                 finalDispatched.push(task);
-
             } catch (txnErr) {
                 logger.log('WARN', `[Dispatcher] Transaction failed for ${task.computation}: ${txnErr.message}`);
             }
@@ -261,13 +264,10 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
         await Promise.all(txnPromises);
 
-        // 4. Publish to Pub/Sub (Segregated by Resources)
         if (finalDispatched.length > 0) {
-            
             const standardTasks = finalDispatched.filter(t => t.resources !== 'high-mem');
             const highMemTasks  = finalDispatched.filter(t => t.resources === 'high-mem');
 
-            // Publish Standard
             if (standardTasks.length > 0) {
                 logger.log('INFO', `[Dispatcher] ✅ Publishing ${standardTasks.length} Standard tasks...`);
                 await pubsubUtils.batchPublishTasks(dependencies, {
@@ -278,7 +278,6 @@ async function dispatchComputationPass(config, dependencies, computationManifest
                 });
             }
 
-            // Publish High-Mem
             if (highMemTasks.length > 0) {
                 logger.log('INFO', `[Dispatcher] 🏋️‍♀️ Publishing ${highMemTasks.length} tasks to HIGH-MEM infrastructure.`);
                 await pubsubUtils.batchPublishTasks(dependencies, {

package/functions/computation-system/helpers/computation_worker.js

@@ -1,8 +1,7 @@
 /**
  * FILENAME: computation-system/helpers/computation_worker.js
  * PURPOSE: Consumes tasks, executes logic, and signals Workflow upon Batch Completion.
- * UPDATED: Implements IAM Auth for Workflow Callbacks.
- * UPDATED: Implements Peak Memory Heartbeat and Resource Tier tracking.
+ * UPDATED: Added "Contention-Aware Retry" for the Batch Counter to fix ABORTED errors.
  */
 
 const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
@@ -99,24 +98,56 @@ async function triggerWorkflowCallback(url, status, logger) {
 }
 
 /**
- * Helper: Decrements 'remainingTasks' in Firestore.
+ * [UPDATED] Helper: Decrements 'remainingTasks' in Firestore.
+ * NOW INCLUDES CONTENTION RETRY LOGIC (The "Sentinel" Fix)
  */
 async function decrementAndCheck(db, metaStatePath, logger) {
     if (!metaStatePath) return null;
-    try {
-        const result = await db.runTransaction(async (t) => {
-            const ref = db.doc(metaStatePath);
-            const doc = await t.get(ref);
-            if (!doc.exists) return null;
-            const data = doc.data();
-            const newRemaining = (data.remainingTasks || 0) - 1;
-            t.update(ref, { remainingTasks: newRemaining, lastUpdated: new Date() });
-            return { remaining: newRemaining, callbackUrl: data.callbackUrl };
-        });
-        if (result && result.remaining <= 0) return result.callbackUrl;
-    } catch (e) {
-        logger.log('ERROR', `[Worker] Failed to decrement batch counter: ${e.message}`);
+    
+    const MAX_CONTENTION_RETRIES = 10;
+    let attempt = 0;
+
+    while (attempt < MAX_CONTENTION_RETRIES) {
+        try {
+            const result = await db.runTransaction(async (t) => {
+                const ref = db.doc(metaStatePath);
+                const doc = await t.get(ref);
+                if (!doc.exists) return null;
+                
+                const data = doc.data();
+                // Safety: Don't decrement below zero
+                const currentRemaining = data.remainingTasks || 0;
+                if (currentRemaining <= 0) return { remaining: 0, callbackUrl: data.callbackUrl };
+
+                const newRemaining = currentRemaining - 1;
+                t.update(ref, { remainingTasks: newRemaining, lastUpdated: new Date() });
+                
+                return { remaining: newRemaining, callbackUrl: data.callbackUrl };
+            });
+
+            // Success! Check if we are the "Sentinel" (the last one)
+            if (result && result.remaining <= 0) return result.callbackUrl;
+            return null; // We decremented successfully, but weren't the last one.
+
+        } catch (e) {
+            // Check if it's a contention error (ABORTED/10 or DEADLINE_EXCEEDED/4)
+            const isContention = e.code === 10 || e.code === 4 || (e.message && e.message.includes('contention'));
+            
+            if (isContention) {
+                attempt++;
+                // JITTER: Random delay between 50ms and 500ms to desynchronize the herd
+                const delay = Math.floor(Math.random() * 450) + 50;
+                logger.log('WARN', `[Worker] Batch counter contention (Attempt ${attempt}/${MAX_CONTENTION_RETRIES}). Retrying in ${delay}ms...`);
+                await new Promise(r => setTimeout(r, delay));
+            } else {
+                // Fatal error (permission, etc)
+                logger.log('ERROR', `[Worker] Fatal error decrementing batch counter: ${e.message}`);
+                return null;
+            }
+        }
     }
+    
+    logger.log('ERROR', `[Worker] Failed to decrement batch counter after ${MAX_CONTENTION_RETRIES} attempts. The count will be inaccurate.`);
     return null;
 }
 

package/functions/computation-system/utils/data_loader.js

@@ -48,7 +48,7 @@ async function loadDataByRefs(config, deps, refs) {
     const { withRetry } = calculationUtils;
     if (!refs || !refs.length) return {};
     const mergedPortfolios = {};
-    const batchSize = config.partRefBatchSize || 50;
+    const batchSize = config.partRefBatchSize || 10;
     for (let i = 0; i < refs.length; i += batchSize) { 
         const batchRefs = refs.slice(i, i + batchSize); 
         const snapshots = await withRetry(() => db.getAll(...batchRefs), `getAll(batch ${Math.floor(i / batchSize)})`);
@@ -145,7 +145,7 @@ async function* streamPortfolioData(config, deps, dateString, providedRefs = nul
     const { logger } = deps;
     const refs = providedRefs || (await getPortfolioPartRefs(config, deps, dateString));
     if (refs.length === 0) { logger.log('WARN', `[streamPortfolioData] No portfolio refs found for ${dateString}. Stream is empty.`); return; }
-    const batchSize = config.partRefBatchSize || 50; 
+    const batchSize = config.partRefBatchSize || 10; 
     logger.log('INFO', `[streamPortfolioData] Streaming ${refs.length} portfolio parts in chunks of ${batchSize}...`);
     for (let i = 0; i < refs.length; i += batchSize) { 
         const batchRefs = refs.slice(i, i + batchSize); 
@@ -160,7 +160,7 @@ async function* streamHistoryData(config, deps, dateString, providedRefs = null)
     const { logger } = deps;
     const refs = providedRefs || (await getHistoryPartRefs(config, deps, dateString));
     if (refs.length === 0) { logger.log('WARN', `[streamHistoryData] No history refs found for ${dateString}. Stream is empty.`); return; }
-    const batchSize = config.partRefBatchSize || 50; 
+    const batchSize = config.partRefBatchSize || 10; 
     logger.log('INFO', `[streamHistoryData] Streaming ${refs.length} history parts in chunks of ${batchSize}...`);
     for (let i = 0; i < refs.length; i += batchSize) { 
         const batchRefs = refs.slice(i, i + batchSize); 

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,6 +1,6 @@
 # Cloud Workflows Definition for BullTrackers Computation Pipeline
 # Orchestrates 5 sequential passes using Event-Driven Callbacks (Zero Polling).
-# FIXED: Restored 'passes' and 'max_retries' variables in init step.
+# UPDATED: Passes 'attempt' count to Dispatcher to trigger Smart Forensics on retries.
 
 main:
   params: [input]
@@ -16,7 +16,7 @@ main:
           - yesterday_str: ${text.substring(time.format(yesterday_timestamp), 0, 10)}
           - date_to_run: ${default(map.get(input, "date"), yesterday_str)}
 
-          # Configuration Variables (Restored)
+          # Configuration Variables
           - passes: ["1", "2", "3", "4", "5"]
           - max_retries: 3
 
@@ -71,6 +71,7 @@ main:
                             body:
                               date: ${date_to_run}
                               callbackUrl: ${callback_url} 
+                              attempt: ${attempt_count} # [UPDATED] Critical for Smart Forensics logic
                             auth:
                               type: OIDC
                             timeout: 1800 # 30 mins max for dispatch analysis
@@ -96,7 +97,7 @@ main:
                           call: events.await_callback
                           args:
                             callback: ${callback_details}
-                            timeout: 10800 # UPDATED: Reduced from 86400 (24h) to 10800 (3h) to detect crashes faster
+                            timeout: 10800 # Reduced to 3h to fail faster if stalled
                           result: callback_request
 
                       # 5. PROCESS SIGNAL

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.300",
+  "version": "1.0.302",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [