bulltrackers-module 1.0.301 → 1.0.302

package/functions/computation-system/executors/StandardExecutor.js

@@ -114,7 +114,7 @@ class StandardExecutor {
 
                 usersSinceLastFlush += chunkSize;
                 const heapStats = v8.getHeapStatistics();
-                if (usersSinceLastFlush >= 5000 || (heapStats.used_heap_size / heapStats.heap_size_limit) > 0.70) {
+                if (usersSinceLastFlush >= 500 || (heapStats.used_heap_size / heapStats.heap_size_limit) > 0.70) {
                     const flushResult = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, 'INTERMEDIATE', true, !hasFlushed);
                     hasFlushed = true;
                     StandardExecutor.mergeReports(aggregatedSuccess, aggregatedFailures, flushResult);

package/functions/computation-system/helpers/computation_worker.js

@@ -1,8 +1,7 @@
 /**
  * FILENAME: computation-system/helpers/computation_worker.js
  * PURPOSE: Consumes tasks, executes logic, and signals Workflow upon Batch Completion.
- * UPDATED: Implements IAM Auth for Workflow Callbacks.
- * UPDATED: Implements Peak Memory Heartbeat and Resource Tier tracking.
+ * UPDATED: Added "Contention-Aware Retry" for the Batch Counter to fix ABORTED errors.
  */
 
 const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
@@ -99,24 +98,56 @@ async function triggerWorkflowCallback(url, status, logger) {
 }
 
 /**
- * Helper: Decrements 'remainingTasks' in Firestore.
+ * [UPDATED] Helper: Decrements 'remainingTasks' in Firestore.
+ * NOW INCLUDES CONTENTION RETRY LOGIC (The "Sentinel" Fix)
  */
 async function decrementAndCheck(db, metaStatePath, logger) {
     if (!metaStatePath) return null;
-    try {
-        const result = await db.runTransaction(async (t) => {
-            const ref = db.doc(metaStatePath);
-            const doc = await t.get(ref);
-            if (!doc.exists) return null;
-            const data = doc.data();
-            const newRemaining = (data.remainingTasks || 0) - 1;
-            t.update(ref, { remainingTasks: newRemaining, lastUpdated: new Date() });
-            return { remaining: newRemaining, callbackUrl: data.callbackUrl };
-        });
-        if (result && result.remaining <= 0) return result.callbackUrl;
-    } catch (e) {
-        logger.log('ERROR', `[Worker] Failed to decrement batch counter: ${e.message}`);
+    
+    const MAX_CONTENTION_RETRIES = 10;
+    let attempt = 0;
+
+    while (attempt < MAX_CONTENTION_RETRIES) {
+        try {
+            const result = await db.runTransaction(async (t) => {
+                const ref = db.doc(metaStatePath);
+                const doc = await t.get(ref);
+                if (!doc.exists) return null;
+                
+                const data = doc.data();
+                // Safety: Don't decrement below zero
+                const currentRemaining = data.remainingTasks || 0;
+                if (currentRemaining <= 0) return { remaining: 0, callbackUrl: data.callbackUrl };
+
+                const newRemaining = currentRemaining - 1;
+                t.update(ref, { remainingTasks: newRemaining, lastUpdated: new Date() });
+                
+                return { remaining: newRemaining, callbackUrl: data.callbackUrl };
+            });
+
+            // Success! Check if we are the "Sentinel" (the last one)
+            if (result && result.remaining <= 0) return result.callbackUrl;
+            return null; // We decremented successfully, but weren't the last one.
+
+        } catch (e) {
+            // Check if it's a contention error (ABORTED/10 or DEADLINE_EXCEEDED/4)
+            const isContention = e.code === 10 || e.code === 4 || (e.message && e.message.includes('contention'));
+            
+            if (isContention) {
+                attempt++;
+                // JITTER: Random delay between 50ms and 500ms to desynchronize the herd
+                const delay = Math.floor(Math.random() * 450) + 50;
+                logger.log('WARN', `[Worker] Batch counter contention (Attempt ${attempt}/${MAX_CONTENTION_RETRIES}). Retrying in ${delay}ms...`);
+                await new Promise(r => setTimeout(r, delay));
+            } else {
+                // Fatal error (permission, etc)
+                logger.log('ERROR', `[Worker] Fatal error decrementing batch counter: ${e.message}`);
+                return null;
+            }
+        }
     }
+    
+    logger.log('ERROR', `[Worker] Failed to decrement batch counter after ${MAX_CONTENTION_RETRIES} attempts. The count will be inaccurate.`);
     return null;
 }
 

package/functions/computation-system/utils/data_loader.js

@@ -48,7 +48,7 @@ async function loadDataByRefs(config, deps, refs) {
     const { withRetry } = calculationUtils;
     if (!refs || !refs.length) return {};
     const mergedPortfolios = {};
-    const batchSize = config.partRefBatchSize || 50;
+    const batchSize = config.partRefBatchSize || 10;
     for (let i = 0; i < refs.length; i += batchSize) { 
         const batchRefs = refs.slice(i, i + batchSize); 
         const snapshots = await withRetry(() => db.getAll(...batchRefs), `getAll(batch ${Math.floor(i / batchSize)})`);
@@ -145,7 +145,7 @@ async function* streamPortfolioData(config, deps, dateString, providedRefs = nul
     const { logger } = deps;
     const refs = providedRefs || (await getPortfolioPartRefs(config, deps, dateString));
     if (refs.length === 0) { logger.log('WARN', `[streamPortfolioData] No portfolio refs found for ${dateString}. Stream is empty.`); return; }
-    const batchSize = config.partRefBatchSize || 50; 
+    const batchSize = config.partRefBatchSize || 10; 
     logger.log('INFO', `[streamPortfolioData] Streaming ${refs.length} portfolio parts in chunks of ${batchSize}...`);
     for (let i = 0; i < refs.length; i += batchSize) { 
         const batchRefs = refs.slice(i, i + batchSize); 
@@ -160,7 +160,7 @@ async function* streamHistoryData(config, deps, dateString, providedRefs = null)
     const { logger } = deps;
     const refs = providedRefs || (await getHistoryPartRefs(config, deps, dateString));
     if (refs.length === 0) { logger.log('WARN', `[streamHistoryData] No history refs found for ${dateString}. Stream is empty.`); return; }
-    const batchSize = config.partRefBatchSize || 50; 
+    const batchSize = config.partRefBatchSize || 10; 
     logger.log('INFO', `[streamHistoryData] Streaming ${refs.length} history parts in chunks of ${batchSize}...`);
     for (let i = 0; i < refs.length; i += batchSize) { 
         const batchRefs = refs.slice(i, i + batchSize); 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.301",
+  "version": "1.0.302",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [