bulltrackers-module 1.0.301 → 1.0.303

package/functions/computation-system/executors/StandardExecutor.js

@@ -114,7 +114,7 @@ class StandardExecutor {
 
                 usersSinceLastFlush += chunkSize;
                 const heapStats = v8.getHeapStatistics();
-                if (usersSinceLastFlush >= 5000 || (heapStats.used_heap_size / heapStats.heap_size_limit) > 0.70) {
+                if (usersSinceLastFlush >= 500 || (heapStats.used_heap_size / heapStats.heap_size_limit) > 0.70) {
                     const flushResult = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, 'INTERMEDIATE', true, !hasFlushed);
                     hasFlushed = true;
                     StandardExecutor.mergeReports(aggregatedSuccess, aggregatedFailures, flushResult);

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,303 +1,149 @@
 /**
  * FILENAME: computation-system/helpers/computation_dispatcher.js
- * PURPOSE: "Smart Dispatcher" - Analyzes state, initializes Run Counters, and dispatches tasks.
- * UPDATED: Optimized Forensics - Only runs on Retries (Attempt > 1) or Single-Day runs.
+ * PURPOSE: Sequential Cursor-Based Dispatcher.
+ * IMPLEMENTS: Dirty-Date Discovery, Forensics Rerouting, and Satiation Sweeps.
  */
 
 const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
-const { groupByPass, analyzeDateExecution }     = require('../WorkflowOrchestrator.js');
-const { PubSubUtils }                           = require('../../core/utils/pubsub_utils');
-const { fetchComputationStatus, updateComputationStatus } = require('../persistence/StatusRepository');
-const { checkRootDataAvailability }             = require('../data/AvailabilityChecker');
-const { generateCodeHash }                      = require('../topology/HashManager');
-const pLimit                                    = require('p-limit');
-const crypto                                    = require('crypto');
+const { groupByPass, analyzeDateExecution } = require('../WorkflowOrchestrator.js');
+const { PubSubUtils } = require('../../core/utils/pubsub_utils');
+const { fetchComputationStatus } = require('../persistence/StatusRepository');
+const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
 
-const STATUS_IMPOSSIBLE   = 'IMPOSSIBLE';
-
-// Threshold to trigger high-mem routing (e.g., 1.5 GB for a 2GB worker)
 const OOM_THRESHOLD_MB = 1500; 
+const SECONDS_PER_CALC_MARGIN = 25; // 20s base + safety margin
 
 /**
- * [NEW] Forensics: Checks if the calculation crashed previously due to Memory.
- * Reads the 'telemetry.lastMemory' from the audit ledger.
+ * Checks if specific tasks on a date need a high-memory reroute.
+ * Returns only tasks that failed on 'standard' and haven't been tried on 'high-mem'.
  */
-async function checkCrashForensics(db, date, pass, computationName) {
-    try {
-        const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${computationName}`;
+async function getHighMemReroutes(db, date, pass, tasks) {
+    const reroutes = [];
+    for (const task of tasks) {
+        const name = normalizeName(task.name);
+        const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${name}`;
         const doc = await db.doc(ledgerPath).get();
-
-        // Default to standard
-        if (!doc.exists) return 'standard';
-
-        const data = doc.data();
         
-        // Check if we have telemetry from a previous run
-        if (data.telemetry && data.telemetry.lastMemory) {
-            const lastRSS = data.telemetry.lastMemory.rssMB || 0;
+        if (doc.exists) {
+            const data = doc.data();
+            // Check if it failed due to memory and hasn't been attempted on high-mem yet
+            const isOOM = (data.status === 'FAILED' || data.status === 'CRASH') && 
+                          (data.resourceTier !== 'high-mem') &&
+                          ((data.peakMemoryMB > OOM_THRESHOLD_MB) || (data.error && /memory/i.test(data.error.message)));
             
-            if (lastRSS > OOM_THRESHOLD_MB) {
-                // console.log(`[Dispatcher] 🕵️‍♀️ Forensics: ${computationName} likely OOM'd at ${lastRSS}MB. Routing to HIGH-MEM.`);
-                return 'high-mem';
+            if (isOOM) {
+                reroutes.push({ ...task, resources: 'high-mem' });
             }
         }
-        
-        // Also check if it's explicitly marked FAILED with 'Memory' in error
-        if (data.status === 'FAILED' && data.error && /memory/i.test(data.error)) {
-            return 'high-mem';
-        }
-
-    } catch (e) {
-        console.warn(`[Dispatcher] Forensics check failed for ${computationName}: ${e.message}`);
     }
-    
-    return 'standard';
+    return reroutes;
 }
 
-/**
- * Dispatches computation tasks for a specific pass.
- * @param {Object} config - System config (Injected with topics)
- * @param {Object} dependencies - { db, logger, ... }
- * @param {Array} computationManifest - List of calculations
- * @param {Object} reqBody - (Optional) HTTP Body containing 'callbackUrl', 'date', and 'attempt'
- */
 async function dispatchComputationPass(config, dependencies, computationManifest, reqBody = {}) {
     const { logger, db } = dependencies;
-    const pubsubUtils    = new PubSubUtils(dependencies);
-    const passToRun      = String(config.COMPUTATION_PASS_TO_RUN);
+    const pubsubUtils = new PubSubUtils(dependencies);
     
-    // Extract Date and Callback from request body (pushed by Workflow)
-    const dateStr     = reqBody.date || config.date; 
-    const callbackUrl = reqBody.callbackUrl || null;
-    // [NEW] Get Attempt Count (Default to 1 if missing)
-    const attemptCount = reqBody.attempt ? parseInt(reqBody.attempt) : 1;
-
-    if (!passToRun) { return logger.log('ERROR', '[Dispatcher] No pass defined (COMPUTATION_PASS_TO_RUN). Aborting.'); }
-    if (!dateStr)   { return logger.log('ERROR', '[Dispatcher] No date defined. Aborting.'); }
+    // Inputs from Workflow Cursor
+    const passToRun = String(reqBody.pass || config.COMPUTATION_PASS_TO_RUN);
+    const targetCursorN = parseInt(reqBody.cursorIndex || 1);
+    const dateLimitStr = reqBody.date || config.date; 
 
-    const currentManifestHash = generateCodeHash( computationManifest.map(c => c.hash).sort().join('|') );
-
-    const passes          = groupByPass(computationManifest);
+    const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
+    const passes = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
 
-    if (!calcsInThisPass.length) { return logger.log('WARN', `[Dispatcher] No calcs for Pass ${passToRun}. Exiting.`); }
-
-    logger.log('INFO', `🚀 [Dispatcher] Smart-Dispatching PASS ${passToRun} (Target: ${dateStr}) [Attempt ${attemptCount}]`);
-    
-    // -- DATE ANALYSIS LOGIC (FIXED: RANGE SCAN) --
-    
-    // 1. Determine the absolute start of data history
-    const earliestDates = await getEarliestDataDates(config, dependencies);
-    const startDate     = earliestDates.absoluteEarliest;
-    const endDate       = new Date(dateStr + 'T00:00:00Z');
-
-    // 2. Generate the full range of dates to check
-    let allExpectedDates = getExpectedDateStrings(startDate, endDate);
-
-    // Safety fallback
-    if (!allExpectedDates || allExpectedDates.length === 0) {
-        logger.log('WARN', `[Dispatcher] Date range calculation returned empty (Start: ${startDate.toISOString()} -> End: ${endDate.toISOString()}). Defaulting to single target date.`);
-        allExpectedDates = [dateStr];
-    } else {
-        logger.log('INFO', `[Dispatcher] 📅 Analysis Range: ${allExpectedDates.length} days (${allExpectedDates[0]} to ${allExpectedDates[allExpectedDates.length-1]})`);
+    if (!calcsInThisPass.length) {
+        return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0 };
     }
 
-    const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
-    const tasksToDispatch = [];
+    // 1. Discover all "Dirty" Dates (Dates needing work)
+    const earliestDates = await getEarliestDataDates(config, dependencies);
+    const allDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date(dateLimitStr + 'T00:00:00Z'));
     
-    // [FIX] Separate concurrency limits
-    const dateLimit      = pLimit(20); 
-    const forensicsLimit = pLimit(50); 
-
-    // [NEW] SMART FORENSICS TRIGGER
-    // 1. If scanning > 5 days (Backfill), SKIP (Too expensive).
-    // 2. If attempt == 1 (First Run), SKIP (Assume Standard).
-    // 3. Only run if Attempt > 1 AND Small Batch.
-    const isBulkBackfill = allExpectedDates.length > 5;
-    const shouldRunForensics = (attemptCount > 1) && !isBulkBackfill;
-
-    if (!shouldRunForensics) {
-        if (isBulkBackfill) logger.log('INFO', `[Dispatcher] ⏩ Bulk Backfill (${allExpectedDates.length} days). Skipping Forensics.`);
-        else logger.log('INFO', `[Dispatcher] ⏩ First Attempt. Skipping Forensics (Defaulting to Standard).`);
-    } else {
-        logger.log('WARN', `[Dispatcher] 🕵️‍♀️ Retry Detected (Attempt ${attemptCount}). Enabling Forensic Crash Analysis.`);
-    }
-
-    const analysisPromises = allExpectedDates.map(d => dateLimit(async () => {
-        try {
-            const fetchPromises = [
-                fetchComputationStatus(d, config, dependencies),
-                checkRootDataAvailability(d, config, dependencies, DEFINITIVE_EARLIEST_DATES)
-            ];
-
-            let prevDateStr = null;
-            if (calcsInThisPass.some(c => c.isHistorical)) {
-                const prevDate = new Date(d + 'T00:00:00Z');
-                prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-                prevDateStr = prevDate.toISOString().slice(0, 10);
-                if (prevDate >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
-                    fetchPromises.push(fetchComputationStatus(prevDateStr, config, dependencies));
-                }
-            }
-
-            const results         = await Promise.all(fetchPromises);
-            const dailyStatus     = results[0];
-            const availability    = results[1];
-            const prevDailyStatus = (prevDateStr && results[2]) ? results[2] : (prevDateStr ? {} : null);
-
-            const rootDataStatus = availability ? availability.status : {
-                hasPortfolio: false, hasHistory: false, hasSocial: false, hasInsights: false, hasPrices: false
-            };
-
-            const report = analyzeDateExecution(d, calcsInThisPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus);
-
-            const statusUpdates = {};
-            report.impossible.forEach(item => {
-                if (dailyStatus[item.name]?.hash !== STATUS_IMPOSSIBLE) {
-                    statusUpdates[item.name] = { hash: STATUS_IMPOSSIBLE, category: 'unknown', reason: item.reason };
-                }
-            });
-            report.blocked.forEach(item => {
-                 statusUpdates[item.name] = { hash: false, category: 'unknown', reason: item.reason };
-            });
-            report.failedDependency.forEach(item => {
-                 const missingStr = item.missing ? item.missing.join(', ') : 'unknown';
-                 statusUpdates[item.name] = { hash: false, category: 'unknown', reason: `Dependency Missing: ${missingStr}` };
-            });
-
-            if (Object.keys(statusUpdates).length > 0) {
-                await updateComputationStatus(d, statusUpdates, config, dependencies);
-            }
-
-            const validToRun = [...report.runnable, ...report.reRuns];
-            
-            await Promise.all(validToRun.map(item => forensicsLimit(async () => {
-                const compName = normalizeName(item.name);
-                
-                // [UPDATED] Conditional Forensics
-                let requiredResource = 'standard';
-                if (shouldRunForensics) {
-                    requiredResource = await checkCrashForensics(db, d, passToRun, compName);
-                }
-                
-                const uniqueDispatchId = crypto.randomUUID(); 
-                tasksToDispatch.push({
-                    action: 'RUN_COMPUTATION_DATE',
-                    dispatchId: uniqueDispatchId,
-                    date: d,
-                    pass: passToRun,
-                    computation: compName,
-                    hash: item.hash || item.newHash, 
-                    previousCategory: item.previousCategory || null,
-                    triggerReason: item.reason || "Unknown",
-                    dependencyResultHashes: item.dependencyResultHashes || {},
-                    timestamp: Date.now(),
-                    resources: requiredResource
-                });
-            })));
-
-            logger.log('INFO', `[Dispatcher] Analyzed ${d}: ${validToRun.length} tasks (Cumulative: ${tasksToDispatch.length})`);
-
-        } catch (e) {
-            logger.log('ERROR', `[Dispatcher] Failed analysis for ${d}: ${e.message}`);
+    const dirtyDates = [];
+    for (const d of allDates) {
+        const dailyStatus = await fetchComputationStatus(d, config, dependencies);
+        const availability = await checkRootDataAvailability(d, config, dependencies, DEFINITIVE_EARLIEST_DATES);
+        
+        const report = analyzeDateExecution(d, calcsInThisPass, availability.status, dailyStatus, manifestMap, null);
+        const tasks = [...report.runnable, ...report.reRuns];
+        
+        if (tasks.length > 0) {
+            dirtyDates.push({ date: d, tasks });
         }
-    }));
-
-    await Promise.all(analysisPromises);
+    }
 
-    // -- CALLBACK & COUNTER INITIALIZATION --
+    let selectedDate = null;
+    let selectedTasks = [];
+    let isReroute = false;
+    let isSweep = false;
 
-    if (tasksToDispatch.length > 0) {
-        
-        const runId = crypto.randomUUID();
-        const metaStatePath = `computation_runs/${runId}`;
+    // 2. Logic: Prioritize Reroute for N-1
+    if (targetCursorN > 1 && (targetCursorN - 2) < dirtyDates.length) {
+        const prevEntry = dirtyDates[targetCursorN - 2];
+        const reroutes = await getHighMemReroutes(db, prevEntry.date, passToRun, prevEntry.tasks);
         
-        if (callbackUrl) {
-            await db.doc(metaStatePath).set({
-                createdAt: new Date(),
-                date: dateStr, 
-                pass: passToRun,
-                totalTasks: tasksToDispatch.length,
-                remainingTasks: tasksToDispatch.length, 
-                callbackUrl: callbackUrl,               
-                status: 'IN_PROGRESS'
-            });
-            logger.log('INFO', `[Dispatcher] 🏁 Run State Initialized: ${runId}. Tasks: ${tasksToDispatch.length}`);
+        if (reroutes.length > 0) {
+            selectedDate = prevEntry.date;
+            selectedTasks = reroutes;
+            isReroute = true;
+            logger.log('INFO', `[Dispatcher] Reroute detected for ${selectedDate}. Pausing N increment.`);
         }
+    }
 
-        tasksToDispatch.forEach(task => {
-            task.runId = runId;
-            task.metaStatePath = callbackUrl ? metaStatePath : null; 
-        });
-        
-        const finalDispatched = [];
-        const txnLimit        = pLimit(50);
-        
-        const txnPromises   = tasksToDispatch.map(task => txnLimit(async () => {
-            const ledgerRef = db.collection(`computation_audit_ledger/${task.date}/passes/${task.pass}/tasks`).doc(task.computation);
-            try {
-                await db.runTransaction(async (t) => {
-                    const doc = await t.get(ledgerRef);
-                    if (doc.exists) {
-                        const data = doc.data();
-                        if (data.status === 'COMPLETED') return false;
-                    }
-                    t.set(ledgerRef, {
-                        status: 'PENDING',
-                        dispatchId: task.dispatchId,
-                        runId: task.runId,
-                        computation: task.computation,
-                        expectedHash: task.hash || 'unknown',
-                        createdAt: new Date(),
-                        dispatcherHash: currentManifestHash,
-                        triggerReason: task.triggerReason,
-                        resources: task.resources,
-                        retries: 0
-                    }, { merge: true });
-                    return true;
-                });
-                finalDispatched.push(task);
-            } catch (txnErr) {
-                logger.log('WARN', `[Dispatcher] Transaction failed for ${task.computation}: ${txnErr.message}`);
+    // 3. Logic: N-th Dirty Date or Final Sweep
+    if (!selectedDate) {
+        if (targetCursorN <= dirtyDates.length) {
+            const entry = dirtyDates[targetCursorN - 1];
+            selectedDate = entry.date;
+            selectedTasks = entry.tasks;
+        } else {
+            // Final Satiation Sweep: Check if anything was missed (recovery)
+            if (dirtyDates.length > 0) {
+                isSweep = true;
+                selectedDate = dirtyDates[0].date;
+                selectedTasks = dirtyDates[0].tasks;
             }
-        }));
-
-        await Promise.all(txnPromises);
+        }
+    }
 
-        if (finalDispatched.length > 0) {
-            const standardTasks = finalDispatched.filter(t => t.resources !== 'high-mem');
-            const highMemTasks  = finalDispatched.filter(t => t.resources === 'high-mem');
+    // 4. Signal Pass Completion
+    if (!selectedDate) {
+        return { status: 'MOVE_TO_NEXT_PASS', dispatched: 0, etaSeconds: 0 };
+    }
 
-            if (standardTasks.length > 0) {
-                logger.log('INFO', `[Dispatcher] ✅ Publishing ${standardTasks.length} Standard tasks...`);
-                await pubsubUtils.batchPublishTasks(dependencies, {
-                    topicName: config.computationTopicStandard || 'computation-tasks',
-                    tasks: standardTasks,
-                    taskType: `computation-pass-${passToRun}-std`,
-                    maxPubsubBatchSize: 100
-                });
-            }
+    // 5. Dispatch to PubSub (Standard vs. High-Mem)
+    const standardTasks = selectedTasks.filter(t => t.resources !== 'high-mem').map(t => ({ ...t, date: selectedDate, pass: passToRun }));
+    const highMemTasks = selectedTasks.filter(t => t.resources === 'high-mem').map(t => ({ ...t, date: selectedDate, pass: passToRun }));
 
-            if (highMemTasks.length > 0) {
-                logger.log('INFO', `[Dispatcher] 🏋️‍♀️ Publishing ${highMemTasks.length} tasks to HIGH-MEM infrastructure.`);
-                await pubsubUtils.batchPublishTasks(dependencies, {
-                    topicName: config.computationTopicHighMem || 'computation-tasks-highmem',
-                    tasks: highMemTasks,
-                    taskType: `computation-pass-${passToRun}-highmem`,
-                    maxPubsubBatchSize: 100
-                });
-            }
-            
-            return { dispatched: finalDispatched.length, runId };
-        } else {
-            logger.log('INFO', `[Dispatcher] All tasks were already COMPLETED.`);
-            return { dispatched: 0 };
-        }
-
-    } else {
-        logger.log('INFO', `[Dispatcher] No valid tasks found (Up to date).`);
-        return { dispatched: 0 };
+    const pubPromises = [];
+    if (standardTasks.length > 0) {
+        pubPromises.push(pubsubUtils.batchPublishTasks(dependencies, {
+            topicName: config.computationTopicStandard || 'computation-tasks',
+            tasks: standardTasks,
+            taskType: `pass-${passToRun}-std`
+        }));
     }
+    if (highMemTasks.length > 0) {
+        pubPromises.push(pubsubUtils.batchPublishTasks(dependencies, {
+            topicName: config.computationTopicHighMem || 'computation-tasks-highmem',
+            tasks: highMemTasks,
+            taskType: `pass-${passToRun}-high`
+        }));
+    }
+    await Promise.all(pubPromises);
+
+    const etaSeconds = Math.max(20, selectedTasks.length * SECONDS_PER_CALC_MARGIN);
+    
+    logger.log('INFO', `[Dispatcher] ${isReroute ? 'Reroute' : (isSweep ? 'Sweep' : 'Standard')} Run: ${selectedDate}. Tasks: ${selectedTasks.length}. ETA: ${etaSeconds}s`);
+
+    return {
+        status: isSweep ? 'RECOVERY' : 'CONTINUE_PASS',
+        dateProcessed: selectedDate,
+        dispatched: selectedTasks.length,
+        n_cursor_ignored: isReroute, // Tell workflow to stay on same N
+        etaSeconds: etaSeconds
+    };
 }
 
 module.exports = { dispatchComputationPass };

package/functions/computation-system/helpers/computation_worker.js

@@ -1,8 +1,7 @@
 /**
  * FILENAME: computation-system/helpers/computation_worker.js
  * PURPOSE: Consumes tasks, executes logic, and signals Workflow upon Batch Completion.
- * UPDATED: Implements IAM Auth for Workflow Callbacks.
- * UPDATED: Implements Peak Memory Heartbeat and Resource Tier tracking.
+ * UPDATED: Added "Contention-Aware Retry" for the Batch Counter to fix ABORTED errors.
  */
 
 const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
@@ -99,24 +98,56 @@ async function triggerWorkflowCallback(url, status, logger) {
 }
 
 /**
- * Helper: Decrements 'remainingTasks' in Firestore.
+ * [UPDATED] Helper: Decrements 'remainingTasks' in Firestore.
+ * NOW INCLUDES CONTENTION RETRY LOGIC (The "Sentinel" Fix)
  */
 async function decrementAndCheck(db, metaStatePath, logger) {
     if (!metaStatePath) return null;
-    try {
-        const result = await db.runTransaction(async (t) => {
-            const ref = db.doc(metaStatePath);
-            const doc = await t.get(ref);
-            if (!doc.exists) return null;
-            const data = doc.data();
-            const newRemaining = (data.remainingTasks || 0) - 1;
-            t.update(ref, { remainingTasks: newRemaining, lastUpdated: new Date() });
-            return { remaining: newRemaining, callbackUrl: data.callbackUrl };
-        });
-        if (result && result.remaining <= 0) return result.callbackUrl;
-    } catch (e) {
-        logger.log('ERROR', `[Worker] Failed to decrement batch counter: ${e.message}`);
+    
+    const MAX_CONTENTION_RETRIES = 10;
+    let attempt = 0;
+
+    while (attempt < MAX_CONTENTION_RETRIES) {
+        try {
+            const result = await db.runTransaction(async (t) => {
+                const ref = db.doc(metaStatePath);
+                const doc = await t.get(ref);
+                if (!doc.exists) return null;
+                
+                const data = doc.data();
+                // Safety: Don't decrement below zero
+                const currentRemaining = data.remainingTasks || 0;
+                if (currentRemaining <= 0) return { remaining: 0, callbackUrl: data.callbackUrl };
+
+                const newRemaining = currentRemaining - 1;
+                t.update(ref, { remainingTasks: newRemaining, lastUpdated: new Date() });
+                
+                return { remaining: newRemaining, callbackUrl: data.callbackUrl };
+            });
+
+            // Success! Check if we are the "Sentinel" (the last one)
+            if (result && result.remaining <= 0) return result.callbackUrl;
+            return null; // We decremented successfully, but weren't the last one.
+
+        } catch (e) {
+            // Check if it's a contention error (ABORTED/10 or DEADLINE_EXCEEDED/4)
+            const isContention = e.code === 10 || e.code === 4 || (e.message && e.message.includes('contention'));
+            
+            if (isContention) {
+                attempt++;
+                // JITTER: Random delay between 50ms and 500ms to desynchronize the herd
+                const delay = Math.floor(Math.random() * 450) + 50;
+                logger.log('WARN', `[Worker] Batch counter contention (Attempt ${attempt}/${MAX_CONTENTION_RETRIES}). Retrying in ${delay}ms...`);
+                await new Promise(r => setTimeout(r, delay));
+            } else {
+                // Fatal error (permission, etc)
+                logger.log('ERROR', `[Worker] Fatal error decrementing batch counter: ${e.message}`);
+                return null;
+            }
+        }
     }
+    
+    logger.log('ERROR', `[Worker] Failed to decrement batch counter after ${MAX_CONTENTION_RETRIES} attempts. The count will be inaccurate.`);
     return null;
 }
 

package/functions/computation-system/utils/data_loader.js

@@ -48,7 +48,7 @@ async function loadDataByRefs(config, deps, refs) {
     const { withRetry } = calculationUtils;
     if (!refs || !refs.length) return {};
     const mergedPortfolios = {};
-    const batchSize = config.partRefBatchSize || 50;
+    const batchSize = config.partRefBatchSize || 10;
     for (let i = 0; i < refs.length; i += batchSize) { 
         const batchRefs = refs.slice(i, i + batchSize); 
         const snapshots = await withRetry(() => db.getAll(...batchRefs), `getAll(batch ${Math.floor(i / batchSize)})`);
@@ -145,7 +145,7 @@ async function* streamPortfolioData(config, deps, dateString, providedRefs = nul
     const { logger } = deps;
     const refs = providedRefs || (await getPortfolioPartRefs(config, deps, dateString));
     if (refs.length === 0) { logger.log('WARN', `[streamPortfolioData] No portfolio refs found for ${dateString}. Stream is empty.`); return; }
-    const batchSize = config.partRefBatchSize || 50; 
+    const batchSize = config.partRefBatchSize || 10; 
     logger.log('INFO', `[streamPortfolioData] Streaming ${refs.length} portfolio parts in chunks of ${batchSize}...`);
     for (let i = 0; i < refs.length; i += batchSize) { 
         const batchRefs = refs.slice(i, i + batchSize); 
@@ -160,7 +160,7 @@ async function* streamHistoryData(config, deps, dateString, providedRefs = null)
     const { logger } = deps;
     const refs = providedRefs || (await getHistoryPartRefs(config, deps, dateString));
     if (refs.length === 0) { logger.log('WARN', `[streamHistoryData] No history refs found for ${dateString}. Stream is empty.`); return; }
-    const batchSize = config.partRefBatchSize || 50; 
+    const batchSize = config.partRefBatchSize || 10; 
     logger.log('INFO', `[streamHistoryData] Streaming ${refs.length} history parts in chunks of ${batchSize}...`);
     for (let i = 0; i < refs.length; i += batchSize) { 
         const batchRefs = refs.slice(i, i + batchSize); 

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,148 +1,53 @@
-# Cloud Workflows Definition for BullTrackers Computation Pipeline
-# Orchestrates 5 sequential passes using Event-Driven Callbacks (Zero Polling).
-# UPDATED: Passes 'attempt' count to Dispatcher to trigger Smart Forensics on retries.
-
+# Cloud Workflows: Precision Cursor-Based Orchestrator
 main:
   params: [input]
   steps:
     - init:
         assign:
           - project: ${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}
-          - location: "europe-west1"
-          
-          # T-1 Date Logic (Process Yesterday)
-          - now: ${sys.now()}
-          - yesterday_timestamp: ${now - 86400} 
-          - yesterday_str: ${text.substring(time.format(yesterday_timestamp), 0, 10)}
-          - date_to_run: ${default(map.get(input, "date"), yesterday_str)}
-
-          # Configuration Variables
           - passes: ["1", "2", "3", "4", "5"]
-          - max_retries: 3
+          - date_to_run: ${default(map.get(input, "date"), "2025-01-01")}
 
-    # ======================================================
-    # MAIN LOOP: Iterate through Passes 1 to 5
-    # ======================================================
-    - run_passes:
+    - run_sequential_passes:
         for:
           value: pass_id
           in: ${passes}
           steps:
-            - init_pass_vars:
+            - init_cursor:
                 assign:
-                  - attempt_count: 0
-                  - pass_success: false
-                  # URL of the Cloud Function acting as the Dispatcher
-                  - dispatcher_url: ${"https://europe-west1-" + project + ".cloudfunctions.net/computation-pass-" + pass_id}
+                  - n_cursor: 1
+                  - pass_complete: false
 
-            # -----------------------------------------------
-            # RETRY LOOP: Try to complete the pass up to 3 times
-            # -----------------------------------------------
-            - pass_retry_loop:
+            - sequential_date_loop:
                 switch:
-                  - condition: ${attempt_count < max_retries and not pass_success}
+                  - condition: ${not pass_complete}
                     steps:
-                      - increment_attempt:
-                          assign:
-                            - attempt_count: ${attempt_count + 1}
-                      
-                      # 1. GENERATE CALLBACK ENDPOINT
-                      - create_callback:
-                          call: events.create_callback_endpoint
-                          args:
-                              http_callback_method: "POST"
-                          result: callback_details
-
-                      - extract_callback_url:
-                          assign:
-                              - callback_url: ${callback_details.url}
-
-                      - log_start:
-                          call: sys.log
-                          args:
-                            text: ${"Starting Pass " + pass_id + " (Attempt " + attempt_count + ") for " + date_to_run + ". Waiting for signal at " + callback_url}
-                            severity: "INFO"
-
-                      # 2. TRIGGER DISPATCHER
-                      - trigger_dispatcher:
+                      - call_dispatcher:
                           call: http.post
                           args:
-                            url: ${dispatcher_url}
+                            url: ${"https://europe-west1-" + project + ".cloudfunctions.net/dispatch-pass-" + pass_id}
                             body:
+                              pass: ${pass_id}
+                              cursorIndex: ${n_cursor}
                               date: ${date_to_run}
-                              callbackUrl: ${callback_url} 
-                              attempt: ${attempt_count} # [UPDATED] Critical for Smart Forensics logic
-                            auth:
-                              type: OIDC
-                            timeout: 1800 # 30 mins max for dispatch analysis
-                          result: dispatch_response
+                            auth: { type: OIDC }
+                          result: dispatch_res
 
-                      # 3. CHECK FOR "NOTHING TO DO"
-                      - check_immediate_completion:
+                      - evaluate_dispatch:
                           switch:
-                            - condition: ${dispatch_response.body.dispatched == 0}
+                            # State 1: Dispatcher signals pass is fully completed
+                            - condition: ${dispatch_res.body.status == "MOVE_TO_NEXT_PASS"}
+                              assign: [pass_complete: true]
+                            
+                            # State 2: Tasks dispatched (Standard, Reroute, or Recovery)
+                            - condition: ${dispatch_res.body.dispatched > 0}
                               steps:
-                                - log_empty:
-                                    call: sys.log
-                                    args:
-                                      text: ${"Pass " + pass_id + " had 0 tasks. Moving to next pass immediately."}
-                                - mark_success_empty:
+                                - wait_for_eta:
+                                    call: sys.sleep
+                                    args: { seconds: ${dispatch_res.body.etaSeconds} }
+                                - update_cursor:
                                     assign:
-                                      - pass_success: true
-                                - next_pass_empty:
-                                    next: pass_retry_loop
-
-                      # 4. WAIT FOR WORKER SIGNAL
-                      - wait_for_completion:
-                          call: events.await_callback
-                          args:
-                            callback: ${callback_details}
-                            timeout: 10800 # Reduced to 3h to fail faster if stalled
-                          result: callback_request
-
-                      # 5. PROCESS SIGNAL
-                      - parse_signal:
-                          assign:
-                            - signal_data: ${callback_request.http_request.body}
-
-                      - evaluate_signal:
-                          switch:
-                            - condition: ${signal_data.status == "SUCCESS"}
-                              steps:
-                                - log_success:
-                                    call: sys.log
-                                    args:
-                                      text: ${"Pass " + pass_id + " signaled COMPLETION via Callback."}
-                                      severity: "INFO"
-                                - mark_success:
-                                    assign:
-                                      - pass_success: true
-                                - proceed:
-                                    next: pass_retry_loop
-
-                            - condition: ${signal_data.status == "FAILURE"}
-                              steps:
-                                - log_failure:
-                                    call: sys.log
-                                    args:
-                                      text: ${"Pass " + pass_id + " signaled FAILURE (DLQ limit hit). Retrying pass."}
-                                      severity: "WARNING"
-                                - retry_pass:
-                                    next: pass_retry_loop
-
-            # -----------------------------------------------
-            # END RETRY LOOP
-            # -----------------------------------------------
-            
-            - check_final_status:
-                switch:
-                  - condition: ${not pass_success}
-                    steps:
-                      - log_giving_up:
-                          call: sys.log
-                          args:
-                            text: ${"Pass " + pass_id + " failed after " + max_retries + " attempts. Proceeding with potential gaps."}
-                            severity: "ERROR"
-
-    - finish:
-        return: "Pipeline Execution Complete"
+                                      # If it was a reroute, stay on the same N to retry the sequence
+                                      - n_cursor: ${if(dispatch_res.body.n_cursor_ignored, n_cursor, n_cursor + 1)}
+                                - next_step:
+                                    next: sequential_date_loop

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.301",
+  "version": "1.0.303",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [