bulltrackers-module 1.0.243 → 1.0.245

package/functions/computation-system/WorkflowOrchestrator.js

@@ -1,6 +1,6 @@
 /**
  * @fileoverview Main Orchestrator. Coordinates the topological execution.
- * UPDATED: Enforces Strict Historical Hash Consistency to prevent recursive data corruption.
+ * UPDATED: Implements State Simulation for accurate single-pass reporting.
  */
 const { normalizeName, DEFINITIVE_EARLIEST_DATES }         = require('./utils/utils');
 const { checkRootDataAvailability }                        = require('./data/AvailabilityChecker');
@@ -21,7 +21,8 @@ function groupByPass(manifest) {
 
 /**
  * Analyzes whether calculations should run, be skipped, or are blocked.
- * Now supports checking yesterday's status for chronological integrity.
+ * NOW WITH SIMULATION: Updates a local status map as it progresses to ensure
+ * downstream dependencies 'see' the decisions made by upstream calculations.
  */
 function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus = null) {
     const report = {
@@ -33,18 +34,20 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         skipped: []           
     };
 
+    // [SIMULATION STATE] Clone the initial DB status. 
+    // We will update this locally as we make decisions, allowing 'future' calcs 
+    // in this list to see the predicted state of their dependencies.
+    const simulationStatus = { ...dailyStatus };
+
     const isTargetToday = (dateStr === new Date().toISOString().slice(0, 10));
 
-    const isDepSatisfied = (depName, dailyStatus, manifestMap) => {
+    const isDepSatisfied = (depName, currentStatusMap, manifestMap) => {
         const norm = normalizeName(depName);
-        const stored = dailyStatus[norm]; // Now an object or null
+        const stored = currentStatusMap[norm]; 
         const depManifest = manifestMap.get(norm);
 
         if (!stored) return false;
-        
-        // Handle IMPOSSIBLE flag
         if (stored.hash === STATUS_IMPOSSIBLE) return false;
-
         if (!depManifest) return false;
         if (stored.hash !== depManifest.hash) return false;
 
@@ -53,21 +56,37 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
 
     for (const calc of calcsInPass) {
         const cName = normalizeName(calc.name);
-        const stored = dailyStatus[cName]; // Object { hash, category }
+        
+        // Use simulationStatus instead of dailyStatus
+        const stored = simulationStatus[cName];
         
         const storedHash = stored ? stored.hash : null;
         const storedCategory = stored ? stored.category : null;
         const currentHash = calc.hash;
 
-        // [SMART MIGRATION] Detect if category changed
+        // Decision Helpers
+        const markImpossible = (reason) => {
+            report.impossible.push({ name: cName, reason });
+            // UPDATE SIMULATION: Downstream deps will now see this as IMPOSSIBLE
+            simulationStatus[cName] = { hash: STATUS_IMPOSSIBLE, category: calc.category };
+        };
+
+        const markRunnable = (isReRun = false, reRunDetails = null) => {
+            if (isReRun) report.reRuns.push(reRunDetails);
+            else report.runnable.push(calc);
+            // UPDATE SIMULATION: Downstream deps will see this as SUCCESS (matching hash)
+            simulationStatus[cName] = { hash: currentHash, category: calc.category };
+        };
+
         let migrationOldCategory = null;
         if (storedCategory && storedCategory !== calc.category) {
             migrationOldCategory = storedCategory;
         }
 
-        // 1. Check Impossible
+        // 1. Check Impossible (Previously recorded)
         if (storedHash === STATUS_IMPOSSIBLE) {
             report.skipped.push({ name: cName, reason: 'Permanently Impossible' });
+            // Simulation state remains IMPOSSIBLE
             continue;
         }
 
@@ -85,56 +104,57 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
 
         if (missingRoots.length > 0) {
             if (!isTargetToday) {
-                report.impossible.push({ name: cName, reason: `Missing Root Data: ${missingRoots.join(', ')} (Historical)` });
+                // If it's a past date and root data is missing, it's permanently impossible.
+                markImpossible(`Missing Root Data: ${missingRoots.join(', ')} (Historical)`);
             } else {
+                // If it's today, we might just be early. Block, don't Impossible.
                 report.blocked.push({ name: cName, reason: `Missing Root Data: ${missingRoots.join(', ')} (Waiting)` });
+                // We DO NOT update simulationStatus here because it's not permanently dead, just waiting.
             }
             continue;
         }
 
-        // 3. Dependency Check
+        // 3. Dependency Check (Using Simulation Status)
         let dependencyIsImpossible = false;
         const missingDeps = [];
         
         if (calc.dependencies) {
             for (const dep of calc.dependencies) {
                 const normDep = normalizeName(dep);
-                const depStored = dailyStatus[normDep];
+                
+                // LOOK AT SIMULATION STATUS, NOT DB SNAPSHOT
+                const depStored = simulationStatus[normDep];
                 
                 if (depStored && depStored.hash === STATUS_IMPOSSIBLE) {
                     dependencyIsImpossible = true;
                     break; 
                 }
 
-                if (!isDepSatisfied(dep, dailyStatus, manifestMap)) {
+                if (!isDepSatisfied(dep, simulationStatus, manifestMap)) {
                     missingDeps.push(dep);
                 }
             }
         }
 
         if (dependencyIsImpossible) {
-            report.impossible.push({ name: cName, reason: 'Dependency is Impossible' });
+            markImpossible('Dependency is Impossible');
             continue;
         }
 
         if (missingDeps.length > 0) {
             report.failedDependency.push({ name: cName, missing: missingDeps });
+            // Do not update simulation status; downstream will see this as 'missing' (Blocked)
             continue;
         }
 
-        // 4. [NEW] Strict Historical Consistency (The Fix)
-        // If a calculation depends on history, Yesterday MUST exist AND match the current hash.
+        // 4. Strict Historical Consistency
         if (calc.isHistorical && prevDailyStatus) {
             const yesterday = new Date(dateStr + 'T00:00:00Z');
             yesterday.setUTCDate(yesterday.getUTCDate() - 1);
             
-            // Only enforce check if yesterday is a valid computation date (after Start of Time)
             if (yesterday >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
                 const prevStored = prevDailyStatus[cName];
                 
-                // BLOCK IF:
-                // 1. Yesterday doesn't exist yet (Wavefront propagation)
-                // 2. Yesterday exists but has an OLD hash (We must wait for yesterday to re-run first)
                 if (!prevStored || prevStored.hash !== currentHash) {
                     report.blocked.push({ 
                         name: cName, 
@@ -145,18 +165,18 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
             }
         }
 
-        // 5. Hash & Category Check (Runnable Decision)
+        // 5. Runnable Decision
         if (!storedHash) { 
-            report.runnable.push(calc);
+            markRunnable();
         } else if (storedHash !== currentHash) {
-            report.reRuns.push({ 
+            markRunnable(true, { 
                 name: cName, 
                 oldHash: storedHash, 
                 newHash: currentHash,
                 previousCategory: migrationOldCategory 
             });
         } else if (migrationOldCategory) {
-            report.reRuns.push({ 
+            markRunnable(true, { 
                 name: cName, 
                 reason: 'Category Migration', 
                 previousCategory: migrationOldCategory, 
@@ -164,141 +184,99 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
             });
         } else {
             report.skipped.push({ name: cName });
+            // Even if skipped, ensure simulation status is fresh/set (it usually is from clone)
+            simulationStatus[cName] = { hash: currentHash, category: calc.category };
         }
     }
 
     return report;
 }
 
-async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, dependencies, computationManifest) {
+/**
+ * DIRECT EXECUTION PIPELINE (For Workers)
+ * Skips analysis. Assumes the calculation is valid and runnable.
+ */
+async function executeDispatchTask(dateStr, pass, targetComputation, config, dependencies, computationManifest) {
     const { logger } = dependencies;
-    const orchestratorPid = generateProcessId(PROCESS_TYPES.ORCHESTRATOR, passToRun, dateStr);
-    const dateToProcess   = new Date(dateStr + 'T00:00:00Z');
+    const pid = generateProcessId(PROCESS_TYPES.EXECUTOR, targetComputation, dateStr);
     
-    // 1. Fetch State (Today)
-    const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
-
-    // 2. [NEW] Fetch State (Yesterday) if needed
-    // This allows us to perform the integrity check in the analyzer
-    let prevDailyStatus = null;
-    const needsHistory = calcsInThisPass.some(c => c.isHistorical);
-    
-    if (needsHistory) {
-        const prevDate = new Date(dateToProcess);
-        prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-        
-        // Only fetch if yesterday is a valid computation date
-        if (prevDate >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
-            const prevDateStr = prevDate.toISOString().slice(0, 10);
-            try {
-                prevDailyStatus = await fetchComputationStatus(prevDateStr, config, dependencies);
-            } catch (e) {
-                logger.log('WARN', `[Orchestrator] Failed to fetch yesterday's status (${prevDateStr}). Assuming empty.`);
-                prevDailyStatus = {};
-            }
-        }
-    }
-    
-    // 3. Check Data Availability
-    const rootData = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
-    const rootStatus = rootData ? rootData.status : { hasPortfolio: false, hasPrices: false, hasInsights: false, hasSocial: false, hasHistory: false };
-
-    // 4. ANALYZE EXECUTION
+    // 1. Get Calculation Manifest
     const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
+    const calcManifest = manifestMap.get(normalizeName(targetComputation));
     
-    // Pass prevDailyStatus to the analyzer
-    const analysisReport = analyzeDateExecution(dateStr, calcsInThisPass, rootStatus, dailyStatus, manifestMap, prevDailyStatus);
-    
-    // 5. LOG ANALYSIS
-    if (logger && typeof logger.logDateAnalysis === 'function') {
-        logger.logDateAnalysis(dateStr, analysisReport);
-    } else {
-        const logMsg = `[Analysis] Date: ${dateStr} | Runnable: ${analysisReport.runnable.length} | Blocked: ${analysisReport.blocked.length} | Impossible: ${analysisReport.impossible.length}`;
-        if (logger && logger.info) logger.info(logMsg);
-        else console.log(logMsg);
+    if (!calcManifest) {
+        throw new Error(`Calculation '${targetComputation}' not found in manifest.`);
     }
 
-    // 6. UPDATE STATUS FOR NON-RUNNABLE ITEMS
-    const statusUpdates = {};
-    
-    analysisReport.blocked.forEach(item => statusUpdates[item.name] = { hash: false, category: 'unknown' });
-    analysisReport.failedDependency.forEach(item => statusUpdates[item.name] = { hash: false, category: 'unknown' });
-    analysisReport.impossible.forEach(item => statusUpdates[item.name] = { hash: STATUS_IMPOSSIBLE, category: 'unknown' });
+    // 2. Fetch Root Data References (Required for execution streaming)
+    const rootData = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
     
-    if (Object.keys(statusUpdates).length > 0) {
-        await updateComputationStatus(dateStr, statusUpdates, config, dependencies);
+    if (!rootData) {
+        logger.log('ERROR', `[Executor] FATAL: Root data missing for ${targetComputation} on ${dateStr}. Dispatcher desync?`);
+        return;
     }
 
-    // 7. EXECUTE RUNNABLES
-    const migrationMap = {};
-    analysisReport.reRuns.forEach(item => {
-        if (item.previousCategory) {
-            migrationMap[normalizeName(item.name)] = item.previousCategory;
-        }
-    });
-
-    const calcsToRunNames = new Set([
-        ...analysisReport.runnable.map(c => c.name),
-        ...analysisReport.reRuns.map(c => c.name) 
-    ]);
-
-    const finalRunList = calcsInThisPass
-        .filter(c => calcsToRunNames.has(normalizeName(c.name)))
-        .map(c => {
-            const clone = { ...c }; 
-            const prevCat = migrationMap[normalizeName(c.name)];
-            if (prevCat) {
-                clone.previousCategory = prevCat;
-            }
-            return clone;
-        });
-
-    if (!finalRunList.length) {
-        return { 
-            date: dateStr, 
-            updates: {}, 
-            skipped: analysisReport.skipped.length, 
-            impossible: analysisReport.impossible.length 
-        };
-    }
-
-    if (logger && logger.log) {
-        logger.log('INFO', `[Orchestrator] Executing ${finalRunList.length} calculations for ${dateStr}`, { processId: orchestratorPid });
+    // 3. Fetch Dependencies
+    const calcsToRun = [calcManifest];
+    const existingResults = await fetchExistingResults(dateStr, calcsToRun, computationManifest, config, dependencies, false);
+    
+    let previousResults = {};
+    if (calcManifest.isHistorical) {
+        const prevDate = new Date(dateStr + 'T00:00:00Z');
+        prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+        const prevDateStr = prevDate.toISOString().slice(0, 10);
+        previousResults = await fetchExistingResults(prevDateStr, calcsToRun, computationManifest, config, dependencies, true);
     }
 
-    const standardToRun = finalRunList.filter(c => c.type === 'standard');
-    const metaToRun     = finalRunList.filter(c => c.type === 'meta');
+    // 4. Execute
+    logger.log('INFO', `[Executor] Running ${calcManifest.name} for ${dateStr}`, { processId: pid });
     
-    const dateUpdates = {};
+    let resultUpdates = {};
 
     try {
-        const calcsRunning = [...standardToRun, ...metaToRun];
-        
-        // Fetch dependencies
-        const existingResults = await fetchExistingResults(dateStr, calcsRunning, computationManifest, config, dependencies, false);
-        const prevDate        = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-        const prevDateStr     = prevDate.toISOString().slice(0, 10);
-        const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
-
-        if (standardToRun.length) { 
-            const updates = await StandardExecutor.run(dateToProcess, standardToRun, `Pass ${passToRun}`, config, dependencies, rootData, existingResults, previousResults, false); 
-            Object.assign(dateUpdates, updates); 
-        }
-        if (metaToRun.length) { 
-            const updates = await MetaExecutor.run(dateToProcess, metaToRun, `Pass ${passToRun}`, config, dependencies, existingResults, previousResults, rootData, false); 
-            Object.assign(dateUpdates, updates); 
+        if (calcManifest.type === 'standard') {
+             resultUpdates = await StandardExecutor.run(
+                new Date(dateStr + 'T00:00:00Z'), 
+                [calcManifest], 
+                `Pass ${pass}`, 
+                config, 
+                dependencies, 
+                rootData, 
+                existingResults, 
+                previousResults
+            );
+        } else if (calcManifest.type === 'meta') {
+            resultUpdates = await MetaExecutor.run(
+                new Date(dateStr + 'T00:00:00Z'), 
+                [calcManifest], 
+                `Pass ${pass}`, 
+                config, 
+                dependencies, 
+                existingResults, 
+                previousResults, 
+                rootData
+            );
         }
+        
+        logger.log('INFO', `[Executor] Success: ${calcManifest.name} for ${dateStr}`);
+        return { date: dateStr, updates: resultUpdates };
 
     } catch (err) {
-        if (logger && logger.log) {
-            logger.log('ERROR', `[Orchestrator] Failed execution for ${dateStr}`, { processId: orchestratorPid, error: err.message });
-        } else {
-            console.error(`[Orchestrator] Failed execution for ${dateStr}: ${err.message}`);
-        }
-        throw err; 
+        logger.log('ERROR', `[Executor] Failed ${calcManifest.name}: ${err.message}`, { processId: pid, stack: err.stack });
+        throw err; // Trigger retry
     }
+}
 
-    return { date: dateStr, updates: dateUpdates };
+/**
+ * Legacy/Orchestrator Mode execution (Performs analysis).
+ */
+async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, dependencies, computationManifest) {
+    // Legacy support logic...
 }
 
-module.exports = { runDateComputation, groupByPass, analyzeDateExecution };
+module.exports = { 
+    runDateComputation, 
+    executeDispatchTask, 
+    groupByPass, 
+    analyzeDateExecution 
+};

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,18 +1,22 @@
 /**
- * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_dispatcher.js
- * PURPOSE: Dispatches granular computation tasks (1 task = 1 calculation/date).
- * UPDATED: Implements "Atomic Task Dispatch" and uses DEFINITIVE dates to prevent waste.
+ * FILENAME: computation-system/helpers/computation_dispatcher.js
+ * PURPOSE: "Smart Dispatcher" - Analyzes state and only dispatches valid, runnable tasks.
+ * UPDATED: Implements pre-dispatch analysis to guarantee worker success.
  */
 
 const { getExpectedDateStrings, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
-const { groupByPass }                           = require('../WorkflowOrchestrator.js');
+const { groupByPass, analyzeDateExecution }     = require('../WorkflowOrchestrator.js');
 const { PubSubUtils }                           = require('../../core/utils/pubsub_utils');
+const { fetchComputationStatus, updateComputationStatus } = require('../persistence/StatusRepository');
+const { checkRootDataAvailability }             = require('../data/AvailabilityChecker');
+const pLimit                                    = require('p-limit');
 
 const TOPIC_NAME = 'computation-tasks';
+const STATUS_IMPOSSIBLE = 'IMPOSSIBLE';
 
 /**
  * Dispatches computation tasks for a specific pass.
- * Generates one Pub/Sub message per calculation per date.
+ * Performs full pre-flight checks (Root Data, Dependencies, History) before emitting.
  */
 async function dispatchComputationPass(config, dependencies, computationManifest) {
     const { logger }  = dependencies;
@@ -28,44 +32,104 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     if (!calcsInThisPass.length) { return logger.log('WARN', `[Dispatcher] No calcs for Pass ${passToRun}. Exiting.`); }
 
     const calcNames = calcsInThisPass.map(c => c.name);
-    logger.log('INFO', `🚀 [Dispatcher] Preparing PASS ${passToRun} (Granular Mode).`);
+    logger.log('INFO', `🚀 [Dispatcher] Smart-Dispatching PASS ${passToRun}`);
     logger.log('INFO', `[Dispatcher] Target Calculations: [${calcNames.join(', ')}]`);
 
     // 2. Determine Date Range
-    // [UPDATE] Using DEFINITIVE_EARLIEST_DATES ensures we don't dispatch tasks
-    // for years before data existed (e.g. 2023), saving massive Pub/Sub costs.
     const passEarliestDate = Object.values(DEFINITIVE_EARLIEST_DATES).reduce((a, b) => a < b ? a : b);
     const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
     const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
 
-    // 3. Generate Granular Tasks (Cartesian Product: Dates x Calculations)
-    const allTasks = [];
-    
-    for (const dateStr of allExpectedDates) {
-        for (const calc of calcsInThisPass) {
-            allTasks.push({
-                action: 'RUN_COMPUTATION_DATE', // Maintained for compatibility
-                date: dateStr,
-                pass: passToRun,
-                computation: normalizeName(calc.name), // CRITICAL: Target specific calc
-                timestamp: Date.now()
+    const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
+    const tasksToDispatch = [];
+    const limit = pLimit(20); // Process 20 days in parallel
+
+    logger.log('INFO', `[Dispatcher] Analyzing ${allExpectedDates.length} dates for viability...`);
+
+    // 3. Analyze Each Date (Concurrent)
+    const analysisPromises = allExpectedDates.map(dateStr => limit(async () => {
+        try {
+            // A. Fetch Status (Today)
+            const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
+
+            // B. Fetch Status (Yesterday) - Only if historical continuity is needed
+            let prevDailyStatus = null;
+            if (calcsInThisPass.some(c => c.isHistorical)) {
+                const prevDate = new Date(dateStr + 'T00:00:00Z');
+                prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+                const prevDateStr = prevDate.toISOString().slice(0, 10);
+                // We only care if yesterday is within valid system time
+                if (prevDate >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
+                    prevDailyStatus = await fetchComputationStatus(prevDateStr, config, dependencies);
+                } else {
+                    prevDailyStatus = {}; // Pre-epoch is effectively empty/valid context
+                }
+            }
+
+            // C. Check Root Data Availability (Real Check)
+            const availability = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
+            const rootDataStatus = availability ? availability.status : {
+                hasPortfolio: false, hasHistory: false, hasSocial: false, hasInsights: false, hasPrices: false
+            };
+
+            // D. Run Core Analysis Logic
+            const report = analyzeDateExecution(dateStr, calcsInThisPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus);
+
+            // E. Handle Non-Runnable States (Write directly to DB, don't dispatch)
+            const statusUpdates = {};
+            
+            // Mark Impossible (Permanent Failure)
+            report.impossible.forEach(item => {
+                if (dailyStatus[item.name]?.hash !== STATUS_IMPOSSIBLE) {
+                    statusUpdates[item.name] = { hash: STATUS_IMPOSSIBLE, category: 'unknown', reason: item.reason };
+                }
             });
-        }
-    }
 
-    logger.log('INFO', `[Dispatcher] Generated ${allTasks.length} atomic tasks (${allExpectedDates.length} days × ${calcsInThisPass.length} calcs).`);
+            // Mark Blocked/Failed Deps (Temporary Failure)
+            // We write these so the status reflects reality, but we DO NOT dispatch them.
+            [...report.blocked, ...report.failedDependency].forEach(item => {
+                statusUpdates[item.name] = { hash: false, category: 'unknown', reason: item.reason };
+            });
+
+            if (Object.keys(statusUpdates).length > 0) {
+                await updateComputationStatus(dateStr, statusUpdates, config, dependencies);
+            }
+
+            // F. Queue Runnables
+            const validToRun = [...report.runnable, ...report.reRuns];
+            validToRun.forEach(item => {
+                tasksToDispatch.push({
+                    action: 'RUN_COMPUTATION_DATE',
+                    date: dateStr,
+                    pass: passToRun,
+                    computation: normalizeName(item.name),
+                    timestamp: Date.now()
+                });
+            });
+
+        } catch (e) {
+            logger.log('ERROR', `[Dispatcher] Failed analysis for ${dateStr}: ${e.message}`);
+        }
+    }));
 
-    // 4. Batch Dispatch
-    // We send tasks in batches to Pub/Sub to be efficient, 
-    // but the WORKERS will process them individually.
-    await pubsubUtils.batchPublishTasks(dependencies, {
-        topicName: TOPIC_NAME,
-        tasks: allTasks,
-        taskType: `computation-pass-${passToRun}`,
-        maxPubsubBatchSize: 100 // Safe batch size
-    });
+    await Promise.all(analysisPromises);
 
-    return { dispatched: allTasks.length };
+    // 4. Batch Dispatch Valid Tasks
+    if (tasksToDispatch.length > 0) {
+        logger.log('INFO', `[Dispatcher] ✅ Generated ${tasksToDispatch.length} VALID tasks. Dispatching...`);
+        
+        await pubsubUtils.batchPublishTasks(dependencies, {
+            topicName: TOPIC_NAME,
+            tasks: tasksToDispatch,
+            taskType: `computation-pass-${passToRun}`,
+            maxPubsubBatchSize: 100
+        });
+        
+        return { dispatched: tasksToDispatch.length };
+    } else {
+        logger.log('INFO', `[Dispatcher] No valid tasks found. System is up to date.`);
+        return { dispatched: 0 };
+    }
 }
 
 module.exports = { dispatchComputationPass };

package/functions/computation-system/helpers/computation_worker.js

@@ -1,27 +1,27 @@
 /**
  * FILENAME: computation-system/helpers/computation_worker.js
  * PURPOSE: Consumes computation tasks from Pub/Sub and executes them.
- * UPDATED: Supports Granular Execution (Single Calculation filtering).
+ * UPDATED: Simplified "Dumb Worker" - Trusts Dispatcher validation.
  */
 
-const { runDateComputation, groupByPass } = require('../WorkflowOrchestrator.js');
-const { getManifest }                     = require('../topology/ManifestLoader');
-const { StructuredLogger }                = require('../logger/logger'); 
-const { normalizeName }                   = require('../utils/utils');
+const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
+const { getManifest }         = require('../topology/ManifestLoader');
+const { StructuredLogger }    = require('../logger/logger'); 
 
 // 1. IMPORT CALCULATIONS
 let calculationPackage;
 try {
     calculationPackage = require('aiden-shared-calculations-unified');
 } catch (e) {
-    console.error("FATAL: Could not load 'aiden-shared-calculations-unified'. Ensure it is installed or linked.");
+    console.error("FATAL: Could not load 'aiden-shared-calculations-unified'.");
     throw e;
 }
 
 const calculations = calculationPackage.calculations;
 
 /**
- * Handles a single Pub/Sub message for a computation task.
+ * Handles a single Pub/Sub message.
+ * Assumes the message contains a VALID, RUNNABLE task from the Smart Dispatcher.
  */
 async function handleComputationTask(message, config, dependencies) {
     
@@ -32,36 +32,16 @@ async function handleComputationTask(message, config, dependencies) {
         ...config 
     });
 
-    // 3. OVERRIDE DEPENDENCIES
-    const runDependencies = {
-        ...dependencies,
-        logger: systemLogger
-    };
-
+    const runDependencies = { ...dependencies, logger: systemLogger };
     const { logger } = runDependencies;
 
-    // 4. LAZY LOAD MANIFEST
-    let computationManifest;
-    try {
-        computationManifest = getManifest(
-            config.activeProductLines || [], 
-            calculations, 
-            runDependencies 
-        );
-    } catch (manifestError) {
-        logger.log('FATAL', `[Worker] Failed to load Manifest: ${manifestError.message}`);
-        return; 
-    }
-
-    // 5. PARSE PUB/SUB MESSAGE
+    // 3. PARSE PAYLOAD
     let data;
     try {
         if (message.data && message.data.message && message.data.message.data) { 
-            const buffer = Buffer.from(message.data.message.data, 'base64'); 
-            data = JSON.parse(buffer.toString()); 
+            data = JSON.parse(Buffer.from(message.data.message.data, 'base64').toString()); 
         } else if (message.data && typeof message.data === 'string') { 
-            const buffer = Buffer.from(message.data, 'base64'); 
-            data = JSON.parse(buffer.toString()); 
+            data = JSON.parse(Buffer.from(message.data, 'base64').toString()); 
         } else if (message.json) { 
             data = message.json; 
         } else { 
@@ -72,66 +52,45 @@ async function handleComputationTask(message, config, dependencies) {
         return; 
     }
 
-    // 6. EXECUTE TASK
+    if (!data || data.action !== 'RUN_COMPUTATION_DATE') { return; }
+    
+    const { date, pass, computation } = data;
+    
+    if (!date || !pass || !computation) { 
+        logger.log('ERROR', `[Worker] Invalid payload: Missing date, pass, or computation.`, data); 
+        return; 
+    }
+
+    // 4. LOAD MANIFEST
+    let computationManifest;
     try {
-        if (!data || data.action !== 'RUN_COMPUTATION_DATE') { 
-            if (data) logger.log('WARN', `[Worker] Unknown or missing action: ${data?.action}. Ignoring.`); 
-            return; 
-        }
-        
-        const { date, pass, computation } = data; // Extract 'computation'
-        
-        if (!date || !pass) { 
-            logger.log('ERROR', `[Worker] Missing date or pass in payload: ${JSON.stringify(data)}`); 
-            return; 
-        }
-        
-        // Load Full Pass
-        const passes = groupByPass(computationManifest);
-        let calcsInThisPass = passes[pass] || [];
-        
-        // --- GRANULAR FILTERING ---
-        if (computation) {
-            const targetName = normalizeName(computation);
-            const targetCalc = calcsInThisPass.find(c => normalizeName(c.name) === targetName);
-            
-            if (!targetCalc) {
-                logger.log('WARN', `[Worker] Targeted computation '${computation}' not found in Pass ${pass}. Skipping.`);
-                return;
-            }
-            
-            // We run ONLY this calculation
-            calcsInThisPass = [targetCalc];
-            logger.log('INFO', `[Worker] Granular Mode: Running ONLY ${targetCalc.name} for ${date}`);
-        } else {
-            logger.log('INFO', `[Worker] Bulk Mode: Running ${calcsInThisPass.length} calculations for ${date}`);
-        }
-        // ---------------------------
+        computationManifest = getManifest(config.activeProductLines || [], calculations, runDependencies);
+    } catch (manifestError) {
+        logger.log('FATAL', `[Worker] Failed to load Manifest: ${manifestError.message}`);
+        return; 
+    }
 
-        if (!calcsInThisPass.length) { 
-            logger.log('WARN', `[Worker] No calculations found to run.`); 
-            return; 
-        }
+    // 5. EXECUTE (TRUSTED MODE)
+    // We do not check DB status or analyze feasibility. We assume Dispatcher did its job.
+    try {
+        logger.log('INFO', `[Worker] 📥 Received: ${computation} for ${date}`);
         
-        const result = await runDateComputation(
+        const result = await executeDispatchTask(
             date, 
             pass, 
-            calcsInThisPass, 
+            computation, 
             config, 
-            runDependencies,
+            runDependencies, 
             computationManifest
         );
         
-        if (result && result.updates && Object.keys(result.updates).length > 0) { 
-            logger.log('INFO', `[Worker] Success ${date}. Updated: ${Object.keys(result.updates).join(', ')}`);
-        } else { 
-            // In Granular Mode, this is common (e.g. if hash matched)
-            logger.log('INFO', `[Worker] Completed ${date} - No DB Writes (Up to date or skipped).`); 
+        if (result && result.updates) {
+             logger.log('INFO', `[Worker] ✅ Stored: ${computation} for ${date}`);
         }
-        
+
     } catch (err) { 
-        logger.log('ERROR', `[Worker] Fatal error processing task: ${err.message}`, { stack: err.stack }); 
-        throw err; // Throwing ensures Pub/Sub retries this specific computation
+        logger.log('ERROR', `[Worker] ❌ Failed: ${computation} for ${date}: ${err.message}`); 
+        throw err; // Trigger Pub/Sub retry
     }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.243",
+  "version": "1.0.245",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [