bulltrackers-module 1.0.238 → 1.0.239

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,18 +1,18 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_dispatcher.js
- * PURPOSE: Dispatches computation tasks to Pub/Sub for scalable execution.
- * REFACTORED: Now uses WorkflowOrchestrator for helper functions.
+ * PURPOSE: Dispatches granular computation tasks (1 task = 1 calculation/date).
+ * UPDATED: Implements "Atomic Task Dispatch" for maximum reliability.
  */
 
-const { getExpectedDateStrings } = require('../utils/utils.js');
-const { groupByPass }            = require('../WorkflowOrchestrator.js');
-const { PubSubUtils }            = require('../../core/utils/pubsub_utils');
+const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
+const { groupByPass }                           = require('../WorkflowOrchestrator.js');
+const { PubSubUtils }                           = require('../../core/utils/pubsub_utils');
 
 const TOPIC_NAME = 'computation-tasks';
 
 /**
  * Dispatches computation tasks for a specific pass.
- * Instead of running them, it queues them in Pub/Sub.
+ * Generates one Pub/Sub message per calculation per date.
  */
 async function dispatchComputationPass(config, dependencies, computationManifest) {
     const { logger }  = dependencies;
@@ -21,17 +21,18 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     
     if (!passToRun) { return logger.log('ERROR', '[Dispatcher] No pass defined (COMPUTATION_PASS_TO_RUN). Aborting.'); }
 
-    // 1. Validate Pass Existence
+    // 1. Get Calculations for this Pass
     const passes          = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
 
     if (!calcsInThisPass.length) { return logger.log('WARN', `[Dispatcher] No calcs for Pass ${passToRun}. Exiting.`); }
 
-    const calcNames = calcsInThisPass.map(c => c.name).join(', ');
-    logger.log('INFO', `🚀 [Dispatcher] Preparing PASS ${passToRun}.`);
-    logger.log('INFO', `[Dispatcher] Included Calculations: [${calcNames}]`);
+    const calcNames = calcsInThisPass.map(c => c.name);
+    logger.log('INFO', `🚀 [Dispatcher] Preparing PASS ${passToRun} (Granular Mode).`);
+    logger.log('INFO', `[Dispatcher] Target Calculations: [${calcNames.join(', ')}]`);
 
     // 2. Determine Date Range
+    // (You can make this dynamic or config-driven)
     const earliestDates = {
         portfolio: new Date('2025-09-25T00:00:00Z'),
         history:   new Date('2025-11-05T00:00:00Z'),
@@ -44,35 +45,34 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
     const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
 
-    logger.log('INFO', `[Dispatcher] Dispatches checks for ${allExpectedDates.length} dates (${allExpectedDates[0]} to ${allExpectedDates[allExpectedDates.length - 1]}). Workers will validate dependencies.`);
-
-    // 3. Dispatch Messages
-    let dispatchedCount = 0;
-    const BATCH_SIZE    = 50;
-
-    // We can publish in parallel batches
-    const chunks = [];
-    for (let i = 0; i < allExpectedDates.length; i += BATCH_SIZE) { chunks.push(allExpectedDates.slice(i, i + BATCH_SIZE)); }
-
-    for (const chunk of chunks) {
-        const messages = chunk.map(dateStr => ({
-            json: {
-                action: 'RUN_COMPUTATION_DATE',
+    // 3. Generate Granular Tasks (Cartesian Product: Dates x Calculations)
+    const allTasks = [];
+    
+    for (const dateStr of allExpectedDates) {
+        for (const calc of calcsInThisPass) {
+            allTasks.push({
+                action: 'RUN_COMPUTATION_DATE', // Maintained for compatibility
                 date: dateStr,
                 pass: passToRun,
+                computation: normalizeName(calc.name), // CRITICAL: Target specific calc
                 timestamp: Date.now()
-            }
-        }));
-
-        try {
-            await pubsubUtils.publishMessageBatch(TOPIC_NAME, messages);
-            dispatchedCount += messages.length;
-            logger.log('INFO', `[Dispatcher] Dispatched batch of ${messages.length} tasks.`);
-        } catch (err) { logger.log('ERROR', `[Dispatcher] Failed to dispatch batch: ${err.message}`); }
+            });
+        }
     }
 
-    logger.log('INFO', `[Dispatcher] Finished. Dispatched ${dispatchedCount} checks for Pass ${passToRun}.`);
-    return { dispatched: dispatchedCount };
+    logger.log('INFO', `[Dispatcher] Generated ${allTasks.length} atomic tasks (${allExpectedDates.length} days × ${calcsInThisPass.length} calcs).`);
+
+    // 4. Batch Dispatch
+    // We send tasks in batches to Pub/Sub to be efficient, 
+    // but the WORKERS will process them individually.
+    await pubsubUtils.batchPublishTasks(dependencies, {
+        topicName: TOPIC_NAME,
+        tasks: allTasks,
+        taskType: `computation-pass-${passToRun}`,
+        maxPubsubBatchSize: 100 // Safe batch size
+    });
+
+    return { dispatched: allTasks.length };
 }
 
 module.exports = { dispatchComputationPass };

package/functions/computation-system/helpers/computation_worker.js

@@ -1,34 +1,27 @@
 /**
  * FILENAME: computation-system/helpers/computation_worker.js
  * PURPOSE: Consumes computation tasks from Pub/Sub and executes them.
- * UPDATED: Instantiates the internal StructuredLogger to replace the generic injected logger.
+ * UPDATED: Supports Granular Execution (Single Calculation filtering).
  */
 
 const { runDateComputation, groupByPass } = require('../WorkflowOrchestrator.js');
 const { getManifest }                     = require('../topology/ManifestLoader');
 const { StructuredLogger }                = require('../logger/logger'); 
+const { normalizeName }                   = require('../utils/utils');
 
 // 1. IMPORT CALCULATIONS
-// We import the specific package containing the strategies (gem, pyro, core, etc.)
 let calculationPackage;
 try {
-    // Primary: Try to load from the installed npm package
     calculationPackage = require('aiden-shared-calculations-unified');
 } catch (e) {
     console.error("FATAL: Could not load 'aiden-shared-calculations-unified'. Ensure it is installed or linked.");
     throw e;
 }
 
-
-// The package exports { calculations: { ... }, utils: { ... } }
 const calculations = calculationPackage.calculations;
 
 /**
  * Handles a single Pub/Sub message for a computation task.
- * Supports both Gen 1 (Message) and Gen 2 (CloudEvent) formats.
- * @param {object} message - The Pub/Sub message payload.
- * @param {object} config - System configuration.
- * @param {object} dependencies - Injected dependencies (db, generic logger, etc.).
  */
 async function handleComputationTask(message, config, dependencies) {
     
@@ -45,7 +38,7 @@ async function handleComputationTask(message, config, dependencies) {
         logger: systemLogger
     };
 
-    const { logger } = runDependencies; // Use this for local logging
+    const { logger } = runDependencies;
 
     // 4. LAZY LOAD MANIFEST
     let computationManifest;
@@ -86,19 +79,37 @@ async function handleComputationTask(message, config, dependencies) {
             return; 
         }
         
-        const { date, pass } = data;
+        const { date, pass, computation } = data; // Extract 'computation'
+        
         if (!date || !pass) { 
             logger.log('ERROR', `[Worker] Missing date or pass in payload: ${JSON.stringify(data)}`); 
             return; 
         }
         
-        logger.log('INFO', `[Worker] Received task: Date=${date}, Pass=${pass}`);
-        
+        // Load Full Pass
         const passes = groupByPass(computationManifest);
-        const calcsInThisPass = passes[pass] || [];
+        let calcsInThisPass = passes[pass] || [];
         
+        // --- GRANULAR FILTERING ---
+        if (computation) {
+            const targetName = normalizeName(computation);
+            const targetCalc = calcsInThisPass.find(c => normalizeName(c.name) === targetName);
+            
+            if (!targetCalc) {
+                logger.log('WARN', `[Worker] Targeted computation '${computation}' not found in Pass ${pass}. Skipping.`);
+                return;
+            }
+            
+            // We run ONLY this calculation
+            calcsInThisPass = [targetCalc];
+            logger.log('INFO', `[Worker] Granular Mode: Running ONLY ${targetCalc.name} for ${date}`);
+        } else {
+            logger.log('INFO', `[Worker] Bulk Mode: Running ${calcsInThisPass.length} calculations for ${date}`);
+        }
+        // ---------------------------
+
         if (!calcsInThisPass.length) { 
-            logger.log('WARN', `[Worker] No calculations found for Pass ${pass}.`); 
+            logger.log('WARN', `[Worker] No calculations found to run.`); 
             return; 
         }
         
@@ -112,14 +123,15 @@ async function handleComputationTask(message, config, dependencies) {
         );
         
         if (result && result.updates && Object.keys(result.updates).length > 0) { 
-            logger.log('INFO', `[Worker] Successfully processed ${date} (Pass ${pass}). Updates: ${Object.keys(result.updates).length}`);
+            logger.log('INFO', `[Worker] Success ${date}. Updated: ${Object.keys(result.updates).join(', ')}`);
         } else { 
-            logger.log('INFO', `[Worker] Processed ${date} (Pass ${pass}) - No updates.`); 
+            // In Granular Mode, this is common (e.g. if hash matched)
+            logger.log('INFO', `[Worker] Completed ${date} - No DB Writes (Up to date or skipped).`); 
         }
         
     } catch (err) { 
         logger.log('ERROR', `[Worker] Fatal error processing task: ${err.message}`, { stack: err.stack }); 
-        throw err;
+        throw err; // Throwing ensures Pub/Sub retries this specific computation
     }
 }
 

package/functions/computation-system/tools/BuildReporter.js

@@ -48,7 +48,7 @@ async function ensureBuildReport(config, dependencies, manifest) {
  * @param {number} daysBack - Days to simulate (default 7)
  * @param {string} customBuildId - Optional ID override
  */
-async function generateBuildReport(config, dependencies, manifest, daysBack = 7, customBuildId = null) {
+async function generateBuildReport(config, dependencies, manifest, daysBack = 90, customBuildId = null) {
     const { db, logger } = dependencies;
     const buildId = customBuildId || `manual_${Date.now()}`;
     

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.238",
+  "version": "1.0.239",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [