bulltrackers-module 1.0.199 → 1.0.201

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -0,0 +1,82 @@
+/**
+ * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_dispatcher.js
+ * PURPOSE: Dispatches computation tasks to Pub/Sub for scalable execution.
+ */
+
+const { getExpectedDateStrings } = require('../utils/utils.js');
+const { groupByPass } = require('./orchestration_helpers.js');
+
+const TOPIC_NAME = 'computation-tasks';
+
+/**
+ * Dispatches computation tasks for a specific pass.
+ * Instead of running them, it queues them in Pub/Sub.
+ */
+async function dispatchComputationPass(config, dependencies, computationManifest) {
+    const { logger, pubsubUtils } = dependencies;
+    const passToRun = String(config.COMPUTATION_PASS_TO_RUN);
+
+    if (!passToRun) {
+        return logger.log('ERROR', '[Dispatcher] No pass defined (COMPUTATION_PASS_TO_RUN). Aborting.');
+    }
+
+    logger.log('INFO', `🚀 [Dispatcher] Preparing to dispatch PASS ${passToRun}...`);
+
+    // 1. Determine Date Range (Same logic as PassRunner)
+    // Hardcoded earliest dates - keep synced with PassRunner for now
+    const earliestDates = {
+        portfolio: new Date('2025-09-25T00:00:00Z'),
+        history: new Date('2025-11-05T00:00:00Z'),
+        social: new Date('2025-10-30T00:00:00Z'),
+        insights: new Date('2025-08-26T00:00:00Z'),
+        price: new Date('2025-08-01T00:00:00Z')
+    };
+    const passEarliestDate = Object.values(earliestDates).reduce((a, b) => a < b ? a : b);
+    const endDateUTC = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
+
+    const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
+
+    // 2. Validate Pass Existence
+    const passes = groupByPass(computationManifest);
+    const calcsInThisPass = passes[passToRun] || [];
+
+    if (!calcsInThisPass.length) {
+        return logger.log('WARN', `[Dispatcher] No calcs for Pass ${passToRun}. Exiting.`);
+    }
+
+    logger.log('INFO', `[Dispatcher] Found ${calcsInThisPass.length} calcs for Pass ${passToRun}. Target dates: ${allExpectedDates.length}`);
+
+    // 3. Dispatch Messages
+    let dispatchedCount = 0;
+    const BATCH_SIZE = 50; // Pub/Sub batch publishing size
+
+    // We can publish in parallel batches
+    const chunks = [];
+    for (let i = 0; i < allExpectedDates.length; i += BATCH_SIZE) {
+        chunks.push(allExpectedDates.slice(i, i + BATCH_SIZE));
+    }
+
+    for (const chunk of chunks) {
+        const messages = chunk.map(dateStr => ({
+            json: {
+                action: 'RUN_COMPUTATION_DATE',
+                date: dateStr,
+                pass: passToRun,
+                timestamp: Date.now()
+            }
+        }));
+
+        try {
+            await pubsubUtils.publishMessageBatch(TOPIC_NAME, messages);
+            dispatchedCount += messages.length;
+            logger.log('INFO', `[Dispatcher] Dispatched batch of ${messages.length} tasks.`);
+        } catch (err) {
+            logger.log('ERROR', `[Dispatcher] Failed to dispatch batch: ${err.message}`);
+        }
+    }
+
+    logger.log('INFO', `[Dispatcher] Finished dispatching. Total tasks: ${dispatchedCount}`);
+    return { dispatched: dispatchedCount };
+}
+
+module.exports = { dispatchComputationPass };

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -1,202 +1,158 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_pass_runner.js
- * FIXED: Integrates 'runBatchPriceComputation' to prevent OOM on price calculations.
- * FIXED: Added try/catch around runBatchPriceComputation to prevent crash on failure.
+ * FIXED: 'runDateComputation' now executes ALL calculation types (Standard, Meta, AND Price).
  */
 
-const { 
-    groupByPass, 
-    checkRootDataAvailability, 
-    fetchExistingResults, 
-    fetchComputationStatus,   
+const {
+    groupByPass,
+    checkRootDataAvailability,
+    fetchExistingResults,
+    fetchComputationStatus,
     updateComputationStatus,
-    runStandardComputationPass, 
+    runStandardComputationPass,
     runMetaComputationPass,
     checkRootDependencies,
-    runBatchPriceComputation 
+    runBatchPriceComputation
 } = require('./orchestration_helpers.js');
 
 const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
 
 const PARALLEL_BATCH_SIZE = 7;
 
+/**
+ * LEGACY / MANUAL RUNNER
+ * (Kept for backward compatibility if you run the old HTTP endpoint directly)
+ */
 async function runComputationPass(config, dependencies, computationManifest) {
     const { logger } = dependencies;
-    const passToRun  = String(config.COMPUTATION_PASS_TO_RUN);
-    if (!passToRun) 
-        return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
-    
-    logger.log('INFO', `🚀 Starting PASS ${passToRun} (Targeting /computation_status/{YYYY-MM-DD})...`);
+    const passToRun = String(config.COMPUTATION_PASS_TO_RUN);
+    if (!passToRun) return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
+
+    logger.log('INFO', `🚀 Starting PASS ${passToRun} (Legacy Mode)...`);
 
     // Hardcoded earliest dates
-    const earliestDates = { 
-        portfolio: new Date('2025-09-25T00:00:00Z'), 
-        history:   new Date('2025-11-05T00:00:00Z'), 
-        social:    new Date('2025-10-30T00:00:00Z'), 
-        insights:  new Date('2025-08-26T00:00:00Z'),
-        price:     new Date('2025-08-01T00:00:00Z') // This is slightly arbitrary, the true earliest date is 1 year ago, but there is no need to backfill that far so we just set it to be slightly earlier than the 2nd earliest computation.
+    const earliestDates = {
+        portfolio: new Date('2025-09-25T00:00:00Z'),
+        history: new Date('2025-11-05T00:00:00Z'),
+        social: new Date('2025-10-30T00:00:00Z'),
+        insights: new Date('2025-08-26T00:00:00Z'),
+        price: new Date('2025-08-01T00:00:00Z')
     };
-    earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a,b) => a < b ? a : b);
+    earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a, b) => a < b ? a : b);
 
     const passes = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
 
-    if (!calcsInThisPass.length) 
+    if (!calcsInThisPass.length)
         return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
-    
-    const passEarliestDate = earliestDates.absoluteEarliest; 
-    const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
-    const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
 
-    // --- SEPARATION OF CONCERNS ---
-    // Identify calculations that require the Optimized Price Batch Runner
-    const priceBatchCalcs = calcsInThisPass.filter(c => 
-        c.type === 'meta' && 
-        c.rootDataDependencies && 
-        c.rootDataDependencies.includes('price')
-    );
+    const passEarliestDate = earliestDates.absoluteEarliest;
+    const endDateUTC = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
+    const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
 
-    // Identify calculations for the Standard Date-Loop Runner
+    // Legacy Batch Optimization for Price (Only used in legacy loop)
+    const priceBatchCalcs = calcsInThisPass.filter(c => c.type === 'meta' && c.rootDataDependencies?.includes('price'));
     const standardAndOtherMetaCalcs = calcsInThisPass.filter(c => !priceBatchCalcs.includes(c));
 
-
-    // ========================================================================
-    // 1. EXECUTE OPTIMIZED PRICE BATCH (Shard-First)
-    // ========================================================================
     if (priceBatchCalcs.length > 0) {
-        logger.log('INFO', `[PassRunner] Detected ${priceBatchCalcs.length} Price-Meta calculations. Checking statuses...`);
-        
         try {
-            // Filter dates that actually need these calculations
-            // We do a quick serial check of status docs to avoid re-running satisfied dates
-            const datesNeedingPriceCalc = [];
-            
-            // Check statuses in chunks to avoid blowing up IO
-            const STATUS_CHECK_CHUNK = 20;
-            for (let i = 0; i < allExpectedDates.length; i += STATUS_CHECK_CHUNK) {
-                const dateChunk = allExpectedDates.slice(i, i + STATUS_CHECK_CHUNK);
-                await Promise.all(dateChunk.map(async (dateStr) => {
-                    const status = await fetchComputationStatus(dateStr, config, dependencies);
-                    // If ANY of the price calcs are missing/false, we run the batch for this date
-                    const needsRun = priceBatchCalcs.some(c => status[normalizeName(c.name)] !== true);
-                    if (needsRun) datesNeedingPriceCalc.push(dateStr);
-                }));
-            }
-
-            if (datesNeedingPriceCalc.length > 0) {
-                logger.log('INFO', `[PassRunner] >>> Starting Optimized Batch for ${datesNeedingPriceCalc.length} dates <<<`);
-                
-                // Execute the Shard-First Logic
-                await runBatchPriceComputation(config, dependencies, datesNeedingPriceCalc, priceBatchCalcs);
-
-                // Manually update statuses for these dates/calcs upon completion
-                // (runBatchPriceComputation handles the results, but we must mark the status doc)
-                logger.log('INFO', `[PassRunner] Updating status documents for batch...`);
-                
-                const BATCH_UPDATE_SIZE = 50;
-                for (let i = 0; i < datesNeedingPriceCalc.length; i += BATCH_UPDATE_SIZE) {
-                    const updateChunk = datesNeedingPriceCalc.slice(i, i + BATCH_UPDATE_SIZE);
-                    await Promise.all(updateChunk.map(async (dateStr) => {
-                        const updates = {};
-                        priceBatchCalcs.forEach(c => updates[normalizeName(c.name)] = true);
-                        await updateComputationStatus(dateStr, updates, config, dependencies);
-                    }));
-                }
-                logger.log('INFO', `[PassRunner] >>> Optimized Batch Complete <<<`);
-            } else {
-                logger.log('INFO', `[PassRunner] All Price-Meta calculations are up to date.`);
-            }
-        } catch (batchError) {
-            // FIX: Catch unexpected crashes in the optimized batch runner to allow standard calcs to proceed
-            logger.log('ERROR', `[PassRunner] Optimized Price Batch Failed! Continuing to standard calculations.`, { errorMessage: batchError.message });
-        }
+            await runBatchPriceComputation(config, dependencies, allExpectedDates, priceBatchCalcs); // Simplified for legacy
+        } catch (e) { logger.log('ERROR', 'Legacy Batch Price failed', e); }
     }
 
+    if (standardAndOtherMetaCalcs.length === 0) return;
 
-    // ========================================================================
-    // 2. EXECUTE STANDARD DATE LOOP (Date-First)
-    // ========================================================================
-    if (standardAndOtherMetaCalcs.length === 0) {
-        logger.log('INFO', `[PassRunner] No other calculations remaining. Exiting.`);
-        return;
+    for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) {
+        const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
+        await Promise.all(batch.map(dateStr => runDateComputation(dateStr, passToRun, standardAndOtherMetaCalcs, config, dependencies, computationManifest)));
     }
+}
 
-    const standardCalcs = standardAndOtherMetaCalcs.filter(c => c.type === 'standard');
-    const metaCalcs     = standardAndOtherMetaCalcs.filter(c => c.type === 'meta');
+/**
+ * UPDATED: Isolated function to run computations for a single date.
+ * Used by the Pub/Sub Worker.
+ */
+async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, dependencies, computationManifest) {
+    const { logger } = dependencies;
+    const dateToProcess = new Date(dateStr + 'T00:00:00Z');
+
+    // 1. Fetch Status for THIS specific date only
+    const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
+
+    // Helper: Check status
+    const shouldRun = (calc) => {
+        const cName = normalizeName(calc.name);
+        if (dailyStatus[cName] === true) return false;
+        if (calc.dependencies && calc.dependencies.length > 0) {
+            const missing = calc.dependencies.filter(depName => dailyStatus[normalizeName(depName)] !== true);
+            if (missing.length > 0) return false;
+        }
+        return true;
+    };
 
-    // Process a single date
-    const processDate = async (dateStr) => {
-        const dateToProcess = new Date(dateStr + 'T00:00:00Z');
+    // --- FIX: Run ALL calc types (Standard, Meta, Price) ---
+    const calcsToAttempt = calcsInThisPass.filter(shouldRun);
 
-        // 1. Fetch Status for THIS specific date only
-        const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
+    if (!calcsToAttempt.length) return null;
 
-        // Helper: Check status
-        const shouldRun = (calc) => {
-            const cName = normalizeName(calc.name);
-            if (dailyStatus[cName] === true) return false;
-            
-            if (calc.dependencies && calc.dependencies.length > 0) {
-                const missing = calc.dependencies.filter(depName => dailyStatus[normalizeName(depName)] !== true);
-                if (missing.length > 0) return false;
-            }
-            return true;
-        };
+    // 2. Check Root Data Availability
+    const earliestDates = {
+        portfolio: new Date('2025-09-25T00:00:00Z'),
+        history: new Date('2025-11-05T00:00:00Z'),
+        social: new Date('2025-10-30T00:00:00Z'),
+        insights: new Date('2025-08-26T00:00:00Z'),
+        price: new Date('2025-08-01T00:00:00Z')
+    };
 
-        const standardToRun = standardCalcs.filter(shouldRun);
-        const metaToRun     = metaCalcs.filter(shouldRun);
-        
-        if (!standardToRun.length && !metaToRun.length) return null; 
+    const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
+    if (!rootData) {
+        logger.log('INFO', `[DateRunner] Root data missing for ${dateStr}. Skipping.`);
+        return null;
+    }
 
-        // 2. Check Root Data Availability
-        const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
-        if (!rootData) return null;
+    // 3. Filter again based on Root Data availability
+    const runnableCalcs = calcsToAttempt.filter(c => checkRootDependencies(c, rootData.status).canRun);
 
-        // 3. Filter again based on Root Data availability
-        const finalStandardToRun = standardToRun.filter(c => checkRootDependencies(c, rootData.status).canRun);
-        const finalMetaToRun     = metaToRun.filter(c => checkRootDependencies(c, rootData.status).canRun);
+    if (!runnableCalcs.length) return null;
 
-        if (!finalStandardToRun.length && !finalMetaToRun.length) return null;
+    // Split into Standard (Streaming) and Meta (Once-Per-Day/Price)
+    const standardToRun = runnableCalcs.filter(c => c.type === 'standard');
+    // Note: Meta includes Price calcs in this flow
+    const metaToRun = runnableCalcs.filter(c => c.type === 'meta');
 
-        logger.log('INFO', `[PassRunner] Running ${dateStr}: ${finalStandardToRun.length} std, ${finalMetaToRun.length} meta`);
+    logger.log('INFO', `[DateRunner] Running ${dateStr}: ${standardToRun.length} std, ${metaToRun.length} meta`);
 
-        const dateUpdates = {}; 
+    const dateUpdates = {};
 
-        try {
-            const calcsRunning = [...finalStandardToRun, ...finalMetaToRun];
-            const existingResults = await fetchExistingResults(dateStr, calcsRunning, computationManifest, config, dependencies, false);
-            const prevDate        = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-            const prevDateStr     = prevDate.toISOString().slice(0, 10);
-            const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
-            
-            // Changed skipstatus write to false to ensure updates are recorded, allowing for proper tracking and avoiding re-computation in future passes. NOTE : Writing true here introduces significant bugs and should be avoided.
-            if (finalStandardToRun.length) {
-                const updates = await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, false); 
-                Object.assign(dateUpdates, updates);
-            }
-            if (finalMetaToRun.length) { // Again, writing true here introduces significant bugs and should be avoided.
-                const updates = await runMetaComputationPass(dateToProcess, finalMetaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, false); 
-                Object.assign(dateUpdates, updates);
-            }
-        } catch (err) { 
-            logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message });
-            [...finalStandardToRun, ...finalMetaToRun].forEach(c => dateUpdates[normalizeName(c.name)] = false);
-        }
+    try {
+        const calcsRunning = [...standardToRun, ...metaToRun];
 
-        if (Object.keys(dateUpdates).length > 0) {
-            await updateComputationStatus(dateStr, dateUpdates, config, dependencies);
-        }
+        // Fetch dependencies (results from this day or yesterday)
+        const existingResults = await fetchExistingResults(dateStr, calcsRunning, computationManifest, config, dependencies, false);
+        const prevDate = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+        const prevDateStr = prevDate.toISOString().slice(0, 10);
+        const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
 
-        return { date: dateStr, updates: dateUpdates };
-    };
+        if (standardToRun.length) {
+            const updates = await runStandardComputationPass(dateToProcess, standardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, false);
+            Object.assign(dateUpdates, updates);
+        }
+        if (metaToRun.length) {
+            // runMetaComputationPass uses the Controller, which handles Price Sharding logic internally for single dates.
+            const updates = await runMetaComputationPass(dateToProcess, metaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, false);
+            Object.assign(dateUpdates, updates);
+        }
+    } catch (err) {
+        logger.log('ERROR', `[DateRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message });
+        [...standardToRun, ...metaToRun].forEach(c => dateUpdates[normalizeName(c.name)] = false);
+        throw err; // Re-throw to trigger Pub/Sub retry
+    }
 
-    // Batch process dates
-    for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) { 
-        const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
-        await Promise.all(batch.map(processDate));
+    if (Object.keys(dateUpdates).length > 0) {
+        await updateComputationStatus(dateStr, dateUpdates, config, dependencies);
     }
-    
-    logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
+
+    return { date: dateStr, updates: dateUpdates };
 }
 
-module.exports = { runComputationPass };
+module.exports = { runComputationPass, runDateComputation };

package/functions/computation-system/helpers/computation_worker.js

@@ -0,0 +1,56 @@
+/**
+ * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_worker.js
+ * PURPOSE: Consumes computation tasks from Pub/Sub and executes them.
+ */
+
+const { runDateComputation } = require('./computation_pass_runner.js');
+const { groupByPass } = require('./orchestration_helpers.js');
+
+/**
+ * Handles a single Pub/Sub message for a computation task.
+ */
+async function handleComputationTask(message, config, dependencies, computationManifest) {
+    const { logger } = dependencies;
+
+    try {
+        const data = message.json || message; // Handle both raw payload and parsed JSON
+
+        if (data.action !== 'RUN_COMPUTATION_DATE') {
+            logger.log('WARN', `[Worker] Unknown action: ${data.action}. Ignoring.`);
+            return;
+        }
+
+        const { date, pass } = data;
+
+        if (!date || !pass) {
+            logger.log('ERROR', `[Worker] Missing date or pass in payload: ${JSON.stringify(data)}`);
+            return;
+        }
+
+        logger.log('INFO', `[Worker] Received task: Date=${date}, Pass=${pass}`);
+
+        // Resolve calculations for this pass
+        const passes = groupByPass(computationManifest);
+        const calcsInThisPass = passes[pass] || [];
+
+        if (!calcsInThisPass.length) {
+            logger.log('WARN', `[Worker] No calculations found for Pass ${pass}.`);
+            return;
+        }
+
+        // Execute the computation for this specific date
+        const result = await runDateComputation(date, pass, calcsInThisPass, config, dependencies, computationManifest);
+
+        if (result) {
+            logger.log('INFO', `[Worker] Successfully processed ${date} (Pass ${pass}). Updates: ${Object.keys(result.updates || {}).length}`);
+        } else {
+            logger.log('INFO', `[Worker] Processed ${date} (Pass ${pass}) - No action needed or data missing.`);
+        }
+
+    } catch (err) {
+        logger.log('ERROR', `[Worker] Fatal error processing task: ${err.message}`, { stack: err.stack });
+        throw err; // Re-throw to trigger Pub/Sub retry
+    }
+}
+
+module.exports = { handleComputationTask };

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,8 +1,8 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/orchestration_helpers.js
- * FIXED: TS Error (controller.loader.mappings)
- * ADDED: Smart Shard Lookup for specific tickers
- * OPTIMIZED: Added Concurrency (Parallel Commits & Pipelined Shards) for runBatchPriceComputation
+ * FIXED: Context math mapping in runBatchPriceComputation (resolves 'undefined' crash).
+ * IMPROVED: Explicit logging for every calculation run (Start, Success, Failure).
+ * OPTIMIZED: Parallel Commits & Pipelined Shards.
  */
 
 const { ComputationController }              = require('../controllers/computation_controller');
@@ -14,7 +14,14 @@ const {
     getRelevantShardRefs, loadDataByRefs
 } = require('../utils/data_loader');
 
-const pLimit = require('p-limit'); // TODO, THIS OUGHT TO BE INJECTED.
+// --- FIX 1: Import Math Layer Primitives for Correct Context Mapping ---
+const { 
+    DataExtractor, HistoryExtractor, MathPrimitives, Aggregators, 
+    Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics, 
+    TimeSeries, priceExtractor 
+} = require('../layers/math_primitives.js');
+
+const pLimit = require('p-limit'); 
 
 /**
  * Groups calculations from a manifest by their 'pass' property.
@@ -22,29 +29,23 @@ const pLimit = require('p-limit'); // TODO, THIS OUGHT TO BE INJECTED.
 function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
 
 /**
- * --- NEW HELPER: PASSIVE DATA VALIDATION ---
+ * --- PASSIVE DATA VALIDATION ---
  * Scans a result set for suspicious patterns (e.g., a field is NULL for 100% of tickers).
- * Logs warnings but DOES NOT block the commit.
  */
-function validateResultPatterns(logger, calcName, results, category) { // TODO, THIS COULD BE MUCH MORE SOPHISTICATED, WE WILL NEVER FORCE FAIL A COMPUTATION REGARDLESS, BUT IT COULD BE A GREAT WARNING SYSTEM, USE GLOUD LOG SINKS TO DETECT PROBLEMS BASED ON THESE LOGS
-    // 1. Skip Speculators (Too sparse, nulls are expected)
+function validateResultPatterns(logger, calcName, results, category) { 
     if (category === 'speculator' || category === 'speculators') return;
 
     const tickers = Object.keys(results);
     const totalItems = tickers.length;
     
-    // 2. Need a decent sample size to judge patterns
     if (totalItems < 5) return; 
 
-    // 3. Get all keys from the first valid object
-    // We assume schema is roughly consistent across tickers
     const sampleTicker = tickers.find(t => results[t] && typeof results[t] === 'object');
     if (!sampleTicker) return;
     
     const keys = Object.keys(results[sampleTicker]);
     
     keys.forEach(key => {
-        // Skip internal keys or metadata
         if (key.startsWith('_')) return;
 
         let nullCount = 0;
@@ -58,14 +59,11 @@ function validateResultPatterns(logger, calcName, results, category) { // TODO,
             if (typeof val === 'number' && isNaN(val)) nanCount++;
         }
 
-        // 4. Define Thresholds
-        // If 100% of data is NaN or Undefined, that's almost certainly a bug.
         if (nanCount === totalItems) {
             logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is NaN for 100% of ${totalItems} items. Code bug likely.`);
         } else if (undefinedCount === totalItems) {
             logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is UNDEFINED for 100% of ${totalItems} items. Code bug likely.`);
         } 
-        // 5. Nulls are tricky. warn if >90%, but don't error (might be valid logic like "no shorts")
         else if (nullCount > (totalItems * 0.9)) {
             logger.log('WARN', `[DataQuality] Calc '${calcName}' field '${key}' is NULL for ${nullCount}/${totalItems} items. Check logic if this is unexpected.`);
         }
@@ -329,10 +327,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             }
 
             if (Object.keys(standardRes).length) {
-                // --- NEW: Run Passive Validation ---
-                // We do this BEFORE marking it completed, but we do NOT stop the write.
                 validateResultPatterns(deps.logger, name, standardRes, calc.manifest.category);
-                // -----------------------------------
 
                 standardRes._completed = true;
                 writes.push({
@@ -388,10 +383,10 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
  * OPTIMIZED: Implements concurrency for both Shard Processing and Write Commits
  */
 async function runBatchPriceComputation(config, deps, dateStrings, calcs, targetTickers = []) {
-    const { logger, db, calculationUtils } = deps; // Ensure calculationUtils is available for retry
+    const { logger, db, calculationUtils } = deps; 
     const controller = new ComputationController(config, deps);
     
-    // 1. FIX: Call loadMappings() correctly and get the result
+    // 1. Call loadMappings() correctly and get the result
     const mappings = await controller.loader.loadMappings(); 
     
     // 2. Resolve Shards (All or Subset)
@@ -413,12 +408,9 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
     }
 
     // 3. Execution Planning
-    // CONCURRENCY SETTING:
-    // Limit outer concurrency (processing shard chunks) to 2 to prevent contention on daily result docs.
-    // While Firestore handles concurrent writes to the same doc, limiting this avoids excessive retries.
     const OUTER_CONCURRENCY_LIMIT = 2; 
     const SHARD_BATCH_SIZE = 20;
-    const WRITE_BATCH_LIMIT = 50; // Keep write batch size small (payload safety)
+    const WRITE_BATCH_LIMIT = 50; 
 
     logger.log('INFO', `[BatchPrice] Execution Plan: ${dateStrings.length} days, ${allShardRefs.length} shards. Concurrency: ${OUTER_CONCURRENCY_LIMIT}.`);
 
@@ -440,7 +432,6 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
             // Optional Filtering for Subset Mode
             if (targetInstrumentIds.length > 0) {
                 const requestedSet = new Set(targetInstrumentIds);
-                // Iterate over the loaded data and delete anything we didn't ask for
                 for (const loadedInstrumentId in pricesData) {
                     if (!requestedSet.has(loadedInstrumentId)) {
                         delete pricesData[loadedInstrumentId];
@@ -450,26 +441,44 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
             const writes = [];
             
             // --- CALCULATION PHASE ---
-            // This builds up the array of writes (one per date)
             for (const dateStr of dateStrings) {
+                // --- FIX 2: Manually map math primitives to their alias names ---
+                // This matches the ContextBuilder logic in ComputationController
+                // and fixes the "Cannot read properties of undefined (reading 'standardDeviation')" error.
                 const context = {
                     mappings,
                     prices: { history: pricesData },
                     date: { today: dateStr },
-                    math: require('../layers/math_primitives.js')
+                    math: { 
+                        extract: DataExtractor, 
+                        history: HistoryExtractor, 
+                        compute: MathPrimitives, 
+                        aggregate: Aggregators, 
+                        validate: Validators, 
+                        signals: SignalPrimitives, 
+                        schemas: SCHEMAS, 
+                        distribution : DistributionAnalytics, 
+                        TimeSeries: TimeSeries, 
+                        priceExtractor : priceExtractor 
+                    }
                 };
 
                 for (const calcManifest of calcs) {
                     try {
+                        // --- LOGGING FIX: Log start of calculation ---
+                        logger.log('INFO', `[BatchPrice] >> Running ${calcManifest.name} for ${dateStr}...`);
+
                         const instance = new calcManifest.class();
                         await instance.process(context);
                         const result = await instance.getResult();
                         
+                        let hasContent = false;
                         if (result && Object.keys(result).length > 0) {
                             let dataToWrite = result;
                             if (result.by_instrument) dataToWrite = result.by_instrument; 
                             
                             if (Object.keys(dataToWrite).length > 0) {
+                                hasContent = true;
                                 const docRef = db.collection(config.resultsCollection).doc(dateStr)
                                     .collection(config.resultsSubcollection).doc(calcManifest.category)
                                     .collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
@@ -481,22 +490,28 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
                                 });
                             }
                         }
+
+                        // --- LOGGING FIX: Log success/completion ---
+                        if (hasContent) {
+                            logger.log('INFO', `[BatchPrice] \u2714 Finished ${calcManifest.name} for ${dateStr}. Found data.`);
+                        } else {
+                            logger.log('INFO', `[BatchPrice] - Finished ${calcManifest.name} for ${dateStr}. No result data.`);
+                        }
+
                     } catch (err) {
-                        logger.log('ERROR', `[BatchPrice] Calc ${calcManifest.name} failed for ${dateStr}`, { error: err.message });
+                        // --- LOGGING FIX: Explicit failure log ---
+                        logger.log('ERROR', `[BatchPrice] \u2716 Failed ${calcManifest.name} for ${dateStr}: ${err.message}`);
                     }
                 }
             }
             
             // --- PARALLEL COMMIT PHASE ---
-            // Instead of committing sequentially via commitBatchInChunks, we process these writes in parallel.
-            // Since each write targets a DIFFERENT date (different document), parallelizing this is safe and fast.
             if (writes.length > 0) {
                 const commitBatches = [];
                 for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) {
                     commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT));
                 }
 
-                // Use a higher concurrency for commits since they target disjoint documents
                 const commitLimit = pLimit(10); 
                 
                 await Promise.all(commitBatches.map((batchWrites, bIndex) => commitLimit(async () => {
@@ -507,7 +522,6 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
                         await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`);
                     } catch (commitErr) {
                         logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message });
-                        // We log but don't throw, to allow other batches to succeed
                     }
                 })));
             }

package/functions/core/utils/pubsub_utils.js

@@ -1,36 +1,82 @@
 /**
  * @fileoverview Core Pub/Sub utility functions.
- * REFACTORED: All functions are now stateless and receive dependencies.
- * 'pubsub' (PubSub instance) and 'logger' are passed via a 'dependencies' object.
+ * REFACTORED: Hybrid module supporting both Stateless functions and Stateful Class.
+ * Fixes "PubSubUtils is not a constructor" error.
  */
 
 /**
- * Publishes an array of tasks to a specified Pub/Sub topic in batches.
- * @async
- * @param {object} dependencies - Contains pubsub, logger.
- * @param {object} config - Configuration object.
- * @param {string} config.topicName - The name of the Pub/Sub topic.
- * @param {Array<object>} config.tasks - The tasks to publish.
- * @param {string} config.taskType - A descriptor for the task type (for logging).
- * @param {number} [config.maxPubsubBatchSize=500] - Max messages to publish in one client batch.
- * @returns {Promise<void>}
+ * Stateless Function: Publishes tasks in batches.
+ * @param {object} dependencies - { pubsub, logger }
+ * @param {object} config - { topicName, tasks, taskType, maxPubsubBatchSize }
  */
 async function batchPublishTasks(dependencies, config) {
     const { pubsub, logger } = dependencies;
     const { topicName, tasks, taskType, maxPubsubBatchSize = 500 } = config;
-    if (!tasks || tasks.length === 0) { logger.log('INFO',`[Core Utils] No ${taskType} tasks to publish to ${topicName}.`); return; }
-    logger.log('INFO',`[Core Utils] Publishing ${tasks.length} ${taskType} tasks to ${topicName}...`);
+
+    if (!tasks || tasks.length === 0) {
+        logger.log('INFO', `[Core Utils] No ${taskType} tasks to publish to ${topicName}.`);
+        return;
+    }
+
+    logger.log('INFO', `[Core Utils] Publishing ${tasks.length} ${taskType} tasks to ${topicName}...`);
     const topic = pubsub.topic(topicName);
     let messagesPublished = 0;
+
     try {
-        for (let i = 0; i < tasks.length; i += maxPubsubBatchSize) { const batchTasks = tasks.slice(i, i + maxPubsubBatchSize);
-            const batchPromises = batchTasks.map(task => { const dataBuffer = Buffer.from(JSON.stringify(task));
-                return topic.publishMessage({ data: dataBuffer }) .catch(err => logger.log('ERROR', `[Core Utils] Failed to publish single message for ${taskType}`, { error: err.message, task: task })); });
+        for (let i = 0; i < tasks.length; i += maxPubsubBatchSize) {
+            const batchTasks = tasks.slice(i, i + maxPubsubBatchSize);
+            const batchPromises = batchTasks.map(task => {
+                const dataBuffer = Buffer.from(JSON.stringify(task));
+                return topic.publishMessage({ data: dataBuffer })
+                    .catch(err => logger.log('ERROR', `[Core Utils] Failed to publish single message for ${taskType}`, { error: err.message, task: task }));
+            });
+
             await Promise.all(batchPromises);
             messagesPublished += batchTasks.length;
-            logger.log('TRACE', `[Core Utils] Published batch ${Math.ceil((i + 1) / maxPubsubBatchSize)} for ${taskType} (${batchTasks.length} messages)`); }
+            logger.log('TRACE', `[Core Utils] Published batch ${Math.ceil((i + 1) / maxPubsubBatchSize)} for ${taskType} (${batchTasks.length} messages)`);
+        }
         logger.log('SUCCESS', `[Core Utils] Finished publishing ${messagesPublished} ${taskType} tasks to ${topicName}.`);
-    } catch (error) { logger.log('ERROR', `[Core Utils] Error during batch publishing of ${taskType} tasks to ${topicName}`, { errorMessage: error.message }); throw error; }
+    } catch (error) {
+        logger.log('ERROR', `[Core Utils] Error during batch publishing of ${taskType} tasks to ${topicName}`, { errorMessage: error.message });
+        throw error;
+    }
+}
+
+/**
+ * Stateful Class Wrapper
+ * Allows usage like: const utils = new PubSubUtils(deps); utils.batchPublishTasks(...)
+ */
+class PubSubUtils {
+    constructor(dependencies) {
+        this.dependencies = dependencies;
+    }
+
+    /**
+     * Hybrid method: Supports both (config) and (dependencies, config) signatures.
+     */
+    async batchPublishTasks(arg1, arg2) {
+        // If called as (dependencies, config), use passed dependencies (Stateless/Legacy style)
+        if (arg2) {
+            return batchPublishTasks(arg1, arg2);
+        }
+        // If called as (config), use this.dependencies (Stateful style)
+        return batchPublishTasks(this.dependencies, arg1);
+    }
+
+    /**
+     * Helper for Computation System (Dispatcher)
+     * Maps (topic, messages) -> batchPublishTasks
+     */
+    async publishMessageBatch(topicName, messages) {
+        // Unpack {json: ...} structure if present
+        const tasks = messages.map(m => m.json || m);
+        const config = {
+            topicName,
+            tasks,
+            taskType: 'computation-batch'
+        };
+        return batchPublishTasks(this.dependencies, config);
+    }
 }
 
-module.exports = { batchPublishTasks };
+module.exports = { batchPublishTasks, PubSubUtils };

package/index.js

@@ -3,56 +3,77 @@
  * Export the pipes!
  */
 
-                            // Core
-const core =              { IntelligentHeaderManager : require('./functions/core/utils/intelligent_header_manager')                .IntelligentHeaderManager,
-                            IntelligentProxyManager  : require('./functions/core/utils/intelligent_proxy_manager')                 .IntelligentProxyManager,
-                            FirestoreBatchManager    : require('./functions/task-engine/utils/firestore_batch_manager')            .FirestoreBatchManager,
-                            firestoreUtils           : require('./functions/core/utils/firestore_utils'),
-                            pubsubUtils              : require('./functions/core/utils/pubsub_utils') };
-
-                            // Orchestrator
-const orchestrator =      { runDiscoveryOrchestrator : require('./functions/orchestrator/index')                                    .runDiscoveryOrchestrator,
-                            runUpdateOrchestrator    : require('./functions/orchestrator/index')                                    .runUpdateOrchestrator,
-                            checkDiscoveryNeed       : require('./functions/orchestrator/helpers/discovery_helpers')                .checkDiscoveryNeed,
-                            getDiscoveryCandidates   : require('./functions/orchestrator/helpers/discovery_helpers')                .getDiscoveryCandidates,
-                            dispatchDiscovery        : require('./functions/orchestrator/helpers/discovery_helpers')                .dispatchDiscovery,
-                            getUpdateTargets         : require('./functions/orchestrator/helpers/update_helpers')                   .getUpdateTargets,
-                            dispatchUpdates          : require('./functions/orchestrator/helpers/update_helpers')                   .dispatchUpdates };
-
-                            // Dispatcher
-const dispatcher =        { handleRequest            : require('./functions/dispatcher/index')                                      .handleRequest          ,
-                            dispatchTasksInBatches   : require('./functions/dispatcher/helpers/dispatch_helpers')                   .dispatchTasksInBatches };
-
-                            // Task Engine
-const taskEngine =        { handleRequest            : require('./functions/task-engine/handler_creator')                           .handleRequest ,
-                            handleDiscover           : require('./functions/task-engine/helpers/discover_helpers')                  .handleDiscover,
-                            handleVerify             : require('./functions/task-engine/helpers/verify_helpers')                    .handleVerify  ,
-                            handleUpdate             : require('./functions/task-engine/helpers/update_helpers')                    .handleUpdate };
+// Import the PubSub Module
+const pubsubModule = require('./functions/core/utils/pubsub_utils');
+
+// Core
+const core = {
+    IntelligentHeaderManager: require('./functions/core/utils/intelligent_header_manager').IntelligentHeaderManager,
+    IntelligentProxyManager: require('./functions/core/utils/intelligent_proxy_manager').IntelligentProxyManager,
+    FirestoreBatchManager: require('./functions/task-engine/utils/firestore_batch_manager').FirestoreBatchManager,
+    firestoreUtils: require('./functions/core/utils/firestore_utils'),
+
+    // EXPORT FIX:
+    pubsubUtils: pubsubModule,              // Keeps stateless function access
+    PubSubUtils: pubsubModule.PubSubUtils   // Exposes the Class for 'new pipe.core.PubSubUtils()'
+};
+
+// Orchestrator
+const orchestrator = {
+    runDiscoveryOrchestrator: require('./functions/orchestrator/index').runDiscoveryOrchestrator,
+    runUpdateOrchestrator: require('./functions/orchestrator/index').runUpdateOrchestrator,
+    checkDiscoveryNeed: require('./functions/orchestrator/helpers/discovery_helpers').checkDiscoveryNeed,
+    getDiscoveryCandidates: require('./functions/orchestrator/helpers/discovery_helpers').getDiscoveryCandidates,
+    dispatchDiscovery: require('./functions/orchestrator/helpers/discovery_helpers').dispatchDiscovery,
+    getUpdateTargets: require('./functions/orchestrator/helpers/update_helpers').getUpdateTargets,
+    dispatchUpdates: require('./functions/orchestrator/helpers/update_helpers').dispatchUpdates
+};
+
+// Dispatcher
+const dispatcher = {
+    handleRequest: require('./functions/dispatcher/index').handleRequest,
+    dispatchTasksInBatches: require('./functions/dispatcher/helpers/dispatch_helpers').dispatchTasksInBatches
+};
+
+// Task Engine
+const taskEngine = {
+    handleRequest: require('./functions/task-engine/handler_creator').handleRequest,
+    handleDiscover: require('./functions/task-engine/helpers/discover_helpers').handleDiscover,
+    handleVerify: require('./functions/task-engine/helpers/verify_helpers').handleVerify,
+    handleUpdate: require('./functions/task-engine/helpers/update_helpers').handleUpdate
+};
 
 // --- NEW IMPORT ---
 const { build: buildManifestFunc } = require('./functions/computation-system/helpers/computation_manifest_builder');
 
-                            // Computation System
-const computationSystem = { runComputationPass       : require('./functions/computation-system/helpers/computation_pass_runner')    .runComputationPass,    
-                            dataLoader               : require('./functions/computation-system/utils/data_loader'),
-                            computationUtils         : require('./functions/computation-system/utils/utils'),
-                            buildManifest            : buildManifestFunc 
-                        };
-
-                            // API
-const api =               { createApiApp             : require('./functions/generic-api/index')                                       .createApiApp,
-                            helpers                  : require('./functions/generic-api/helpers/api_helpers') };
- 
-                            // Maintenance
-const maintenance =       { runSpeculatorCleanup     : require('./functions/speculator-cleanup-orchestrator/helpers/cleanup_helpers') .runCleanup,
-                            handleInvalidSpeculator  : require('./functions/invalid-speculator-handler/helpers/handler_helpers')      .handleInvalidSpeculator,
-                            runFetchInsights         : require('./functions/fetch-insights/helpers/handler_helpers')                  .fetchAndStoreInsights,
-                            runFetchPrices           : require('./functions/etoro-price-fetcher/helpers/handler_helpers')             .fetchAndStorePrices,
-                            runSocialOrchestrator    : require('./functions/social-orchestrator/helpers/orchestrator_helpers')        .runSocialOrchestrator,
-                            handleSocialTask         : require('./functions/social-task-handler/helpers/handler_helpers')             .handleSocialTask,
-                            runBackfillAssetPrices   : require('./functions/price-backfill/helpers/handler_helpers')                  .runBackfillAssetPrices };
-
-                            // Proxy
-const proxy =             { handlePost               : require('./functions/appscript-api/index')                                      .handlePost };
-
-module.exports =          { pipe: { core, orchestrator, dispatcher, taskEngine, computationSystem, api, maintenance, proxy } };
+// Computation System
+const computationSystem = {
+    runComputationPass: require('./functions/computation-system/helpers/computation_pass_runner').runComputationPass,
+    dispatchComputationPass: require('./functions/computation-system/helpers/computation_dispatcher').dispatchComputationPass,
+    handleComputationTask: require('./functions/computation-system/helpers/computation_worker').handleComputationTask,
+    dataLoader: require('./functions/computation-system/utils/data_loader'),
+    computationUtils: require('./functions/computation-system/utils/utils'),
+    buildManifest: buildManifestFunc
+};
+
+// API
+const api = {
+    createApiApp: require('./functions/generic-api/index').createApiApp,
+    helpers: require('./functions/generic-api/helpers/api_helpers')
+};
+
+// Maintenance
+const maintenance = {
+    runSpeculatorCleanup: require('./functions/speculator-cleanup-orchestrator/helpers/cleanup_helpers').runCleanup,
+    handleInvalidSpeculator: require('./functions/invalid-speculator-handler/helpers/handler_helpers').handleInvalidSpeculator,
+    runFetchInsights: require('./functions/fetch-insights/helpers/handler_helpers').fetchAndStoreInsights,
+    runFetchPrices: require('./functions/etoro-price-fetcher/helpers/handler_helpers').fetchAndStorePrices,
+    runSocialOrchestrator: require('./functions/social-orchestrator/helpers/orchestrator_helpers').runSocialOrchestrator,
+    handleSocialTask: require('./functions/social-task-handler/helpers/handler_helpers').handleSocialTask,
+    runBackfillAssetPrices: require('./functions/price-backfill/helpers/handler_helpers').runBackfillAssetPrices
+};
+
+// Proxy
+const proxy = { handlePost: require('./functions/appscript-api/index').handlePost };
+
+module.exports = { pipe: { core, orchestrator, dispatcher, taskEngine, computationSystem, api, maintenance, proxy } };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.199",
+  "version": "1.0.201",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [