bulltrackers-module 1.0.188 → 1.0.189

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -1,5 +1,6 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_pass_runner.js
+ * FIXED: Integrates 'runBatchPriceComputation' to prevent OOM on price calculations.
  */
 
 const { 
@@ -10,10 +11,11 @@ const {
     updateComputationStatus,
     runStandardComputationPass, 
     runMetaComputationPass,
-    checkRootDependencies
+    checkRootDependencies,
+    runBatchPriceComputation // NEW IMPORT
 } = require('./orchestration_helpers.js');
 
-const { getExpectedDateStrings, normalizeName, getEarliestDataDates } = require('../utils/utils.js');
+const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
 
 const PARALLEL_BATCH_SIZE = 7;
 
@@ -31,12 +33,10 @@ async function runComputationPass(config, dependencies, computationManifest) {
         history: new Date('2025-11-05T00:00:00Z'), 
         social: new Date('2025-10-30T00:00:00Z'), 
         insights: new Date('2025-08-26T00:00:00Z'),
-        price: new Date('2025-08-01T00:00:00Z')  // A few weeks before insights (earliest other data)
-
+        price: new Date('2025-08-01T00:00:00Z')
     };
     earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a,b) => a < b ? a : b);
 
-
     const passes = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
 
@@ -47,46 +47,102 @@ async function runComputationPass(config, dependencies, computationManifest) {
     const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
     const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
 
-    const standardCalcs = calcsInThisPass.filter(c => c.type === 'standard');
-    const metaCalcs     = calcsInThisPass.filter(c => c.type === 'meta');
+    // --- SEPARATION OF CONCERNS ---
+    // Identify calculations that require the Optimized Price Batch Runner
+    const priceBatchCalcs = calcsInThisPass.filter(c => 
+        c.type === 'meta' && 
+        c.rootDataDependencies && 
+        c.rootDataDependencies.includes('price')
+    );
+
+    // Identify calculations for the Standard Date-Loop Runner
+    const standardAndOtherMetaCalcs = calcsInThisPass.filter(c => !priceBatchCalcs.includes(c));
+
+
+    // ========================================================================
+    // 1. EXECUTE OPTIMIZED PRICE BATCH (Shard-First)
+    // ========================================================================
+    if (priceBatchCalcs.length > 0) {
+        logger.log('INFO', `[PassRunner] Detected ${priceBatchCalcs.length} Price-Meta calculations. Checking statuses...`);
+        
+        // Filter dates that actually need these calculations
+        // We do a quick serial check of status docs to avoid re-running satisfied dates
+        const datesNeedingPriceCalc = [];
+        
+        // Check statuses in chunks to avoid blowing up IO
+        const STATUS_CHECK_CHUNK = 20;
+        for (let i = 0; i < allExpectedDates.length; i += STATUS_CHECK_CHUNK) {
+            const dateChunk = allExpectedDates.slice(i, i + STATUS_CHECK_CHUNK);
+            await Promise.all(dateChunk.map(async (dateStr) => {
+                const status = await fetchComputationStatus(dateStr, config, dependencies);
+                // If ANY of the price calcs are missing/false, we run the batch for this date
+                const needsRun = priceBatchCalcs.some(c => status[normalizeName(c.name)] !== true);
+                if (needsRun) datesNeedingPriceCalc.push(dateStr);
+            }));
+        }
+
+        if (datesNeedingPriceCalc.length > 0) {
+            logger.log('INFO', `[PassRunner] >>> Starting Optimized Batch for ${datesNeedingPriceCalc.length} dates <<<`);
+            
+            // Execute the Shard-First Logic
+            await runBatchPriceComputation(config, dependencies, datesNeedingPriceCalc, priceBatchCalcs);
+
+            // Manually update statuses for these dates/calcs upon completion
+            // (runBatchPriceComputation handles the results, but we must mark the status doc)
+            logger.log('INFO', `[PassRunner] Updating status documents for batch...`);
+            
+            const BATCH_UPDATE_SIZE = 50;
+            for (let i = 0; i < datesNeedingPriceCalc.length; i += BATCH_UPDATE_SIZE) {
+                const updateChunk = datesNeedingPriceCalc.slice(i, i + BATCH_UPDATE_SIZE);
+                await Promise.all(updateChunk.map(async (dateStr) => {
+                    const updates = {};
+                    priceBatchCalcs.forEach(c => updates[normalizeName(c.name)] = true);
+                    await updateComputationStatus(dateStr, updates, config, dependencies);
+                }));
+            }
+            logger.log('INFO', `[PassRunner] >>> Optimized Batch Complete <<<`);
+        } else {
+            logger.log('INFO', `[PassRunner] All Price-Meta calculations are up to date.`);
+        }
+    }
+
+
+    // ========================================================================
+    // 2. EXECUTE STANDARD DATE LOOP (Date-First)
+    // ========================================================================
+    if (standardAndOtherMetaCalcs.length === 0) {
+        logger.log('INFO', `[PassRunner] No other calculations remaining. Exiting.`);
+        return;
+    }
+
+    const standardCalcs = standardAndOtherMetaCalcs.filter(c => c.type === 'standard');
+    const metaCalcs     = standardAndOtherMetaCalcs.filter(c => c.type === 'meta');
 
     // Process a single date
     const processDate = async (dateStr) => {
         const dateToProcess = new Date(dateStr + 'T00:00:00Z');
 
         // 1. Fetch Status for THIS specific date only
-        // This ensures Pass 2 sees exactly what Pass 1 wrote for this date.
         const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
 
-        // Helper: Check status using the fetched daily data
+        // Helper: Check status
         const shouldRun = (calc) => {
             const cName = normalizeName(calc.name);
-            
-            // A. If recorded as TRUE -> Ignore (already ran)
             if (dailyStatus[cName] === true) return false;
-
-            // B. If recorded as FALSE or UNDEFINED -> Run it (retry or new)
-            // But first, check if we have the necessary data dependencies.
             
             if (calc.dependencies && calc.dependencies.length > 0) {
-                // Check if prerequisites (from previous passes on THIS date) are complete
                 const missing = calc.dependencies.filter(depName => dailyStatus[normalizeName(depName)] !== true);
-                if (missing.length > 0) {
-                    // Dependency missing: cannot run yet.
-                    return false;
-                }
+                if (missing.length > 0) return false;
             }
-            
-            // If we are here, status is false/undefined AND dependencies are met.
             return true;
         };
 
         const standardToRun = standardCalcs.filter(shouldRun);
         const metaToRun     = metaCalcs.filter(shouldRun);
         
-        if (!standardToRun.length && !metaToRun.length) return null; // No work for this date
+        if (!standardToRun.length && !metaToRun.length) return null; 
 
-        // 2. Check Root Data Availability (Portfolio, History, etc.)
+        // 2. Check Root Data Availability
         const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
         if (!rootData) return null;
 
@@ -98,7 +154,7 @@ async function runComputationPass(config, dependencies, computationManifest) {
 
         logger.log('INFO', `[PassRunner] Running ${dateStr}: ${finalStandardToRun.length} std, ${finalMetaToRun.length} meta`);
 
-        const dateUpdates = {}; // { calcName: true/false }
+        const dateUpdates = {}; 
 
         try {
             const calcsRunning = [...finalStandardToRun, ...finalMetaToRun];
@@ -107,7 +163,6 @@ async function runComputationPass(config, dependencies, computationManifest) {
             const prevDateStr     = prevDate.toISOString().slice(0, 10);
             const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
             
-            // Note: We use skipStatusWrite=true because we want to batch write the status at the end of this function
             if (finalStandardToRun.length) {
                 const updates = await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, true); 
                 Object.assign(dateUpdates, updates);
@@ -121,7 +176,6 @@ async function runComputationPass(config, dependencies, computationManifest) {
             [...finalStandardToRun, ...finalMetaToRun].forEach(c => dateUpdates[normalizeName(c.name)] = false);
         }
 
-        // 4. Write "true" or "false" results for THIS specific date immediately
         if (Object.keys(dateUpdates).length > 0) {
             await updateComputationStatus(dateStr, dateUpdates, config, dependencies);
         }
@@ -138,4 +192,4 @@ async function runComputationPass(config, dependencies, computationManifest) {
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 
-module.exports = { runComputationPass };
+module.exports = { runComputationPass };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.188",
+  "version": "1.0.189",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [