bulltrackers-module 1.0.185 → 1.0.187

package/functions/computation-system/controllers/computation_controller.js

@@ -1,6 +1,6 @@
 /**
  * FIXED: computation_controller.js
- * V3.4: Adds Price Loading & Context Injection.
+ * V4.0: Implements Batch/Sharded Execution for Price Dependencies to prevent OOM.
  */
 
 const { DataExtractor, 
@@ -43,36 +43,54 @@ class DataLoader {
         this.cache.social.set(dateStr, social);
         return social;
     }
+    
     /**
-     * NEW: Loads sharded price data for Meta calculations
+     * NEW: Get references to all price shards without loading data.
+     */
+    async getPriceShardReferences() {
+        const { db, logger } = this.deps;
+        const collection = this.config.priceCollection || 'asset_prices';
+        try {
+            const refs = await db.collection(collection).listDocuments();
+            logger.log('INFO', `[DataLoader] Found ${refs.length} price shards to process.`);
+            return refs;
+        } catch (e) {
+            logger.log('ERROR', `[DataLoader] Failed to list price shards: ${e.message}`);
+            return [];
+        }
+    }
+
+    /**
+     * NEW: Load a single price shard.
+     */
+    async loadPriceShard(docRef) {
+        try {
+            const snap = await docRef.get();
+            if (!snap.exists) return {};
+            return snap.data();
+        } catch (e) {
+            console.error(`Error loading shard ${docRef.path}:`, e);
+            return {};
+        }
+    }
+
+    /**
+     * LEGACY: Kept for non-batched calls if ever needed, but effectively replaced by batching.
      */
     async loadPrices() {
-        if (this.cache.prices) return this.cache.prices;
         const { db, logger } = this.deps;
         const collection = this.config.priceCollection || 'asset_prices';
-        
-        logger.log('INFO', `[DataLoader] Loading all price shards from ${collection}...`);
-        
+        logger.log('WARN', `[DataLoader] loadPrices() called. This is memory intensive!`);
         try {
             const snapshot = await db.collection(collection).get();
             if (snapshot.empty) return { history: {} };
-
             const historyMap = {};
-            
             snapshot.forEach(doc => {
                 const shardData = doc.data();
-                // Merge shard keys (instrumentIds) into main map
                 if (shardData) Object.assign(historyMap, shardData);
             });
-
-            logger.log('INFO', `[DataLoader] Loaded prices for ${Object.keys(historyMap).length} instruments.`);
-            
-            // Cache as an object with 'history' map to match priceExtractor expectations
-            this.cache.prices = { history: historyMap };
-            return this.cache.prices;
-
+            return { history: historyMap };
         } catch (e) {
-            logger.log('ERROR', `[DataLoader] Failed to load prices: ${e.message}`);
             return { history: {} };
         }
     }
@@ -81,21 +99,9 @@ class DataLoader {
 class ContextBuilder {
     static buildPerUserContext(options) {
         const { 
-            todayPortfolio, 
-            yesterdayPortfolio, 
-            todayHistory, 
-            yesterdayHistory,
-            userId, 
-            userType, 
-            dateStr, 
-            metadata, 
-            mappings, 
-            insights, 
-            socialData,
-            computedDependencies, 
-            previousComputedDependencies,
-            config, 
-            deps
+            todayPortfolio, yesterdayPortfolio, todayHistory, yesterdayHistory,
+            userId, userType, dateStr, metadata, mappings, insights, socialData,
+            computedDependencies, previousComputedDependencies, config, deps
         } = options;
 
         return {
@@ -131,23 +137,16 @@ class ContextBuilder {
 
     static buildMetaContext(options) {
         const {
-            dateStr, 
-            metadata, 
-            mappings, 
-            insights, 
-            socialData,
-            prices, // <--- ADDED THIS
-            computedDependencies, 
-            previousComputedDependencies,
-            config, 
-            deps
+            dateStr, metadata, mappings, insights, socialData,
+            prices, 
+            computedDependencies, previousComputedDependencies, config, deps
         } = options;
 
         return {
             date: { today: dateStr },
             insights: { today: insights?.today, yesterday: insights?.yesterday },
             social: { today: socialData?.today, yesterday: socialData?.yesterday },
-            prices: prices || {}, // <--- INJECTED HERE
+            prices: prices || {},
             mappings: mappings || {},
             math: { 
                 extract: DataExtractor,
@@ -207,27 +206,70 @@ class ComputationExecutor {
         }
     }
 
+    /**
+     * REFACTORED: Batched execution for Price dependencies
+     */
     async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps) {
         const mappings = await this.loader.loadMappings();
+        const { logger } = this.deps;
         
-        // Load standard dependencies
         const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await this.loader.loadInsights(dateStr) } : null;
         const social = metadata.rootDataDependencies?.includes('social') ? { today: await this.loader.loadSocial(dateStr) } : null;
         
-        // NEW: Load Price dependencies if required
-        let prices = null;
+        // CHECK: Does this calculation require price history?
         if (metadata.rootDataDependencies?.includes('price')) {
-            prices = await this.loader.loadPrices();
-        }
+            logger.log('INFO', `[Executor] Running Batched/Sharded Execution for ${metadata.name}`);
+            
+            // 1. Get Shard References (Low Memory)
+            const shardRefs = await this.loader.getPriceShardReferences();
+            
+            if (shardRefs.length === 0) {
+                 logger.log('WARN', '[Executor] No price shards found.');
+                 return {};
+            }
+
+            // 2. Iterate Shards (One at a time in memory)
+            let processedCount = 0;
+            for (const ref of shardRefs) {
+                // Load ONE shard
+                const shardData = await this.loader.loadPriceShard(ref);
+                
+                // Construct a "Partial Context" containing only this shard's prices
+                const partialContext = ContextBuilder.buildMetaContext({
+                    dateStr, metadata, mappings, insights, socialData: social,
+                    prices: { history: shardData }, // Inject partial history
+                    computedDependencies: computedDeps,
+                    previousComputedDependencies: prevDeps,
+                    config: this.config, deps: this.deps
+                });
+
+                // Process the chunk
+                // IMPORTANT: The calc instance must ACCUMULATE results, not overwrite.
+                await calcInstance.process(partialContext);
+                
+                // Force dereference for GC
+                partialContext.prices = null; 
+                processedCount++;
+                
+                if (processedCount % 10 === 0) {
+                     if (global.gc) { global.gc(); } // Optional: Hint GC if exposed
+                }
+            }
+            
+            logger.log('INFO', `[Executor] Finished Batched Execution for ${metadata.name} (${processedCount} shards).`);
+            return calcInstance.getResult ? await calcInstance.getResult() : {};
 
-        const context = ContextBuilder.buildMetaContext({
-            dateStr, metadata, mappings, insights, socialData: social,
-            prices, // Pass prices to builder
-            computedDependencies: computedDeps,
-            previousComputedDependencies: prevDeps,
-            config: this.config, deps: this.deps
-        });
-        return await calcInstance.process(context);
+        } else {
+            // Standard execution for non-price calculations
+            const context = ContextBuilder.buildMetaContext({
+                dateStr, metadata, mappings, insights, socialData: social,
+                prices: {}, 
+                computedDependencies: computedDeps,
+                previousComputedDependencies: prevDeps,
+                config: this.config, deps: this.deps
+            });
+            return await calcInstance.process(context);
+        }
     }
 }
 

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -25,9 +25,18 @@ async function runComputationPass(config, dependencies, computationManifest) {
     
     logger.log('INFO', `🚀 Starting PASS ${passToRun} (Targeting /computation_status/{YYYY-MM-DD})...`);
 
-    const earliestDates = await getEarliestDataDates(config, dependencies); // Now not hardcoded.
+    // Hardcoded earliest dates
+    const earliestDates = { 
+        portfolio: new Date('2025-09-25T00:00:00Z'), 
+        history: new Date('2025-11-05T00:00:00Z'), 
+        social: new Date('2025-10-30T00:00:00Z'), 
+        insights: new Date('2025-08-26T00:00:00Z'),
+        price: new Date('2025-08-01T00:00:00Z')  // A few weeks before insights (earliest other data)
+
+    };
     earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a,b) => a < b ? a : b);
 
+
     const passes = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
 

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -37,7 +37,7 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
  * Checks for the availability of all required root data for a specific date.
  */
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
-    const { logger, db } = dependencies;
+    const { logger } = dependencies;
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
     let portfolioRefs = [], historyRefs = [];
     let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
@@ -50,8 +50,8 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
         if (dateToProcess >= earliestDates.social) tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
         if (dateToProcess >= earliestDates.history) tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
         
-        // NEW: Check if price data exists (proper validation)
-        if (dateToProcess >= (earliestDates.price || earliestDates.absoluteEarliest)) {
+        // NEW: Check if price data exists - simple validation
+        if (dateToProcess >= earliestDates.price) {
             tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; }));
         }
         
@@ -74,14 +74,14 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
 }
 
 /**
- * NEW HELPER: Check if price data collection has any data
+ * NEW HELPER: Simple check if price collection has any data
  */
 async function checkPriceDataAvailability(config, dependencies) {
     const { db, logger } = dependencies;
     const collection = config.priceCollection || 'asset_prices';
     
     try {
-        // Check if the collection has at least one shard document
+        // Just check if the collection has at least one document
         const snapshot = await db.collection(collection).limit(1).get();
         
         if (snapshot.empty) {
@@ -89,16 +89,6 @@ async function checkPriceDataAvailability(config, dependencies) {
             return false;
         }
         
-        // Check if the first shard actually has price data
-        const firstDoc = snapshot.docs[0];
-        const data = firstDoc.data();
-        
-        if (!data || Object.keys(data).length === 0) {
-            logger.log('WARN', `[checkPriceData] Price shard exists but is empty`);
-            return false;
-        }
-        
-        logger.log('TRACE', `[checkPriceData] Price data available in ${collection}`);
         return true;
         
     } catch (e) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.185",
+  "version": "1.0.187",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [