bulltrackers-module 1.0.186 → 1.0.187

package/functions/computation-system/controllers/computation_controller.js

@@ -1,6 +1,6 @@
 /**
  * FIXED: computation_controller.js
- * V3.4: Adds Price Loading & Context Injection.
+ * V4.0: Implements Batch/Sharded Execution for Price Dependencies to prevent OOM.
  */
 
 const { DataExtractor, 
@@ -43,36 +43,54 @@ class DataLoader {
         this.cache.social.set(dateStr, social);
         return social;
     }
+    
     /**
-     * NEW: Loads sharded price data for Meta calculations
+     * NEW: Get references to all price shards without loading data.
+     */
+    async getPriceShardReferences() {
+        const { db, logger } = this.deps;
+        const collection = this.config.priceCollection || 'asset_prices';
+        try {
+            const refs = await db.collection(collection).listDocuments();
+            logger.log('INFO', `[DataLoader] Found ${refs.length} price shards to process.`);
+            return refs;
+        } catch (e) {
+            logger.log('ERROR', `[DataLoader] Failed to list price shards: ${e.message}`);
+            return [];
+        }
+    }
+
+    /**
+     * NEW: Load a single price shard.
+     */
+    async loadPriceShard(docRef) {
+        try {
+            const snap = await docRef.get();
+            if (!snap.exists) return {};
+            return snap.data();
+        } catch (e) {
+            console.error(`Error loading shard ${docRef.path}:`, e);
+            return {};
+        }
+    }
+
+    /**
+     * LEGACY: Kept for non-batched calls if ever needed, but effectively replaced by batching.
      */
     async loadPrices() {
-        if (this.cache.prices) return this.cache.prices;
         const { db, logger } = this.deps;
         const collection = this.config.priceCollection || 'asset_prices';
-        
-        logger.log('INFO', `[DataLoader] Loading all price shards from ${collection}...`);
-        
+        logger.log('WARN', `[DataLoader] loadPrices() called. This is memory intensive!`);
         try {
             const snapshot = await db.collection(collection).get();
             if (snapshot.empty) return { history: {} };
-
             const historyMap = {};
-            
             snapshot.forEach(doc => {
                 const shardData = doc.data();
-                // Merge shard keys (instrumentIds) into main map
                 if (shardData) Object.assign(historyMap, shardData);
             });
-
-            logger.log('INFO', `[DataLoader] Loaded prices for ${Object.keys(historyMap).length} instruments.`);
-            
-            // Cache as an object with 'history' map to match priceExtractor expectations
-            this.cache.prices = { history: historyMap };
-            return this.cache.prices;
-
+            return { history: historyMap };
         } catch (e) {
-            logger.log('ERROR', `[DataLoader] Failed to load prices: ${e.message}`);
             return { history: {} };
         }
     }
@@ -81,21 +99,9 @@ class DataLoader {
 class ContextBuilder {
     static buildPerUserContext(options) {
         const { 
-            todayPortfolio, 
-            yesterdayPortfolio, 
-            todayHistory, 
-            yesterdayHistory,
-            userId, 
-            userType, 
-            dateStr, 
-            metadata, 
-            mappings, 
-            insights, 
-            socialData,
-            computedDependencies, 
-            previousComputedDependencies,
-            config, 
-            deps
+            todayPortfolio, yesterdayPortfolio, todayHistory, yesterdayHistory,
+            userId, userType, dateStr, metadata, mappings, insights, socialData,
+            computedDependencies, previousComputedDependencies, config, deps
         } = options;
 
         return {
@@ -131,23 +137,16 @@ class ContextBuilder {
 
     static buildMetaContext(options) {
         const {
-            dateStr, 
-            metadata, 
-            mappings, 
-            insights, 
-            socialData,
-            prices, // <--- ADDED THIS
-            computedDependencies, 
-            previousComputedDependencies,
-            config, 
-            deps
+            dateStr, metadata, mappings, insights, socialData,
+            prices, 
+            computedDependencies, previousComputedDependencies, config, deps
         } = options;
 
         return {
             date: { today: dateStr },
             insights: { today: insights?.today, yesterday: insights?.yesterday },
             social: { today: socialData?.today, yesterday: socialData?.yesterday },
-            prices: prices || {}, // <--- INJECTED HERE
+            prices: prices || {},
             mappings: mappings || {},
             math: { 
                 extract: DataExtractor,
@@ -207,27 +206,70 @@ class ComputationExecutor {
         }
     }
 
+    /**
+     * REFACTORED: Batched execution for Price dependencies
+     */
     async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps) {
         const mappings = await this.loader.loadMappings();
+        const { logger } = this.deps;
         
-        // Load standard dependencies
         const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await this.loader.loadInsights(dateStr) } : null;
         const social = metadata.rootDataDependencies?.includes('social') ? { today: await this.loader.loadSocial(dateStr) } : null;
         
-        // NEW: Load Price dependencies if required
-        let prices = null;
+        // CHECK: Does this calculation require price history?
         if (metadata.rootDataDependencies?.includes('price')) {
-            prices = await this.loader.loadPrices();
-        }
+            logger.log('INFO', `[Executor] Running Batched/Sharded Execution for ${metadata.name}`);
+            
+            // 1. Get Shard References (Low Memory)
+            const shardRefs = await this.loader.getPriceShardReferences();
+            
+            if (shardRefs.length === 0) {
+                 logger.log('WARN', '[Executor] No price shards found.');
+                 return {};
+            }
+
+            // 2. Iterate Shards (One at a time in memory)
+            let processedCount = 0;
+            for (const ref of shardRefs) {
+                // Load ONE shard
+                const shardData = await this.loader.loadPriceShard(ref);
+                
+                // Construct a "Partial Context" containing only this shard's prices
+                const partialContext = ContextBuilder.buildMetaContext({
+                    dateStr, metadata, mappings, insights, socialData: social,
+                    prices: { history: shardData }, // Inject partial history
+                    computedDependencies: computedDeps,
+                    previousComputedDependencies: prevDeps,
+                    config: this.config, deps: this.deps
+                });
+
+                // Process the chunk
+                // IMPORTANT: The calc instance must ACCUMULATE results, not overwrite.
+                await calcInstance.process(partialContext);
+                
+                // Force dereference for GC
+                partialContext.prices = null; 
+                processedCount++;
+                
+                if (processedCount % 10 === 0) {
+                     if (global.gc) { global.gc(); } // Optional: Hint GC if exposed
+                }
+            }
+            
+            logger.log('INFO', `[Executor] Finished Batched Execution for ${metadata.name} (${processedCount} shards).`);
+            return calcInstance.getResult ? await calcInstance.getResult() : {};
 
-        const context = ContextBuilder.buildMetaContext({
-            dateStr, metadata, mappings, insights, socialData: social,
-            prices, // Pass prices to builder
-            computedDependencies: computedDeps,
-            previousComputedDependencies: prevDeps,
-            config: this.config, deps: this.deps
-        });
-        return await calcInstance.process(context);
+        } else {
+            // Standard execution for non-price calculations
+            const context = ContextBuilder.buildMetaContext({
+                dateStr, metadata, mappings, insights, socialData: social,
+                prices: {}, 
+                computedDependencies: computedDeps,
+                previousComputedDependencies: prevDeps,
+                config: this.config, deps: this.deps
+            });
+            return await calcInstance.process(context);
+        }
     }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.186",
+  "version": "1.0.187",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [