bulltrackers-module 1.0.190 → 1.0.192

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -169,12 +169,13 @@ async function runComputationPass(config, dependencies, computationManifest) {
             const prevDateStr     = prevDate.toISOString().slice(0, 10);
             const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
             
+            // Changed skipstatus write to false to ensure updates are recorded
             if (finalStandardToRun.length) {
-                const updates = await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, true); 
+                const updates = await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, false); 
                 Object.assign(dateUpdates, updates);
             }
             if (finalMetaToRun.length) {
-                const updates = await runMetaComputationPass(dateToProcess, finalMetaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, true); 
+                const updates = await runMetaComputationPass(dateToProcess, finalMetaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, false); 
                 Object.assign(dateUpdates, updates);
             }
         } catch (err) { 

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -2,7 +2,7 @@
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/orchestration_helpers.js
  * FIXED: TS Error (controller.loader.mappings)
  * ADDED: Smart Shard Lookup for specific tickers
- * FIXED: Payload Size Limits & Crash Resilience in runBatchPriceComputation
+ * OPTIMIZED: Added Concurrency (Parallel Commits & Pipelined Shards) for runBatchPriceComputation
  */
 
 const { ComputationController }              = require('../controllers/computation_controller');
@@ -13,7 +13,7 @@ const {
     getHistoryPartRefs, streamPortfolioData, streamHistoryData,
     getRelevantShardRefs, loadDataByRefs
 } = require('../utils/data_loader');
-
+const pLimit = require('p-limit'); // Ensure p-limit is required
 
 /**
  * Groups calculations from a manifest by their 'pass' property.
@@ -328,10 +328,10 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 /**
  * --- UPDATED: runBatchPriceComputation ---
  * Now supports subset/specific ticker execution via 'targetTickers'
- * FIXED: Uses local batch config and try/catch for resilience.
+ * OPTIMIZED: Implements concurrency for both Shard Processing and Write Commits
  */
 async function runBatchPriceComputation(config, deps, dateStrings, calcs, targetTickers = []) {
-    const { logger, db } = deps;
+    const { logger, db, calculationUtils } = deps; // Ensure calculationUtils is available for retry
     const controller = new ComputationController(config, deps);
     
     // 1. FIX: Call loadMappings() correctly and get the result
@@ -340,17 +340,14 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
     // 2. Resolve Shards (All or Subset)
     let targetInstrumentIds = [];
     if (targetTickers && targetTickers.length > 0) {
-        // Convert Tickers -> InstrumentIDs
         const tickerToInst = mappings.tickerToInstrument || {};
         targetInstrumentIds = targetTickers.map(t => tickerToInst[t]).filter(id => id);
-        
         if (targetInstrumentIds.length === 0) {
             logger.log('WARN', '[BatchPrice] Target tickers provided but no IDs found. Aborting.');
             return;
         }
     }
     
-    // Uses the new data_loader function to look up specific shards if ids are present
     const allShardRefs = await getRelevantShardRefs(config, deps, targetInstrumentIds);
     
     if (!allShardRefs.length) {
@@ -358,73 +355,107 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
         return;
     }
 
-    // 3. Process in Chunks
+    // 3. Execution Planning
+    // CONCURRENCY SETTING:
+    // Limit outer concurrency (processing shard chunks) to 2 to prevent contention on daily result docs.
+    // While Firestore handles concurrent writes to the same doc, limiting this avoids excessive retries.
+    const OUTER_CONCURRENCY_LIMIT = 2; 
     const SHARD_BATCH_SIZE = 20;
-    logger.log('INFO', `[BatchPrice] Execution Plan: ${dateStrings.length} days, ${allShardRefs.length} shards.`);
+    const WRITE_BATCH_LIMIT = 50; // Keep write batch size small (payload safety)
 
+    logger.log('INFO', `[BatchPrice] Execution Plan: ${dateStrings.length} days, ${allShardRefs.length} shards. Concurrency: ${OUTER_CONCURRENCY_LIMIT}.`);
+
+    // 4. Create Chunks of Shards
+    const shardChunks = [];
     for (let i = 0; i < allShardRefs.length; i += SHARD_BATCH_SIZE) {
-        const shardChunkRefs = allShardRefs.slice(i, i + SHARD_BATCH_SIZE);
-        logger.log('INFO', `[BatchPrice] Processing chunk ${Math.floor(i/SHARD_BATCH_SIZE) + 1} (${shardChunkRefs.length} shards)...`);
-        
-        const pricesData = await loadDataByRefs(config, deps, shardChunkRefs);
-        
-        // --- FILTERING (Optional but Recommended) ---
-        if (targetInstrumentIds.length > 0) {
-           // (Filtering logic omitted for brevity as per previous implementation)
-        }
+        shardChunks.push(allShardRefs.slice(i, i + SHARD_BATCH_SIZE));
+    }
 
-        const writes = [];
-        
-        for (const dateStr of dateStrings) {
-            const context = {
-                mappings,
-                prices: { history: pricesData },
-                date: { today: dateStr },
-                math: require('../layers/math_primitives.js')
-            };
-
-            for (const calcManifest of calcs) {
-                try {
-                    const instance = new calcManifest.class();
-                    await instance.process(context);
-                    const result = await instance.getResult();
-                    
-                    if (result && Object.keys(result).length > 0) {
-                        let dataToWrite = result;
-                        if (result.by_instrument) dataToWrite = result.by_instrument; 
+    const outerLimit = pLimit(OUTER_CONCURRENCY_LIMIT);
+
+    // 5. Process Shard Chunks Concurrently
+    const chunkPromises = shardChunks.map((shardChunkRefs, index) => outerLimit(async () => {
+        try {
+            logger.log('INFO', `[BatchPrice] Processing chunk ${index + 1}/${shardChunks.length} (${shardChunkRefs.length} shards)...`);
+            
+            const pricesData = await loadDataByRefs(config, deps, shardChunkRefs);
+            
+            // Optional Filtering for Subset Mode
+            if (targetInstrumentIds.length > 0) {
+                // (Logic omitted for brevity, but safe to include if strictly needed)
+            }
+
+            const writes = [];
+            
+            // --- CALCULATION PHASE ---
+            // This builds up the array of writes (one per date)
+            for (const dateStr of dateStrings) {
+                const context = {
+                    mappings,
+                    prices: { history: pricesData },
+                    date: { today: dateStr },
+                    math: require('../layers/math_primitives.js')
+                };
+
+                for (const calcManifest of calcs) {
+                    try {
+                        const instance = new calcManifest.class();
+                        await instance.process(context);
+                        const result = await instance.getResult();
                         
-                        if (Object.keys(dataToWrite).length > 0) {
-                            const docRef = db.collection(config.resultsCollection).doc(dateStr)
-                                .collection(config.resultsSubcollection).doc(calcManifest.category)
-                                .collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
+                        if (result && Object.keys(result).length > 0) {
+                            let dataToWrite = result;
+                            if (result.by_instrument) dataToWrite = result.by_instrument; 
                             
-                            writes.push({
-                                ref: docRef,
-                                data: { ...dataToWrite, _completed: true },
-                                options: { merge: true }
-                            });
+                            if (Object.keys(dataToWrite).length > 0) {
+                                const docRef = db.collection(config.resultsCollection).doc(dateStr)
+                                    .collection(config.resultsSubcollection).doc(calcManifest.category)
+                                    .collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
+                                
+                                writes.push({
+                                    ref: docRef,
+                                    data: { ...dataToWrite, _completed: true },
+                                    options: { merge: true }
+                                });
+                            }
                         }
+                    } catch (err) {
+                        logger.log('ERROR', `[BatchPrice] Calc ${calcManifest.name} failed for ${dateStr}`, { error: err.message });
                     }
-                } catch (err) {
-                    logger.log('ERROR', `[BatchPrice] Calc ${calcManifest.name} failed for ${dateStr}`, { error: err.message });
                 }
             }
-        }
-        
-        if (writes.length > 0) {
-            // FIX: Use a lower batch limit for price batches because these result documents are aggregates
-            // and often exceed the 10MB payload limit when batched aggressively (450).
-            const safeBatchConfig = { ...config, batchSizeLimit: 50 };
-
-            try {
-                await commitBatchInChunks(safeBatchConfig, deps, writes, `BatchPrice Chunk ${Math.floor(i/SHARD_BATCH_SIZE)}`);
-            } catch (err) {
-                // FIX: Catch the commit failure (e.g. payload size exceeded) but DO NOT CRASH the whole system.
-                // Log it and allow the next chunk of shards to process.
-                logger.log('ERROR', `[BatchPrice] Failed to commit results for chunk ${Math.floor(i/SHARD_BATCH_SIZE)}. Proceeding to next chunk.`, { error: err.message });
+            
+            // --- PARALLEL COMMIT PHASE ---
+            // Instead of committing sequentially via commitBatchInChunks, we process these writes in parallel.
+            // Since each write targets a DIFFERENT date (different document), parallelizing this is safe and fast.
+            if (writes.length > 0) {
+                const commitBatches = [];
+                for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) {
+                    commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT));
+                }
+
+                // Use a higher concurrency for commits since they target disjoint documents
+                const commitLimit = pLimit(10); 
+                
+                await Promise.all(commitBatches.map((batchWrites, bIndex) => commitLimit(async () => {
+                    const batch = db.batch();
+                    batchWrites.forEach(w => batch.set(w.ref, w.data, w.options));
+                    
+                    try {
+                        await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`);
+                    } catch (commitErr) {
+                        logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message });
+                        // We log but don't throw, to allow other batches to succeed
+                    }
+                })));
             }
+
+        } catch (chunkErr) {
+            logger.log('ERROR', `[BatchPrice] Fatal error processing Chunk ${index}.`, { error: chunkErr.message });
         }
-    }
+    }));
+
+    await Promise.all(chunkPromises);
     logger.log('INFO', '[BatchPrice] Optimization pass complete.');
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.190",
+  "version": "1.0.192",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [