bulltrackers-module 1.0.204 → 1.0.205

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,27 +1,26 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/orchestration_helpers.js
- * FIXED: Context math mapping in runBatchPriceComputation (resolves 'undefined' crash).
- * IMPROVED: Explicit logging for every calculation run (Start, Success, Failure).
- * OPTIMIZED: Parallel Commits & Pipelined Shards.
+ * FIXED: 'commitResults' now isolates commits PER COMPUTATION.
+ * A single failure (e.g., size limit) will only fail that specific calculation,
+ * allowing others in the same pass/date to succeed and be recorded.
  */
 
-const { ComputationController }              = require('../controllers/computation_controller');
-const { batchStoreSchemas }                  = require('../utils/schema_capture');
+const { ComputationController } = require('../controllers/computation_controller');
+const { batchStoreSchemas } = require('../utils/schema_capture');
 const { normalizeName, commitBatchInChunks } = require('../utils/utils');
-const { 
-    getPortfolioPartRefs, loadDailyInsights, loadDailySocialPostInsights, 
+const {
+    getPortfolioPartRefs, loadDailyInsights, loadDailySocialPostInsights,
     getHistoryPartRefs, streamPortfolioData, streamHistoryData,
     getRelevantShardRefs, loadDataByRefs
 } = require('../utils/data_loader');
 
-// --- FIX 1: Import Math Layer Primitives for Correct Context Mapping ---
-const { 
-    DataExtractor, HistoryExtractor, MathPrimitives, Aggregators, 
-    Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics, 
-    TimeSeries, priceExtractor 
+const {
+    DataExtractor, HistoryExtractor, MathPrimitives, Aggregators,
+    Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics,
+    TimeSeries, priceExtractor
 } = require('../layers/math_primitives.js');
 
-const pLimit = require('p-limit'); 
+const pLimit = require('p-limit');
 
 /**
  * Groups calculations from a manifest by their 'pass' property.
@@ -30,21 +29,20 @@ function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[ca
 
 /**
  * --- PASSIVE DATA VALIDATION ---
- * Scans a result set for suspicious patterns (e.g., a field is NULL for 100% of tickers).
  */
-function validateResultPatterns(logger, calcName, results, category) { 
+function validateResultPatterns(logger, calcName, results, category) {
     if (category === 'speculator' || category === 'speculators') return;
 
     const tickers = Object.keys(results);
     const totalItems = tickers.length;
-    
-    if (totalItems < 5) return; 
+
+    if (totalItems < 5) return;
 
     const sampleTicker = tickers.find(t => results[t] && typeof results[t] === 'object');
     if (!sampleTicker) return;
-    
+
     const keys = Object.keys(results[sampleTicker]);
-    
+
     keys.forEach(key => {
         if (key.startsWith('_')) return;
 
@@ -60,28 +58,25 @@ function validateResultPatterns(logger, calcName, results, category) {
         }
 
         if (nanCount === totalItems) {
-            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is NaN for 100% of ${totalItems} items. Code bug likely.`);
+            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is NaN for 100% of ${totalItems} items.`);
         } else if (undefinedCount === totalItems) {
-            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is UNDEFINED for 100% of ${totalItems} items. Code bug likely.`);
-        } 
+            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is UNDEFINED for 100% of ${totalItems} items.`);
+        }
         else if (nullCount > (totalItems * 0.9)) {
-            logger.log('WARN', `[DataQuality] Calc '${calcName}' field '${key}' is NULL for ${nullCount}/${totalItems} items. Check logic if this is unexpected.`);
+            logger.log('WARN', `[DataQuality] Calc '${calcName}' field '${key}' is NULL for ${nullCount}/${totalItems} items.`);
         }
     });
 }
 
-/**
- * Checks if all root data dependencies for a given calculation are met.
- */
 function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
     if (!calcManifest.rootDataDependencies) return { canRun: true, missing };
     for (const dep of calcManifest.rootDataDependencies) {
-        if      (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
-        else if (dep === 'insights'  && !rootDataStatus.hasInsights)  missing.push('insights');
-        else if (dep === 'social'    && !rootDataStatus.hasSocial)    missing.push('social');
-        else if (dep === 'history'   && !rootDataStatus.hasHistory)   missing.push('history');
-        else if (dep === 'price'     && !rootDataStatus.hasPrices)    missing.push('price');
+        if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
+        else if (dep === 'insights' && !rootDataStatus.hasInsights) missing.push('insights');
+        else if (dep === 'social' && !rootDataStatus.hasSocial) missing.push('social');
+        else if (dep === 'history' && !rootDataStatus.hasHistory) missing.push('history');
+        else if (dep === 'price' && !rootDataStatus.hasPrices) missing.push('price');
     }
     return { canRun: missing.length === 0, missing };
 }
@@ -89,30 +84,30 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
     const { logger } = dependencies;
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-    let portfolioRefs   = [], historyRefs = [];
-    let hasPortfolio    = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
-    let insightsData    = null, socialData = null;
+    let portfolioRefs = [], historyRefs = [];
+    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
+    let insightsData = null, socialData = null;
 
     try {
         const tasks = [];
         if (dateToProcess >= earliestDates.portfolio) tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio = !!r.length; }));
-        if (dateToProcess >= earliestDates.insights)  tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
-        if (dateToProcess >= earliestDates.social)    tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
-        if (dateToProcess >= earliestDates.history)   tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
-        
+        if (dateToProcess >= earliestDates.insights) tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
+        if (dateToProcess >= earliestDates.social) tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
+        if (dateToProcess >= earliestDates.history) tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
+
         if (dateToProcess >= earliestDates.price) {
             tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; }));
         }
-        
+
         await Promise.all(tasks);
-        
+
         if (!(hasPortfolio || hasInsights || hasSocial || hasHistory || hasPrices)) return null;
-        
-        return { 
-            portfolioRefs, 
-            historyRefs, 
-            todayInsights: insightsData, 
-            todaySocialPostInsights: socialData, 
+
+        return {
+            portfolioRefs,
+            historyRefs,
+            todayInsights: insightsData,
+            todaySocialPostInsights: socialData,
             status: { hasPortfolio, hasInsights, hasSocial, hasHistory, hasPrices }
         };
 
@@ -140,8 +135,8 @@ async function checkPriceDataAvailability(config, dependencies) {
 
 async function fetchComputationStatus(dateStr, config, { db }) {
     const collection = config.computationStatusCollection || 'computation_status';
-    const docRef     = db.collection(collection).doc(dateStr);
-    const snap       = await docRef.get();
+    const docRef = db.collection(collection).doc(dateStr);
+    const snap = await docRef.get();
     return snap.exists ? snap.data() : {};
 }
 
@@ -155,8 +150,8 @@ async function fetchGlobalComputationStatus(config, { db }) {
 async function updateComputationStatus(dateStr, updates, config, { db }) {
     if (!updates || Object.keys(updates).length === 0) return;
     const collection = config.computationStatusCollection || 'computation_status';
-    const docRef     = db.collection(collection).doc(dateStr);
-    await docRef.set(updates, { merge: true }); 
+    const docRef = db.collection(collection).doc(dateStr);
+    await docRef.set(updates, { merge: true });
 }
 
 async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
@@ -172,38 +167,41 @@ async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
     try {
         await docRef.update(flattenUpdates);
     } catch (err) {
-        if (err.code === 5) { 
-             const deepObj = {};
-             for (const [date, statuses] of Object.entries(updatesByDate)) {
-                 deepObj[date] = statuses;
-             }
-             await docRef.set(deepObj, { merge: true });
+        if (err.code === 5) {
+            const deepObj = {};
+            for (const [date, statuses] of Object.entries(updatesByDate)) {
+                deepObj[date] = statuses;
+            }
+            await docRef.set(deepObj, { merge: true });
         } else {
-             throw err;
+            throw err;
         }
     }
 }
 
 async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
-    const manifestMap  = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
+    const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
     const calcsToFetch = new Set();
     for (const calc of calcsInPass) {
-        if (calc.dependencies)                { calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); }
+        if (calc.dependencies) { calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); }
         if (includeSelf && calc.isHistorical) { calcsToFetch.add(normalizeName(calc.name)); }
     }
     if (!calcsToFetch.size) return {};
     const fetched = {};
     const docRefs = [];
-    const names   = [];
+    const names = [];
     for (const name of calcsToFetch) {
         const m = manifestMap.get(name);
-        if (m) { docRefs.push(db.collection(config.resultsCollection).doc(dateStr)
-                                .collection(config.resultsSubcollection).doc(m.category || 'unknown')
-                                .collection(config.computationsSubcollection).doc(name));
-                                names.push(name); } }
+        if (m) {
+            docRefs.push(db.collection(config.resultsCollection).doc(dateStr)
+                .collection(config.resultsSubcollection).doc(m.category || 'unknown')
+                .collection(config.computationsSubcollection).doc(name));
+            names.push(name);
+        }
+    }
     if (docRefs.length) {
         const snaps = await db.getAll(...docRefs);
-        snaps.forEach((doc, i) => {  if(doc.exists && doc.data()._completed) { fetched[names[i]] = doc.data();  } });
+        snaps.forEach((doc, i) => { if (doc.exists && doc.data()._completed) { fetched[names[i]] = doc.data(); } });
     }
     return fetched;
 }
@@ -212,8 +210,8 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     const { logger } = deps;
     const controller = new ComputationController(config, deps);
     const calcs = Object.values(state).filter(c => c && c.manifest);
-    const streamingCalcs = calcs.filter(c => 
-        c.manifest.rootDataDependencies.includes('portfolio') || 
+    const streamingCalcs = calcs.filter(c =>
+        c.manifest.rootDataDependencies.includes('portfolio') ||
         c.manifest.rootDataDependencies.includes('history')
     );
 
@@ -227,7 +225,7 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
 
     const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
     const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
-    const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs)  ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs)  : null;
+    const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
     const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
     const tH_iter = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
 
@@ -238,14 +236,14 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
         if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
         if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
 
-        const promises = streamingCalcs.map(calc => 
+        const promises = streamingCalcs.map(calc =>
             controller.executor.executePerUser(
                 calc,
                 calc.manifest,
                 dateStr,
                 tP_chunk,
-                yP_chunk, 
-                tH_chunk, 
+                yP_chunk,
+                tH_chunk,
                 fetchedDeps,
                 previousFetchedDeps
             )
@@ -260,20 +258,20 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
     const logger = deps.logger;
     const fullRoot = { ...rootData };
     if (calcs.some(c => c.isHistorical)) {
-        const prev    = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
+        const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
         const prevStr = prev.toISOString().slice(0, 10);
         fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
     }
 
     const state = {};
     for (const c of calcs) {
-        try { 
-            const inst = new c.class(); 
-            inst.manifest = c; 
-            state[normalizeName(c.name)] = inst; 
+        try {
+            const inst = new c.class();
+            inst.manifest = c;
+            state[normalizeName(c.name)] = inst;
             logger.log('INFO', `${c.name} calculation running for ${dStr}`);
-        } 
-        catch(e)  { logger.log('WARN', `Failed to init ${c.name}`); }
+        }
+        catch (e) { logger.log('WARN', `Failed to init ${c.name}`); }
     }
 
     await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps);
@@ -282,8 +280,8 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
 
 async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, previousFetchedDeps, rootData, skipStatusWrite = false) {
     const controller = new ComputationController(config, deps);
-    const dStr       = date.toISOString().slice(0, 10);
-    const state      = {};
+    const dStr = date.toISOString().slice(0, 10);
+    const state = {};
 
     for (const mCalc of calcs) {
         try {
@@ -297,28 +295,46 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
     return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
 }
 
+/**
+ * --- REFACTORED: commitResults ---
+ * Commits results individually per calculation.
+ * If one calculation fails (e.g. size limit), others still succeed.
+ */
 async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
-    const writes = [], schemas = [], sharded = {};
-    const successUpdates = {}; 
+    const successUpdates = {};
+    const schemas = [];
 
+    // Iterate PER CALCULATION to isolate failures
     for (const name in stateObj) {
         const calc = stateObj[name];
+        let hasData = false;
+
         try {
-            const result = await calc.getResult(); 
+            const result = await calc.getResult();
             if (!result) {
-                deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Failed (Empty Result)`);
+                deps.logger.log('INFO', `${name} for ${dStr}: Skipped (Empty Result)`);
                 continue;
             }
-            
+
             const standardRes = {};
-            let hasData = false;
+            const shardedWrites = [];
+            const calcWrites = []; // Accumulate all writes for THIS specific calculation
 
+            // 1. Separate Standard and Sharded Data
             for (const key in result) {
                 if (key.startsWith('sharded_')) {
                     const sData = result[key];
-                    for (const c in sData) { 
-                        sharded[c] = sharded[c] || {}; 
-                        Object.assign(sharded[c], sData[c]); 
+                    // sData structure: { CollectionName: { DocId: { ...data } } }
+                    for (const colName in sData) {
+                        const docsMap = sData[colName];
+                        for (const docId in docsMap) {
+                            // Support both full path or collection-relative path
+                            const ref = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(colName).doc(docId);
+                            shardedWrites.push({
+                                ref,
+                                data: { ...docsMap[docId], _completed: true }
+                            });
+                        }
                     }
                     if (Object.keys(sData).length > 0) hasData = true;
                 } else {
@@ -326,70 +342,75 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 }
             }
 
+            // 2. Prepare Standard Result Write
             if (Object.keys(standardRes).length) {
                 validateResultPatterns(deps.logger, name, standardRes, calc.manifest.category);
-
                 standardRes._completed = true;
-                writes.push({
-                    ref: deps.db.collection(config.resultsCollection).doc(dStr)
-                        .collection(config.resultsSubcollection).doc(calc.manifest.category)
-                        .collection(config.computationsSubcollection).doc(name),
+
+                const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
+                    .collection(config.resultsSubcollection).doc(calc.manifest.category)
+                    .collection(config.computationsSubcollection).doc(name);
+
+                calcWrites.push({
+                    ref: docRef,
                     data: standardRes
                 });
                 hasData = true;
             }
-            
+
+            // 3. Queue Schema (Safe to accumulate)
             if (calc.manifest.class.getSchema) {
                 const { class: _cls, ...safeMetadata } = calc.manifest;
-                schemas.push({ 
-                    name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata 
+                schemas.push({
+                    name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata
                 });
             }
-            
+
+            // 4. ATTEMPT COMMIT FOR THIS CALCULATION ONLY
             if (hasData) {
-                successUpdates[name] = true;
-                deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Succeeded`);
+                // Combine standard + sharded writes for this unit of work
+                const allWritesForCalc = [...calcWrites, ...shardedWrites];
+
+                if (allWritesForCalc.length > 0) {
+                    await commitBatchInChunks(config, deps, allWritesForCalc, `${name} Results`);
+
+                    // IF we get here, the commit succeeded.
+                    successUpdates[name] = true;
+                    deps.logger.log('INFO', `${name} for ${dStr}: \u2714 Success (Written)`);
+                } else {
+                    deps.logger.log('INFO', `${name} for ${dStr}: - No Data to Write`);
+                }
             } else {
-                deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Unknown (No Data Written)`);
+                deps.logger.log('INFO', `${name} for ${dStr}: - Empty`);
             }
 
-        } catch (e) { 
-            deps.logger.log('ERROR', `Commit failed ${name}: ${e.message}`);
-            deps.logger.log('INFO', `${name} calculation for ${dStr} ran, result : Failed (Exception)`);
+        } catch (e) {
+            // CRITICAL: Catch errors here so the loop continues for other calculations
+            deps.logger.log('ERROR', `${name} for ${dStr}: \u2716 FAILED Commit: ${e.message}`);
+            // Do NOT add to successUpdates
         }
     }
 
-    if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(()=>{});
-    if (writes.length) await commitBatchInChunks(config, deps, writes, `${passName} Results`);
-    for (const col in sharded) {
-        const sWrites = [];
-        for (const id in sharded[col]) {
-            const ref = id.includes('/') ? deps.db.doc(id) : deps.db.collection(col).doc(id);
-            sWrites.push({ ref, data: { ...sharded[col][id], _completed: true } });
-        }
-        if (sWrites.length) await commitBatchInChunks(config, deps, sWrites, `${passName} Sharded ${col}`);
-    }
+    // Save Schemas (Best effort, isolated)
+    if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(() => { });
 
+    // Update Status Document (Only for the ones that succeeded)
     if (!skipStatusWrite && Object.keys(successUpdates).length > 0) {
         await updateComputationStatus(dStr, successUpdates, config, deps);
-        deps.logger.log('INFO', `[${passName}] Updated status document for ${Object.keys(successUpdates).length} computations.`);
+        deps.logger.log('INFO', `[${passName}] Updated status document for ${Object.keys(successUpdates).length} successful computations.`);
     }
     return successUpdates;
 }
 
 /**
  * --- UPDATED: runBatchPriceComputation ---
- * Now supports subset/specific ticker execution via 'targetTickers'
- * OPTIMIZED: Implements concurrency for both Shard Processing and Write Commits
  */
 async function runBatchPriceComputation(config, deps, dateStrings, calcs, targetTickers = []) {
-    const { logger, db, calculationUtils } = deps; 
+    const { logger, db, calculationUtils } = deps;
     const controller = new ComputationController(config, deps);
-    
-    // 1. Call loadMappings() correctly and get the result
-    const mappings = await controller.loader.loadMappings(); 
-    
-    // 2. Resolve Shards (All or Subset)
+
+    const mappings = await controller.loader.loadMappings();
+
     let targetInstrumentIds = [];
     if (targetTickers && targetTickers.length > 0) {
         const tickerToInst = mappings.tickerToInstrument || {};
@@ -399,22 +420,20 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
             return;
         }
     }
-    
+
     const allShardRefs = await getRelevantShardRefs(config, deps, targetInstrumentIds);
-    
+
     if (!allShardRefs.length) {
         logger.log('WARN', '[BatchPrice] No relevant price shards found. Exiting.');
         return;
     }
 
-    // 3. Execution Planning
-    const OUTER_CONCURRENCY_LIMIT = 2; 
+    const OUTER_CONCURRENCY_LIMIT = 2;
     const SHARD_BATCH_SIZE = 20;
-    const WRITE_BATCH_LIMIT = 50; 
+    const WRITE_BATCH_LIMIT = 50;
 
     logger.log('INFO', `[BatchPrice] Execution Plan: ${dateStrings.length} days, ${allShardRefs.length} shards. Concurrency: ${OUTER_CONCURRENCY_LIMIT}.`);
 
-    // 4. Create Chunks of Shards
     const shardChunks = [];
     for (let i = 0; i < allShardRefs.length; i += SHARD_BATCH_SIZE) {
         shardChunks.push(allShardRefs.slice(i, i + SHARD_BATCH_SIZE));
@@ -422,114 +441,104 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
 
     const outerLimit = pLimit(OUTER_CONCURRENCY_LIMIT);
 
-    // 5. Process Shard Chunks Concurrently
-    const chunkPromises = shardChunks.map((shardChunkRefs, index) => outerLimit(async () => {
-        try {
-            logger.log('INFO', `[BatchPrice] Processing chunk ${index + 1}/${shardChunks.length} (${shardChunkRefs.length} shards)...`);
-            
-            const pricesData = await loadDataByRefs(config, deps, shardChunkRefs);
-            
-            // Optional Filtering for Subset Mode
-            if (targetInstrumentIds.length > 0) {
-                const requestedSet = new Set(targetInstrumentIds);
-                for (const loadedInstrumentId in pricesData) {
-                    if (!requestedSet.has(loadedInstrumentId)) {
-                        delete pricesData[loadedInstrumentId];
+    const chunkPromises = [];
+    for (let index = 0; index < shardChunks.length; index++) {
+        const shardChunkRefs = shardChunks[index];
+        chunkPromises.push(outerLimit(async () => {
+            try {
+                logger.log('INFO', `[BatchPrice] Processing chunk ${index + 1}/${shardChunks.length} (${shardChunkRefs.length} shards)...`);
+
+                const pricesData = await loadDataByRefs(config, deps, shardChunkRefs);
+
+                if (targetInstrumentIds.length > 0) {
+                    const requestedSet = new Set(targetInstrumentIds);
+                    for (const loadedInstrumentId in pricesData) {
+                        if (!requestedSet.has(loadedInstrumentId)) {
+                            delete pricesData[loadedInstrumentId];
+                        }
                     }
                 }
-            }
-            const writes = [];
-            
-            // --- CALCULATION PHASE ---
-            for (const dateStr of dateStrings) {
-                // --- FIX 2: Manually map math primitives to their alias names ---
-                // This matches the ContextBuilder logic in ComputationController
-                // and fixes the "Cannot read properties of undefined (reading 'standardDeviation')" error.
-                const context = {
-                    mappings,
-                    prices: { history: pricesData },
-                    date: { today: dateStr },
-                    math: { 
-                        extract: DataExtractor, 
-                        history: HistoryExtractor, 
-                        compute: MathPrimitives, 
-                        aggregate: Aggregators, 
-                        validate: Validators, 
-                        signals: SignalPrimitives, 
-                        schemas: SCHEMAS, 
-                        distribution : DistributionAnalytics, 
-                        TimeSeries: TimeSeries, 
-                        priceExtractor : priceExtractor 
-                    }
-                };
-
-                for (const calcManifest of calcs) {
-                    try {
-                        // --- LOGGING FIX: Log start of calculation ---
-                        logger.log('INFO', `[BatchPrice] >> Running ${calcManifest.name} for ${dateStr}...`);
-
-                        const instance = new calcManifest.class();
-                        await instance.process(context);
-                        const result = await instance.getResult();
-                        
-                        let hasContent = false;
-                        if (result && Object.keys(result).length > 0) {
-                            let dataToWrite = result;
-                            if (result.by_instrument) dataToWrite = result.by_instrument; 
-                            
-                            if (Object.keys(dataToWrite).length > 0) {
-                                hasContent = true;
-                                const docRef = db.collection(config.resultsCollection).doc(dateStr)
-                                    .collection(config.resultsSubcollection).doc(calcManifest.category)
-                                    .collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
-                                
-                                writes.push({
-                                    ref: docRef,
-                                    data: { ...dataToWrite, _completed: true },
-                                    options: { merge: true }
-                                });
-                            }
-                        }
 
-                        // --- LOGGING FIX: Log success/completion ---
-                        if (hasContent) {
-                            logger.log('INFO', `[BatchPrice] \u2714 Finished ${calcManifest.name} for ${dateStr}. Found data.`);
-                        } else {
-                            logger.log('INFO', `[BatchPrice] - Finished ${calcManifest.name} for ${dateStr}. No result data.`);
+                // We now accumulate writes per calc to allow partial success, though batching optimization is tricky here.
+                // For safety, let's keep the existing structure but wrap individual calc processing in try/catch 
+                // inside the write phase if possible. 
+                // However, runBatchPrice is optimized for BULK throughput. 
+                // To prevent total failure, we will use a safe array.
+                const writes = [];
+
+                for (const dateStr of dateStrings) {
+                    const context = {
+                        mappings,
+                        prices: { history: pricesData },
+                        date: { today: dateStr },
+                        math: {
+                            extract: DataExtractor,
+                            history: HistoryExtractor,
+                            compute: MathPrimitives,
+                            aggregate: Aggregators,
+                            validate: Validators,
+                            signals: SignalPrimitives,
+                            schemas: SCHEMAS,
+                            distribution: DistributionAnalytics,
+                            TimeSeries: TimeSeries,
+                            priceExtractor: priceExtractor
+                        }
+                    };
+
+                    for (const calcManifest of calcs) {
+                        try {
+                            // logger.log('INFO', `[BatchPrice] >> Running ${calcManifest.name} for ${dateStr}...`); // Verbose
+                            const instance = new calcManifest.class();
+                            await instance.process(context);
+                            const result = await instance.getResult();
+
+                            if (result && Object.keys(result).length > 0) {
+                                let dataToWrite = result;
+                                if (result.by_instrument) dataToWrite = result.by_instrument;
+
+                                if (Object.keys(dataToWrite).length > 0) {
+                                    const docRef = db.collection(config.resultsCollection).doc(dateStr)
+                                        .collection(config.resultsSubcollection).doc(calcManifest.category)
+                                        .collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
+
+                                    writes.push({
+                                        ref: docRef,
+                                        data: { ...dataToWrite, _completed: true },
+                                        options: { merge: true }
+                                    });
+                                }
+                            }
+                        } catch (err) {
+                            logger.log('ERROR', `[BatchPrice] \u2716 Failed ${calcManifest.name} for ${dateStr}: ${err.message}`);
                         }
-
-                    } catch (err) {
-                        // --- LOGGING FIX: Explicit failure log ---
-                        logger.log('ERROR', `[BatchPrice] \u2716 Failed ${calcManifest.name} for ${dateStr}: ${err.message}`);
                     }
                 }
-            }
-            
-            // --- PARALLEL COMMIT PHASE ---
-            if (writes.length > 0) {
-                const commitBatches = [];
-                for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) {
-                    commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT));
-                }
 
-                const commitLimit = pLimit(10); 
-                
-                await Promise.all(commitBatches.map((batchWrites, bIndex) => commitLimit(async () => {
-                    const batch = db.batch();
-                    batchWrites.forEach(w => batch.set(w.ref, w.data, w.options));
-                    
-                    try {
-                        await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`);
-                    } catch (commitErr) {
-                        logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message });
+                if (writes.length > 0) {
+                    const commitBatches = [];
+                    for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) {
+                        commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT));
                     }
-                })));
-            }
 
-        } catch (chunkErr) {
-            logger.log('ERROR', `[BatchPrice] Fatal error processing Chunk ${index}.`, { error: chunkErr.message });
-        }
-    }));
+                    const commitLimit = pLimit(10);
+
+                    await Promise.all(commitBatches.map((batchWrites, bIndex) => commitLimit(async () => {
+                        const batch = db.batch();
+                        batchWrites.forEach(w => batch.set(w.ref, w.data, w.options));
+
+                        try {
+                            await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`);
+                        } catch (commitErr) {
+                            logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message });
+                        }
+                    })));
+                }
+
+            } catch (chunkErr) {
+                logger.log('ERROR', `[BatchPrice] Fatal error processing Chunk ${index}.`, { error: chunkErr.message });
+            }
+        }));
+    }
 
     await Promise.all(chunkPromises);
     logger.log('INFO', '[BatchPrice] Optimization pass complete.');
@@ -541,9 +550,9 @@ module.exports = {
     checkRootDataAvailability,
     fetchExistingResults,
     fetchComputationStatus,
-    fetchGlobalComputationStatus,   
-    updateComputationStatus, 
-    updateGlobalComputationStatus, 
+    fetchGlobalComputationStatus,
+    updateComputationStatus,
+    updateGlobalComputationStatus,
     runStandardComputationPass,
     runMetaComputationPass,
     runBatchPriceComputation

package/functions/computation-system/helpers/reproduce_error.ts

@@ -0,0 +1,19 @@
+
+// Mock types
+namespace Firestore {
+    export class DocumentReference { }
+}
+
+const pLimit = (concurrency: number) => {
+    return (fn: () => Promise<any>) => fn();
+};
+
+const OUTER_CONCURRENCY_LIMIT = 2;
+const outerLimit = pLimit(OUTER_CONCURRENCY_LIMIT);
+
+const shardChunks: Firestore.DocumentReference[][] = [];
+
+// The problematic code
+const chunkPromises = shardChunks.map((shardChunkRefs, index) => outerLimit(async () => {
+    console.log(index);
+}));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.204",
+  "version": "1.0.205",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [