bulltrackers-module 1.0.115 → 1.0.117

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -157,7 +157,12 @@ async function runComputationOrchestrator(config, dependencies, computationManif
 
         logger.log('INFO', `[Orchestrator] Processing all passes for ${dateStr}...`);
         
+        // This is the cache that accumulates results.
+        // Pass 1 (A,B,C) results are put in.
+        // Pass 2 gets (A,B,C) and adds (E,F,G).
+        // Pass 3 gets (A,B,C,E,F,G) and adds (I,J,K).
         const dailyResultsCache = new Map();
+        
         // --- NEW: Keep track of skipped calcs ---
         const skippedCalculations = new Set();
         let passSuccess = true;
@@ -170,13 +175,15 @@ async function runComputationOrchestrator(config, dependencies, computationManif
             
             const calcsInPass = passes[passNum] || [];
             
+            // "Standard" calcs are your Pass 1 (A,B,C,D)
             const standardCalcs = calcsInPass.filter(c => c.type === 'standard');
+            // "Meta" calcs are your Pass 2, 3, 4 (E,F,G, etc.)
             const metaCalcs = calcsInPass.filter(c => c.type === 'meta');
             
             logger.log('INFO', `[Orchestrator] Starting Pass ${passNum} for ${dateStr} (${standardCalcs.length} standard, ${metaCalcs.length} meta).`);
 
             try {
-                // --- 1. Run standard calcs for this pass ---
+                // --- 1. Run standard calcs for this pass (e.g., Pass 1) ---
                 if (standardCalcs.length > 0) {
                     
                     // --- NEW: Filter calcs based on root data availability ---
@@ -192,6 +199,7 @@ async function runComputationOrchestrator(config, dependencies, computationManif
                     // --- END NEW FILTER ---
 
                     if (standardCalcsToRun.length > 0) {
+                        // This function runs all "standard" calcs
                         const standardResults = await runUnifiedComputation(
                             dateToProcess,
                             standardCalcsToRun, // Pass the filtered list
@@ -201,13 +209,14 @@ async function runComputationOrchestrator(config, dependencies, computationManif
                             rootData 
                         );
                         
+                        // Add results to the accumulating cache
                         for (const [calcName, result] of Object.entries(standardResults)) {
                             dailyResultsCache.set(calcName, result);
                         }
                     }
                 }
 
-                // --- 2. Run meta calcs for this pass ---
+                // --- 2. Run meta calcs for this pass (e.g., Pass 2, 3, 4) ---
                 if (metaCalcs.length > 0) {
 
                     // --- NEW: Filter calcs based on root data AND computation dependencies ---
@@ -244,16 +253,19 @@ async function runComputationOrchestrator(config, dependencies, computationManif
                     // --- END NEW FILTER ---
 
                     if (metaCalcsToRun.length > 0) {
+                        // This function runs all "meta" calcs,
+                        // giving it the *entire cache* of results from all previous passes
                         const metaResults = await runMetaComputation(
                             dateToProcess,
                             metaCalcsToRun, // Pass the filtered list
                             `Pass ${passNum} (Meta)`,
                             config,
                             dependencies,
-                            dailyResultsCache, 
+                            dailyResultsCache, // <-- This is the accumulating cache
                             rootData
                         );
 
+                        // Add results to the accumulating cache
                         for (const [calcName, result] of Object.entries(metaResults)) {
                             dailyResultsCache.set(calcName, result);
                         }
@@ -431,9 +443,10 @@ async function streamAndProcess(
 
 
 /**
- * Internal sub-pipe: Runs "standard" computations for a single date.
+ * Internal sub-pipe: Runs "standard" computations (Pass 1) for a single date.
  * MODIFIED: Accepts pre-fetched rootData.
  * MODIFIED: Returns a map of results for the in-memory cache.
+ * MODIFIED: Applies robustness fix.
  */
 async function runUnifiedComputation(dateToProcess, calculationsToRun, passName, config, dependencies, rootData) {
     const { db, logger } = dependencies;
@@ -533,12 +546,12 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
         for (const calcName in state) { // calcName is already normalized
             const calc = state[calcName];
 
-            // --- START FIX ---
+            // --- FIX #1: ROBUSTNESS FOR STANDARD CALCS ---
             // Ensure every calc from the state has a result (null by default)
             // This prevents the "Missing dependency" error downstream
-            // if the calculation failed to initialize.
+            // if the calculation failed to initialize or was skipped.
             passResults[calcName] = null; 
-            // --- END FIX ---
+            // --- END FIX #1 ---
 
             if (!calc || typeof calc.getResult !== 'function') {
                  if (!calc) {
@@ -654,7 +667,8 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
 
 
 /**
- * Internal sub-pipe: Runs "meta" or "backtest" computations for a single date.
+ * Internal sub-pipe: Runs "meta" or "backtest" computations (Pass 2, 3, 4) for a single date.
+ * MODIFIED: Applies robustness fix.
  */
 async function runMetaComputation(
     dateToProcess, 
@@ -662,13 +676,14 @@ async function runMetaComputation(
     passName, 
     config, 
     dependencies, 
-    dailyResultsCache,
+    dailyResultsCache, // <-- This is the accumulating cache from all previous passes
     rootData
 ) {
     const { db, logger } = dependencies;
     const dateStr = dateToProcess.toISOString().slice(0, 10);
     logger.log('INFO', `[${passName}] Starting run for ${dateStr} with ${calculationsToRun.length} calcs.`);
     
+    // This cache is *only* for the results of this specific pass
     const passResults = {}; 
 
     try {
@@ -685,13 +700,13 @@ async function runMetaComputation(
             const category = manifestCalc.category || 'unknown';
             const CalcClass = manifestCalc.class;
 
-            // --- START FIX ---
-            // Set a default null result. This ensures that if the
-            // calculation is skipped (due to missing deps),
-            // its *own* dependents will see a 'null' value
-            // instead of a "Missing dependency" error.
+            // --- FIX #2: ROBUSTNESS FOR META CALCS ---
+            // Set a default null result. This ensures that if this
+            // calculation is skipped (due to its own missing deps),
+            // its *own* dependents (later in this same pass) will see a
+            // 'null' value instead of a "Missing dependency" error.
             passResults[calcName] = null;
-            // --- END FIX ---
+            // --- END FIX #2 ---
             
             if (typeof CalcClass !== 'function') {
                  logger.log('ERROR', `[${passName}] Invalid class in manifest for ${calcName}. Skipping.`);
@@ -757,11 +772,51 @@ async function runMetaComputation(
                 const summaryData = {};
 
                 if (result && Object.keys(result).length > 0) {
-                    const computationDocRef = resultsCollectionRef.doc(category)
-                                                      .collection(config.computationsSubcollection)
-                                                      .doc(calcName);
-                    pendingWrites.push({ ref: computationDocRef, data: result });
+
+                    // --- START SHARDING FIX ---
+                    let isSharded = false;
+                    const shardedCollections = {
+                        // Add keys from sharded meta-calcs here
+                        'sharded_user_profile': config.shardedUserProfileCollection,
+                        'sharded_user_profitability': config.shardedProfitabilityCollection 
+                    };
+
+                    for (const resultKey in shardedCollections) {
+                        if (result[resultKey]) {
+                            isSharded = true;
+                            const shardCollectionName = shardedCollections[resultKey];
+                            if (!shardCollectionName) {
+                                logger.log('ERROR', `[${passName}] Missing config key for sharded collection: ${resultKey}`);
+                                continue;
+                            }
+                            
+                            const shardedData = result[resultKey];
+                            
+                            for (const shardId in shardedData) {
+                                const shardDocData = shardedData[shardId];
+                                if (shardDocData && (Object.keys(shardDocData).length > 0)) {
+                                    const shardRef = db.collection(shardCollectionName).doc(shardId);
+                                    // Use { merge: true } to safely write to sharded docs
+                                    pendingWrites.push({ ref: shardRef, data: shardDocData, merge: true });
+                                }
+                            }
+                            
+                            // De-structure the sharded key from the result
+                            const { [resultKey]: _, ...otherResults } = result;
+                            result = otherResults; // Re-assign 'result' to be only the non-sharded data
+                        }
+                    }
                     
+                    // After all sharding is handled, save any *remaining* data
+                    // (e.g., 'daily_investor_scores' from user-investment-profile)
+                    if (result && Object.keys(result).length > 0) {
+                        const computationDocRef = resultsCollectionRef.doc(category)
+                                                          .collection(compsSub)
+                                                          .doc(calcName);
+                        pendingWrites.push({ ref: computationDocRef, data: result });
+                    }
+                    // --- END SHARDING FIX ---
+
                     if (!summaryData[category]) summaryData[category] = {};
                     summaryData[category][calcName] = true;
 
@@ -797,7 +852,7 @@ async function runMetaComputation(
         const completionStatus = successCount === calculationsToRun.length ? 'SUCCESS' : 'WARN';
         logger.log(completionStatus, `[${passName}] Completed ${dateStr}. Success: ${successCount}/${calculationsToRun.length}.`);
         
-        return passResults; 
+        return passResults; // Return the results *from this pass only*
 
     } catch (err) {
         logger.log('ERROR', `[${passName}] Fatal error for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
@@ -808,4 +863,4 @@ async function runMetaComputation(
 
 module.exports = {
     runComputationOrchestrator,
-};
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.115",
+  "version": "1.0.117",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [