bulltrackers-module 1.0.108 → 1.0.110

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -41,13 +41,37 @@ function groupByPass(manifest) {
     }, {});
 }
 
+/**
+ * --- NEW HELPER ---
+ * Checks if a calculation's root data dependencies are met.
+ * @param {object} calcManifest - The manifest entry for the calculation.
+ * @param {object} rootDataStatus - The status object from checkRootDataAvailability.
+ * @returns {boolean} True if dependencies are met, false otherwise.
+ */
+function checkRootDependencies(calcManifest, rootDataStatus) {
+    // Default to true if property is missing (legacy support)
+    if (!calcManifest.rootDataDependencies || calcManifest.rootDataDependencies.length === 0) {
+        return true;
+    }
+
+    // Check each required dependency
+    for (const dep of calcManifest.rootDataDependencies) {
+        if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) return false;
+        if (dep === 'insights' && !rootDataStatus.hasInsights) return false;
+        if (dep === 'social' && !rootDataStatus.hasSocial) return false;
+    }
+    
+    return true; // All dependencies were met
+}
+
+
 /**
  * --- NEW HELPER ---
  * Checks if the root data (portfolios, insights, social) exists for a given day.
  * @param {string} dateStr - The date string to check (YYYY-MM-DD).
  * @param {object} config - The computation system configuration object.
  * @param {object} dependencies - Contains db, logger, calculationUtils.
- * @returns {Promise<object>} { portfolioRefs, insightsData, socialData, isAvailable }
+ * @returns {Promise<object>} { portfolioRefs, insightsData, socialData, isAvailable, hasPortfolio, hasInsights, hasSocial }
  */
 async function checkRootDataAvailability(dateStr, config, dependencies) {
     const { logger } = dependencies;
@@ -60,22 +84,33 @@ async function checkRootDataAvailability(dateStr, config, dependencies) {
             loadDailySocialPostInsights(config, dependencies, dateStr)
         ]);
 
-        const isAvailable = (portfolioRefs && portfolioRefs.length > 0) || !!insightsData || !!socialData;
+        const hasPortfolio = (portfolioRefs && portfolioRefs.length > 0);
+        const hasInsights = !!insightsData;
+        const hasSocial = !!socialData;
+        const isAvailable = hasPortfolio || hasInsights || hasSocial;
 
         if (isAvailable) {
-            logger.log('INFO', `[Orchestrator] Root data found for ${dateStr}. (Portfolio parts: ${portfolioRefs.length}, Insights: ${!!insightsData}, Social: ${!!socialData})`);
+            logger.log('INFO', `[Orchestrator] Root data found for ${dateStr}. (Portfolio: ${hasPortfolio}, Insights: ${hasInsights}, Social: ${hasSocial})`);
         }
 
         return {
             portfolioRefs: portfolioRefs || [],
             insightsData: insightsData || null,
             socialData: socialData || null,
-            isAvailable: isAvailable
+            isAvailable: isAvailable,
+            // --- MODIFIED: Return granular status ---
+            hasPortfolio: hasPortfolio,
+            hasInsights: hasInsights,
+            hasSocial: hasSocial
+            // --- END MODIFICATION ---
         };
 
     } catch (err) {
         logger.log('ERROR', `[Orchestrator] Error checking data availability for ${dateStr}`, { errorMessage: err.message });
-        return { portfolioRefs: [], insightsData: null, socialData: null, isAvailable: false };
+        return { 
+            portfolioRefs: [], insightsData: null, socialData: null, isAvailable: false,
+            hasPortfolio: false, hasInsights: false, hasSocial: false 
+        };
     }
 }
 
@@ -111,24 +146,26 @@ async function runComputationOrchestrator(config, dependencies, computationManif
     for (const dateStr of allExpectedDates) {
         const dateToProcess = new Date(dateStr + 'T00:00:00Z');
         
-        // --- NEW: Check for root data *before* processing the day ---
+        // --- MODIFIED: Check for root data *before* processing the day ---
         const rootData = await checkRootDataAvailability(dateStr, config, dependencies);
+        // The 'isAvailable' flag now means "is there *any* data"
         if (!rootData.isAvailable) {
             logger.log('WARN', `[Orchestrator] Skipping all computations for ${dateStr} due to missing root data (no portfolios, insights, or social data found).`);
             continue; // Skip to the next day
         }
-        // --- END NEW CHECK ---
+        // --- END MODIFIED CHECK ---
 
         logger.log('INFO', `[Orchestrator] Processing all passes for ${dateStr}...`);
         
-        // This cache will hold results *for this day only*
         const dailyResultsCache = new Map();
+        // --- NEW: Keep track of skipped calcs ---
+        const skippedCalculations = new Set();
         let passSuccess = true;
 
         for (const passNum of passNumbers) {
             if (!passSuccess) {
                 logger.log('WARN', `[Orchestrator] Skipping Pass ${passNum} for ${dateStr} due to previous pass failure.`);
-                break; // Skip subsequent passes if a previous one failed
+                break; 
             }
             
             const calcsInPass = passes[passNum] || [];
@@ -139,38 +176,87 @@ async function runComputationOrchestrator(config, dependencies, computationManif
             logger.log('INFO', `[Orchestrator] Starting Pass ${passNum} for ${dateStr} (${standardCalcs.length} standard, ${metaCalcs.length} meta).`);
 
             try {
-                // 1. Run standard calcs for this pass
+                // --- 1. Run standard calcs for this pass ---
                 if (standardCalcs.length > 0) {
-                    const standardResults = await runUnifiedComputation(
-                        dateToProcess,
-                        standardCalcs, // Pass the manifest objects
-                        `Pass ${passNum} (Standard)`,
-                        config,
-                        dependencies,
-                        rootData // <-- Pass pre-fetched root data
-                    );
                     
-                    // Add results to cache
-                    for (const [calcName, result] of Object.entries(standardResults)) {
-                        dailyResultsCache.set(calcName, result);
+                    // --- NEW: Filter calcs based on root data availability ---
+                    const standardCalcsToRun = [];
+                    for (const calcManifest of standardCalcs) {
+                        if (checkRootDependencies(calcManifest, rootData)) {
+                            standardCalcsToRun.push(calcManifest);
+                        } else {
+                            logger.log('INFO', `[Pass ${passNum}] Skipping standard calc "${calcManifest.name}" for ${dateStr} due to missing root data.`);
+                            skippedCalculations.add(calcManifest.name);
+                        }
+                    }
+                    // --- END NEW FILTER ---
+
+                    if (standardCalcsToRun.length > 0) {
+                        const standardResults = await runUnifiedComputation(
+                            dateToProcess,
+                            standardCalcsToRun, // Pass the filtered list
+                            `Pass ${passNum} (Standard)`,
+                            config,
+                            dependencies,
+                            rootData 
+                        );
+                        
+                        for (const [calcName, result] of Object.entries(standardResults)) {
+                            dailyResultsCache.set(calcName, result);
+                        }
                     }
                 }
 
-                // 2. Run meta calcs for this pass
+                // --- 2. Run meta calcs for this pass ---
                 if (metaCalcs.length > 0) {
-                    const metaResults = await runMetaComputation(
-                        dateToProcess,
-                        metaCalcs, // Pass the manifest objects
-                        `Pass ${passNum} (Meta)`,
-                        config,
-                        dependencies,
-                        dailyResultsCache, // <-- PASS THE CACHE
-                        rootData // <--- !! MODIFICATION: PASS rootData !!
-                    );
-
-                    // Add results to cache
-                    for (const [calcName, result] of Object.entries(metaResults)) {
-                        dailyResultsCache.set(calcName, result);
+
+                    // --- NEW: Filter calcs based on root data AND computation dependencies ---
+                    const metaCalcsToRun = [];
+                    for (const calcManifest of metaCalcs) {
+                        const calcName = calcManifest.name;
+                        
+                        // Check 1: Are root data dependencies met?
+                        const rootCheck = checkRootDependencies(calcManifest, rootData);
+                        if (!rootCheck) {
+                            logger.log('INFO', `[Pass ${passNum} (Meta)] Skipping meta calc "${calcName}" for ${dateStr} due to missing root data.`);
+                            skippedCalculations.add(calcName);
+                            continue;
+                        }
+                        
+                        // Check 2: Are computation dependencies met (i.e., not skipped)?
+                        let depCheck = true;
+                        let missingDepName = '';
+                        for (const depName of (calcManifest.dependencies || [])) {
+                            if (skippedCalculations.has(normalizeName(depName))) {
+                                depCheck = false;
+                                missingDepName = normalizeName(depName);
+                                break;
+                            }
+                        }
+                        
+                        if (depCheck) {
+                            metaCalcsToRun.push(calcManifest);
+                        } else {
+                            logger.log('INFO', `[Pass ${passNum} (Meta)] Skipping meta calc "${calcName}" for ${dateStr} due to missing computation dependency "${missingDepName}".`);
+                            skippedCalculations.add(calcName);
+                        }
+                    }
+                    // --- END NEW FILTER ---
+
+                    if (metaCalcsToRun.length > 0) {
+                        const metaResults = await runMetaComputation(
+                            dateToProcess,
+                            metaCalcsToRun, // Pass the filtered list
+                            `Pass ${passNum} (Meta)`,
+                            config,
+                            dependencies,
+                            dailyResultsCache, 
+                            rootData
+                        );
+
+                        for (const [calcName, result] of Object.entries(metaResults)) {
+                            dailyResultsCache.set(calcName, result);
+                        }
                     }
                 }
                  logger.log('SUCCESS', `[Orchestrator] Completed Pass ${passNum} for ${dateStr}.`);
@@ -179,7 +265,7 @@ async function runComputationOrchestrator(config, dependencies, computationManif
                 passSuccess = false;
             }
         } // End passes loop
-        logger.log('INFO', `[Orchestrator] Finished processing for ${dateStr}.`);
+        logger.log('INFO', `[Orchestrator] Finished processing for ${dateStr}. Total skipped calculations: ${skippedCalculations.size}`);
     } // End dates loop
 
     logger.log('INFO', '[Orchestrator] Computation orchestration finished.');
@@ -255,6 +341,8 @@ async function streamAndProcess(
             if (!p) continue;
 
             const userType = p.PublicPositions ? 'speculator' : 'normal';
+            // Add userType to context
+            context.userType = userType;
 
             for (const calcName in state) { // calcName is already normalized
                 const calc = state[calcName];
@@ -364,14 +452,10 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
         } = rootData;
         // --- END NEW ---
         
-        // (The check for data availability is now done in the orchestrator)
-
         let yesterdayPortfolios = {};
         let yesterdayInsightsData = null; 
         let yesterdaySocialPostInsightsData = null; 
         
-        // Check if any calc needs yesterday's data
-        // --- MODIFIED: Use the manifest entry ---
         const requiresYesterdayPortfolio = calculationsToRun.some(c => c.isHistorical === true);
         const requiresYesterdayInsights = calculationsToRun.some(c => c.class.prototype.process.toString().includes('yesterdayInsights'));
         const requiresYesterdaySocialPosts = calculationsToRun.some(c => c.class.prototype.process.toString().includes('yesterdaySocialPostInsights'));
@@ -450,13 +534,11 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
             const calc = state[calcName];
             if (!calc || typeof calc.getResult !== 'function') continue;
 
-            // --- MODIFIED: Get category from the attached manifest ---
             const category = calc.manifest.category || 'unknown';
 
             try {
                 const result = await Promise.resolve(calc.getResult());
                 
-                // Add to results map for in-memory cache
                 if (result) {
                     passResults[calcName] = result;
                 }
@@ -465,7 +547,6 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
                 const summaryData = {};
 
                 if (result && Object.keys(result).length > 0) {
-                    // (Special handling for sharded calcs remains the same)
                     let isSharded = false;
                     const shardedCollections = {
                         'sharded_user_profile': config.shardedUserProfileCollection,
@@ -485,7 +566,6 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
                                     pendingWrites.push({ ref: shardRef, data: shardedData[shardId] });
                                 }
                             }
-                            // Get results *not* in the shard key
                             const { [resultKey]: _, ...otherResults } = result;
                             if (Object.keys(otherResults).length > 0) {
                                 const computationDocRef = resultsCollectionRef.doc(category)
@@ -502,7 +582,6 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
                                                           .doc(calcName);
                         pendingWrites.push({ ref: computationDocRef, data: result });
                     }
-                    // --- END SHARDED HANDLING ---
 
                     if (!summaryData[category]) summaryData[category] = {};
                     summaryData[category][calcName] = true;
@@ -522,7 +601,15 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
                         successCount++;
                     }
                 } else {
-                     logger.log('WARN', `[${passName}] Calculation ${calcName} produced no results for ${dateStr}. Skipping write to allow backfill.`);
+                     // --- MODIFICATION: Do not log a warning if the result is null. ---
+                     // This is now expected behavior if a calc (like user-profitability-tracker)
+                     // has no data to process.
+                     // We *do* still need to log if a result is empty.
+                     if (result === null) {
+                         logger.log('INFO', `[${passName}] Calculation ${calcName} returned null for ${dateStr}. This is expected if no data was processed.`);
+                     } else {
+                         logger.log('WARN', `[${passName}] Calculation ${calcName} produced empty results {} for ${dateStr}. Skipping write.`);
+                     }
                 }
             } catch (e) {
                 logger.log('ERROR', `[${passName}] getResult/Commit failed for ${calcName} on ${dateStr}`, { err: e.message });
@@ -532,7 +619,7 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
         const completionStatus = successCount === calculationsToRun.length ? 'SUCCESS' : 'WARN';
         logger.log(completionStatus, `[${passName}] Completed ${dateStr}. Success: ${successCount}/${calculationsToRun.length}.`);
         
-        return passResults; // --- RETURN THE RESULTS ---
+        return passResults; 
 
     } catch (err) {
         logger.log('ERROR', `[${passName}] Fatal error for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
@@ -543,39 +630,33 @@ async function runUnifiedComputation(dateToProcess, calculationsToRun, passName,
 
 /**
  * Internal sub-pipe: Runs "meta" or "backtest" computations for a single date.
- * MODIFIED: Accepts in-memory cache and passes dependencies to calcs.
- * MODIFIED: Accepts rootData and adds it to the dependencies object.
- * Returns a map of results.
  */
 async function runMetaComputation(
     dateToProcess, 
-    calculationsToRun, 
+    calculationsToRun, // This is the *filtered* list
     passName, 
     config, 
     dependencies, 
     dailyResultsCache,
-    rootData // <--- !! MODIFICATION: ACCEPT rootData !!
+    rootData
 ) {
     const { db, logger } = dependencies;
     const dateStr = dateToProcess.toISOString().slice(0, 10);
     logger.log('INFO', `[${passName}] Starting run for ${dateStr} with ${calculationsToRun.length} calcs.`);
     
-    const passResults = {}; // Map to store and return results
+    const passResults = {}; 
 
     try {
         const resultsCollectionRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection);
         let successCount = 0;
         
-        // --- !! MODIFICATION: Add rootData to the dependencies object !! ---
         const dependenciesForMetaCalc = {
             ...dependencies,
             rootData: rootData
         };
-        // --- END MODIFICATION ---
 
         for (const manifestCalc of calculationsToRun) {
             const calcName = normalizeName(manifestCalc.name);
-            // --- MODIFIED: Get category from manifest ---
             const category = manifestCalc.category || 'unknown';
             const CalcClass = manifestCalc.class;
             
@@ -588,12 +669,17 @@ async function runMetaComputation(
 
             try {
                 // --- Gather dependencies from the cache ---
+                // This check is now more robust, as the orchestrator has already
+                // logged the *reason* for a skip. We just need to check the cache.
                 const computedDependencies = {};
                 let missingDep = false;
                 if (manifestCalc.dependencies) {
                     for (const depName of manifestCalc.dependencies) {
                         const normalizedDepName = normalizeName(depName);
                         if (!dailyResultsCache.has(normalizedDepName)) {
+                            // This log is still important, as it indicates a logic error
+                            // if a calc *wasn't* skipped by the orchestrator but its
+                            // dependency is *still* missing (e.g., the dep returned null).
                             logger.log('ERROR', `[${passName}] Missing required dependency "${normalizedDepName}" for calculation "${calcName}". This should not happen. Skipping calc.`);
                             missingDep = true;
                             break;
@@ -606,15 +692,12 @@ async function runMetaComputation(
                 // --- Call process with the dependencies ---
                 const result = await Promise.resolve(instance.process(
                     dateStr, 
-                    dependenciesForMetaCalc, // <-- PASS MODIFIED DEPS
+                    dependenciesForMetaCalc,
                     config, 
-                    computedDependencies // <-- PASS IN-MEMORY DEPS
+                    computedDependencies 
                 ));
                 
-                // Add to results map
-                if (result) {
-                    passResults[calcName] = result;
-                }
+                passResults[calcName] = result;
                 
                 const pendingWrites = [];
                 const summaryData = {};
@@ -647,18 +730,17 @@ async function runMetaComputation(
                 }
             } catch (e) {
                 logger.log('ERROR', `[${passName}] Meta-calc process/commit failed for ${calcName} on ${dateStr}`, { err: e.message, stack: e.stack });
-                // Don't re-throw, allow other meta-calcs in the pass to run
             }
         }
 
         const completionStatus = successCount === calculationsToRun.length ? 'SUCCESS' : 'WARN';
         logger.log(completionStatus, `[${passName}] Completed ${dateStr}. Success: ${successCount}/${calculationsToRun.length}.`);
         
-        return passResults; // --- RETURN THE RESULTS ---
+        return passResults; 
 
     } catch (err) {
         logger.log('ERROR', `[${passName}] Fatal error for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
-        throw err; // Re-throw to stop the orchestrator for this day
+        throw err; 
     }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.108",
+  "version": "1.0.110",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [