bulltrackers-module 1.0.102 → 1.0.103

package/functions/computation-system/computation_dependencies.json

@@ -1,38 +1,120 @@
 {
-    "dependencies": {
-        "smart-cohort-flow": ["user-investment-profile"],
-        "dumb-cohort-flow": ["user-investment-profile"],
-        "crowd_sharpe_ratio_proxy": ["pnl-distribution-per-stock"],
-        "social_flow_correlation": [
-            "social_sentiment_aggregation",
-            "asset_crowd_flow"
-        ],
-        "cash-flow-deployment": [
-            "crowd-cash-flow-proxy",
-            "asset_crowd_flow"
-        ],
-        "cash-flow-liquidation": [
-            "crowd-cash-flow-proxy",
-            "asset-crowd-flow"
-        ],
-        "capital_deployment_strategy": [
-            "crowd-cash-flow-proxy",
-            "new-allocation-percentage",
-            "reallocation-increase-percentage"
-        ],
-        "profit_cohort_divergence": [
-            "in-profit-asset-crowd-flow",
-            "in-loss-asset-crowd-flow"
-        ],
-        "smart-dumb-divergence-index": [
-            "smart-cohort-flow",
-            "dumb-cohort-flow"
-        ],
-        "capital_vintage_performance": ["cash-flow-deployment"],
-        "capital_liquidation_performance": ["cash-flow-liquidation"],
-        "strategy-performance": [
-            "smart-dumb-divergence-index",
-            "profit_cohort_divergence"
-        ]
-    }
+  "dependencies": {
+    "smart-cohort-flow": ["user-investment-profile"],
+    "dumb-cohort-flow": ["user-investment-profile"],
+    "crowd_sharpe_ratio_proxy": ["pnl-distribution-per-stock"],
+    "social_flow_correlation": [
+      "social_sentiment_aggregation",
+      "asset_crowd_flow"
+    ],
+    "cash-flow-deployment": [
+      "crowd-cash-flow-proxy",
+      "asset_crowd_flow"
+    ],
+    "cash-flow-liquidation": [
+      "crowd-cash-flow-proxy",
+      "asset_crowd_flow"
+    ],
+    "capital_deployment_strategy": [
+      "crowd-cash-flow-proxy",
+      "new-allocation-percentage",
+      "reallocation-increase-percentage"
+    ],
+    "profit_cohort_divergence": [
+      "in-profit-asset-crowd-flow",
+      "in-loss-asset-crowd-flow"
+    ],
+    "smart-dumb-divergence-index": [
+      "smart-cohort-flow",
+      "dumb-cohort-flow"
+    ],
+    "capital_vintage_performance": ["cash-flow-deployment"],
+    "capital_liquidation_performance": ["cash-flow-liquidation"],
+    "strategy-performance": [
+      "smart-dumb-divergence-index",
+      "profit_cohort_divergence"
+    ]
+  },
+  "requirements": {
+    "activity_by_pnl_status": ["today_portfolio", "yesterday_portfolio"],
+    "daily_asset_activity": ["today_portfolio", "yesterday_portfolio"],
+    "daily_user_activity_tracker": ["today_portfolio", "yesterday_portfolio"],
+    "speculator_adjustment_activity": ["today_portfolio", "yesterday_portfolio"],
+    "asset_position_size": ["today_portfolio"],
+    "strategy-performance": ["meta"],
+    "asset_crowd_flow": ["today_portfolio", "yesterday_portfolio"],
+    "drawdown_response": ["today_portfolio", "yesterday_portfolio"],
+    "dumb-cohort-flow": ["today_portfolio", "yesterday_portfolio"],
+    "gain_response": ["today_portfolio", "yesterday_portfolio"],
+    "in_loss_asset_crowd_flow": ["today_portfolio", "yesterday_portfolio"],
+    "in_profit_asset_crowd_flow": ["today_portfolio", "yesterday_portfolio"],
+    "paper_vs_diamond_hands": ["today_portfolio", "yesterday_portfolio"],
+    "position_count_pnl": ["today_portfolio"],
+    "smart-cohort-flow": ["today_portfolio", "yesterday_portfolio"],
+    "smart_money_flow": ["today_portfolio", "yesterday_portfolio"],
+    "user-investment-profile": ["today_portfolio", "yesterday_portfolio"],
+    "crowd-cash-flow-proxy": ["today_portfolio", "yesterday_portfolio"],
+    "deposit_withdrawal_percentage": ["today_portfolio", "yesterday_portfolio"],
+    "new_allocation_percentage": ["today_portfolio", "yesterday_portfolio"],
+    "reallocation_increase_percentage": ["today_portfolio", "yesterday_portfolio"],
+    "daily_bought_vs_sold_count": ["today_insights", "yesterday_insights"],
+    "daily_buy_sell_sentiment_count": ["today_insights"],
+    "daily_ownership_delta": ["today_insights", "yesterday_insights"],
+    "daily_total_positions_held": ["today_insights"],
+    "capital_deployment_strategy": ["meta"],
+    "cash-flow-deployment": ["meta"],
+    "cash-flow-liquidation": ["meta"],
+    "crowd_sharpe_ratio_proxy": ["meta"],
+    "profit_cohort_divergence": ["meta"],
+    "smart-dumb-divergence-index": ["meta"],
+    "social_flow_correlation": ["meta"],
+    "capital_liquidation_performance": ["meta"],
+    "capital_vintage_performance": ["meta"],
+    "profitability_migration": ["today_portfolio", "yesterday_portfolio"],
+    "user_profitability_tracker": ["today_portfolio"],
+    "asset_pnl_status": ["today_portfolio"],
+    "average_daily_pnl_all_users": ["today_portfolio"],
+    "average_daily_pnl_per_sector": ["today_portfolio"],
+    "average_daily_pnl_per_stock": ["today_portfolio"],
+    "average_daily_position_pnl": ["today_portfolio"],
+    "pnl_distribution_per_stock": ["today_portfolio"],
+    "profitability_ratio_per_stock": ["today_portfolio"],
+    "profitability_skew_per_stock": ["today_portfolio"],
+    "profitable_and_unprofitable_status": ["today_portfolio"],
+    "users_processed": ["today_portfolio"],
+    "diversification_pnl": ["today_portfolio", "yesterday_portfolio"],
+    "sector_rotation": ["today_portfolio", "yesterday_portfolio"],
+    "total_long_per_sector": ["today_portfolio"],
+    "total_short_per_sector": ["today_portfolio"],
+    "crowd_conviction_score": ["today_portfolio", "yesterday_portfolio"],
+    "long_position_per_stock": ["today_portfolio"],
+    "sentiment_per_stock": ["today_portfolio"],
+    "short_position_per_stock": ["today_portfolio"],
+    "total_long_figures": ["today_portfolio"],
+    "total_short_figures": ["today_portfolio"],
+    "social-asset-posts-trend": ["today_social"],
+    "social-top-mentioned-words": ["today_social", "yesterday_social"],
+    "social-topic-interest-evolution": ["today_social"],
+    "social-word-mentions-trend": ["today_social"],
+    "social_activity_aggregation": ["today_social"],
+    "social_event_correlation": ["today_social", "yesterday_social"],
+    "social_sentiment_aggregation": ["today_social"],
+    "risk_appetite_change": ["today_portfolio", "yesterday_portfolio"],
+    "tsl_effectiveness": ["today_portfolio"],
+    "distance_to_stop_loss_per_leverage": ["today_portfolio"],
+    "distance_to_tp_per_leverage": ["today_portfolio"],
+    "entry_distance_to_sl_per_leverage": ["today_portfolio"],
+    "entry_distance_to_tp_per_leverage": ["today_portfolio"],
+    "holding_duration_per_asset": ["today_portfolio"],
+    "leverage_per_asset": ["today_portfolio"],
+    "leverage_per_sector": ["today_portfolio"],
+    "risk_reward_ratio_per_asset": ["today_portfolio"],
+    "speculator_asset_sentiment": ["today_portfolio"],
+    "speculator_danger_zone": ["today_portfolio"],
+    "stop_loss_distance_by_sector_short_long_breakdown": ["today_portfolio"],
+    "stop_loss_distance_by_ticker_short_long_breakdown": ["today_portfolio"],
+    "stop_loss_per_asset": ["today_portfolio"],
+    "take_profit_per_asset": ["today_portfolio"],
+    "tsl_per_asset": ["today_portfolio"]
+  }
 }

package/functions/computation-system/helpers/worker_helpers.js

@@ -44,56 +44,68 @@ function findCalcClass(calculations, category, name) {
     return null;
 }
 
+// --- NEW: Internal helper for previous date string ---
+function getPrevStr(dateStr) {
+    const prev = new Date(dateStr + 'T00:00:00Z');
+    prev.setUTCDate(prev.getUTCDate() - 1);
+    return prev.toISOString().slice(0, 10);
+}
+
 /**
  * Streams data and runs a *single* Batch 1-style computation.
+ * --- THIS IS THE FULLY UPDATED FUNCTION ---
  */
 async function runStreamingComputation(dateStr, node, config, dependencies, calculations) {
     const { db, logger } = dependencies;
-    const { category, name: calcName, calcClass: CalculationClass } = node;
+    
+    // --- MODIFIED: Destructure 'requires' from the node ---
+    const { category, name: calcName, calcClass: CalculationClass, requires } = node;
     const passName = `[Worker ${dateStr}/${calcName}]`;
     
-    logger.log('INFO', `${passName} Starting streaming computation...`);
-
-    // --- Data Loading ---
-    const todayInsightsData = await loadDailyInsights(config, dependencies, dateStr);
-    const todaySocialPostInsightsData = await loadDailySocialPostInsights(config, dependencies, dateStr);
-    const todayRefs = await getPortfolioPartRefs(config, dependencies, dateStr);
-
-    if (todayRefs.length === 0 && !todayInsightsData && !todaySocialPostInsightsData) {
-        logger.log('WARN', `${passName} No portfolio, instrument insights, OR social post data found for ${dateStr}. Skipping.`);
-        // Return null to indicate "no result, but not an error"
-        return null;
-    }
+    logger.log('INFO', `${passName} Starting streaming computation... (Requires: ${requires.join(', ') || 'none'})`);
 
-    const { HISTORICAL_CALC_NAMES } = categorizeCalculations(calculations);
-    let yesterdayPortfolios = {};
+    // --- Dynamic Data Loading ---
+    let todayInsightsData = null;
     let yesterdayInsightsData = null;
+    let todaySocialPostInsightsData = null;
     let yesterdaySocialPostInsightsData = null;
+    let todayRefs = [];
+    let yesterdayPortfolios = {};
+    const prevStr = getPrevStr(dateStr); // Get yesterday's date string
 
-    const requiresYesterdayPortfolio = HISTORICAL_CALC_NAMES.has(calcName);
-    const requiresYesterdayInsights = category === 'insights';
-    const requiresYesterdaySocialPosts = category === 'socialPosts';
-
-    const prev = new Date(dateStr + 'T00:00:00Z');
-    prev.setUTCDate(prev.getUTCDate() - 1);
-    const prevStr = prev.toISOString().slice(0, 10);
-
-    if (requiresYesterdayPortfolio) {
+    if (requires.includes('today_insights')) {
+        todayInsightsData = await loadDailyInsights(config, dependencies, dateStr);
+    }
+    if (requires.includes('yesterday_insights')) {
+        yesterdayInsightsData = await loadDailyInsights(config, dependencies, prevStr);
+    }
+    if (requires.includes('today_social')) {
+        todaySocialPostInsightsData = await loadDailySocialPostInsights(config, dependencies, dateStr);
+    }
+    if (requires.includes('yesterday_social')) {
+        yesterdaySocialPostInsightsData = await loadDailySocialPostInsights(config, dependencies, prevStr);
+    }
+    if (requires.includes('today_portfolio')) {
+        todayRefs = await getPortfolioPartRefs(config, dependencies, dateStr);
+    }
+    if (requires.includes('yesterday_portfolio')) {
         const yesterdayRefs = await getPortfolioPartRefs(config, dependencies, prevStr);
         if (yesterdayRefs.length > 0) {
             yesterdayPortfolios = await loadFullDayMap(config, dependencies, yesterdayRefs);
-        } else {
-             logger.log('WARN', `${passName} Yesterday's (${prevStr}) portfolio data not found.`);
+        } else if (requires.includes('today_portfolio')) { 
+            // Only warn if today_portfolio was also expected, otherwise this might be normal
+            logger.log('WARN', `${passName} Requires 'yesterday_portfolio' but no data was found for ${prevStr}.`);
         }
     }
-    if (requiresYesterdayInsights) {
-        yesterdayInsightsData = await loadDailyInsights(config, dependencies, prevStr);
-    }
-    if (requiresYesterdaySocialPosts) {
-        yesterdaySocialPostInsightsData = await loadDailySocialPostInsights(config, dependencies, prevStr);
+    // --- End Dynamic Data Loading ---
+
+    // Check if any primary data source was loaded. If not, we can't run.
+    if (todayRefs.length === 0 && !todayInsightsData && !todaySocialPostInsightsData) {
+        logger.log('WARN', `${passName} No primary data sources (portfolio, insights, social) found for ${dateStr}. Skipping.`);
+        return null;
     }
-    // --- End Data Loading ---
 
+    // --- Context Object (no change) ---
     const { instrumentToTicker, instrumentToSector } = await dependencies.calculationUtils.loadInstrumentMappings();
     const context = {
         instrumentMappings: instrumentToTicker,
@@ -106,55 +118,52 @@ async function runStreamingComputation(dateStr, node, config, dependencies, calc
 
     const calc = new CalculationClass();
     const batchSize = config.partRefBatchSize || 10;
-    let isFirstUser = true;
-
-    // --- Stream and Process ---
-    for (let i = 0; i < todayRefs.length; i += batchSize) {
-        const batchRefs = todayRefs.slice(i, i + batchSize);
-        const todayPortfoliosChunk = await loadDataByRefs(config, dependencies, batchRefs);
-
-        for (const uid in todayPortfoliosChunk) {
-            const p = todayPortfoliosChunk[uid];
-            if (!p) continue;
-            const userType = p.PublicPositions ? 'speculator' : 'normal';
-
-            let processArgs;
-            const allContextArgs = [context, todayInsightsData, yesterdayInsightsData, todaySocialPostInsightsData, yesterdaySocialPostInsightsData];
-
-            if (category === 'insights' || category === 'socialPosts') {
-                if (isFirstUser) {
-                    processArgs = [null, null, null, ...allContextArgs];
-                } else {
-                    continue; // Only run once
-                }
-            } else if (requiresYesterdayPortfolio) {
-                const pYesterday = yesterdayPortfolios[uid];
-                if (!pYesterday) continue;
-                processArgs = [p, pYesterday, uid, ...allContextArgs];
-            } else {
+    
+    // --- MODIFIED: Stream and Process ---
+    let processedOnce = false; // Flag for insights/social calcs
+    const allContextArgs = [context, todayInsightsData, yesterdayInsightsData, todaySocialPostInsightsData, yesterdaySocialPostInsightsData];
+
+    // Only stream portfolios if they are required and were found
+    if (requires.includes('today_portfolio') && todayRefs.length > 0) {
+        for (let i = 0; i < todayRefs.length; i += batchSize) {
+            const batchRefs = todayRefs.slice(i, i + batchSize);
+            const todayPortfoliosChunk = await loadDataByRefs(config, dependencies, batchRefs);
+
+            for (const uid in todayPortfoliosChunk) {
+                const p = todayPortfoliosChunk[uid];
+                if (!p) continue;
+                
+                // User-type filtering
+                const userType = p.PublicPositions ? 'speculator' : 'normal';
                 if ((userType === 'normal' && category === 'speculators') || (userType === 'speculator' && !['speculators', 'sanity'].includes(category))) {
                     continue;
                 }
-                processArgs = [p, null, uid, ...allContextArgs];
+                
+                let processArgs;
+                if (requires.includes('yesterday_portfolio')) {
+                    const pYesterday = yesterdayPortfolios[uid];
+                    if (!pYesterday) continue; // Skip if user is missing yesterday's data
+                    processArgs = [p, pYesterday, uid, ...allContextArgs];
+                } else {
+                    processArgs = [p, null, uid, ...allContextArgs];
+                }
+                
+                await Promise.resolve(calc.process(...processArgs));
+                processedOnce = true;
             }
-            await Promise.resolve(calc.process(...processArgs));
-            isFirstUser = false;
         }
     }
     
-    if (todayRefs.length === 0 && isFirstUser && (category === 'insights' || category === 'socialPosts')) {
-        logger.log('INFO', `${passName} No user portfolios. Running ${category} calc once.`);
-        const allContextArgs = [context, todayInsightsData, yesterdayInsightsData, todaySocialPostInsightsData, yesterdaySocialPostInsightsData];
+    // If this is an insights/social calc and no portfolios ran (or were needed), run it once.
+    if (!processedOnce && (requires.includes('today_insights') || requires.includes('today_social'))) {
+        logger.log('INFO', `${passName} Running once for insights/social data.`);
         await Promise.resolve(calc.process(null, null, null, ...allContextArgs));
     }
     // --- End Stream ---
 
-    // --- Get Result and Commit ---
+    // --- Get Result and Commit (no change) ---
     const result = await Promise.resolve(calc.getResult());
     
-    // If a calc returns null (e.g., dependency missing), it's a "soft fail".
-    // We throw an error to be caught by the main handler, which will log it
-    // and NOT mark the node as "success", allowing for a re-trigger.
     if (result === null) {
         throw new Error(`Calculation ${calcName} returned null. Dependency likely missing.`);
     }
@@ -173,7 +182,6 @@ async function runStreamingComputation(dateStr, node, config, dependencies, calc
         for (const resultKey in shardedCollections) {
             if (result[resultKey]) {
                 isSharded = true;
-                // ... (sharding logic from old worker_helpers.js, no changes) ...
                 const shardCollectionName = shardedCollections[resultKey];
                 const shardedData = result[resultKey];
                 for (const shardId in shardedData) {
@@ -212,6 +220,7 @@ async function runStreamingComputation(dateStr, node, config, dependencies, calc
 
 /**
  * Runs a *single* meta/backtest computation.
+ * (This function remains unchanged as 'meta' calcs handle their own data loading)
  */
 async function runMetaComputation(dateStr, node, config, dependencies) {
     const { db, logger } = dependencies;
@@ -253,13 +262,7 @@ async function runMetaComputation(dateStr, node, config, dependencies) {
 
 /**
  * Main pipe: pipe.computationSystem.handleNodeTask
- * This is the Pub/Sub triggered handler for the worker.
- *
- * @param {object} message - The Pub/Sub message { date, nodeName }.
- * @param {object} context - The message context.
- * @param {object} config - The task-engine-specific config.
- * @param {object} dependencies - Contains all clients: db, pubsub, logger, etc.
- * @param {object} calculations - The injected calculations object from 'aiden-shared-calculations-unified'.
+ * (This function remains unchanged)
  */
 async function handleNodeTask(message, context, config, dependencies, calculations) {
     const { logger, pubsubUtils } = dependencies;
@@ -291,7 +294,8 @@ async function handleNodeTask(message, context, config, dependencies, calculatio
     
     try {
         // 1. Route to the correct execution helper
-        if (node.category === 'meta' || node.category === 'backtests') {
+        // --- MODIFIED: Check for 'meta' requirement key ---
+        if (node.requires.includes('meta')) {
             await runMetaComputation(date, node, config, dependencies);
         } else {
             // All other calcs (pnl, behavioural, etc.) use the streaming helper
@@ -317,22 +321,13 @@ async function handleNodeTask(message, context, config, dependencies, calculatio
         
         } else {
             logger.log('SUCCESS', `${taskId} Complete. No new children are ready.`);
-            // Optional: Check if the *entire* DAG is now complete for this day
-            // This is a "nice-to-have" but good for logging.
         }
 
     } catch (error) {
-         // --- THIS IS THE SELF-HEALING LOGIC ---
-         // A calc failed (e.g., dependency missing). We log it and stop.
-         // We do *not* mark it as "success" in the state manager.
-         // We do *not* trigger its children.
-         // The system will wait until the dependency that *caused* this
-         // failure is fixed and re-triggers this node.
          logger.log('WARN', `${taskId} FAILED (Will retry when dependencies are met).`, { 
              err: error.message, 
              stack: error.stack 
          });
-         // Acknowledge the Pub/Sub message so it doesn't retry immediately.
          return;
     }
 }

package/functions/computation-system/utils/dependency_graph.js

@@ -16,6 +16,7 @@ class ComputationNode {
         this.calcClass = calcClass;
         this.children = new Set(); // Nodes that depend on this one
         this.parents = new Set();  // Nodes this one depends on
+        this.requires = []; // <-- ADDED: Stores data requirements
     }
 }
 
@@ -34,14 +35,16 @@ class ComputationGraph {
         this.sortedNodes = []; // Array of node names, in valid execution order
         this.isBuilt = false;
 
-        this._registerNodes(unifiedCalculations);
+        // --- MODIFIED: Pass requirements config to _registerNodes ---
+        this._registerNodes(unifiedCalculations, dependencyConfig.requirements || {});
         this._buildAdjacencyLists(dependencyConfig.dependencies || {});
     }
 
     /**
      * Pass 1: Register all calculations as nodes in the graph.
+     * --- MODIFIED: Added requirementsConfig parameter ---
      */
-    _registerNodes(unifiedCalculations) {
+    _registerNodes(unifiedCalculations, requirementsConfig) {
         this.logger.log('INFO', '[DepGraph] Pass 1: Registering all calculation nodes...');
         for (const category in unifiedCalculations) {
             if (category === 'utils') continue;
@@ -53,14 +56,22 @@ class ComputationGraph {
                 if (typeof item === 'object' && item !== null && !Array.isArray(item)) {
                     for (const calcName in item) {
                         if (typeof item[calcName] === 'function') {
-                            this.nodes.set(calcName, new ComputationNode(category, calcName, item[calcName]));
+                            // --- MODIFIED: Create node, set requirements ---
+                            const node = new ComputationNode(category, calcName, item[calcName]);
+                            node.requires = requirementsConfig[calcName] || [];
+                            this.nodes.set(calcName, node);
+                            // --- END MODIFICATION ---
                         }
                     }
                 }
                 // Handle regular calcs at the root of the category
                 else if (typeof item === 'function') {
                     const calcName = subKey;
-                    this.nodes.set(calcName, new ComputationNode(category, calcName, item[calcName]));
+                    // --- MODIFIED: Create node, set requirements ---
+                    const node = new ComputationNode(category, calcName, item[calcName]);
+                    node.requires = requirementsConfig[calcName] || [];
+                    this.nodes.set(calcName, node);
+                    // --- END MODIFICATION ---
                 }
             }
         }
@@ -69,6 +80,7 @@ class ComputationGraph {
 
     /**
      * Pass 2: Build parent/child relationships from the dependency config.
+     * (This function remains unchanged as it only reads the 'dependencies' key)
      */
     _buildAdjacencyLists(dependencies) {
         this.logger.log('INFO', '[DepGraph] Pass 2: Building adjacency lists...');
@@ -99,6 +111,7 @@ class ComputationGraph {
     /**
      * Pass 3: Perform Topological Sort (Kahn's Algorithm) to find execution order
      * and detect cycles. This is the "build" step.
+     * (This function remains unchanged)
      */
     build() {
         if (this.isBuilt) return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.102",
+  "version": "1.0.103",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [