aiden-shared-calculations-unified 1.0.10 → 1.0.12

package/calculations/behavioural/asset_crowd_flow.js

@@ -0,0 +1,151 @@
+const { loadAllPriceData, getDailyPriceChange } = require('../../utils/price_data_provider');
+const { loadInstrumentMappings } = require('../../utils/sector_mapping_provider');
+
+/**
+ * @fileoverview Calculates "Net Crowd Flow" for each asset.
+ *
+ * This isolates the change in an asset's average portfolio percentage
+ * that is *not* explained by the asset's own price movement.
+ *
+ * Net Crowd Flow = (Actual % Change) - (Expected % Change from Price Move)
+ *
+ * A positive value means the crowd actively bought (flowed into) the asset.
+ * A negative value means the crowd actively sold (flowed out of) the asset.
+ */
+class AssetCrowdFlow {
+    constructor() {
+        this.asset_values = {}; // Stores { day1_value_sum: 0, day2_value_sum: 0 }
+        this.user_count = 0;
+        this.priceMap = null;
+        this.mappings = null;
+        this.dates = {}; // To store { today: '...', yesterday: '...' }
+    }
+
+    /**
+     * Helper to safely initialize an asset entry.
+     */
+    _initAsset(instrumentId) {
+        if (!this.asset_values[instrumentId]) {
+            this.asset_values[instrumentId] = {
+                day1_value_sum: 0,
+                day2_value_sum: 0
+            };
+        }
+    }
+
+    /**
+     * Helper to sum the 'Value' field from an AggregatedPositions array.
+     */
+    _sumAssetValue(positions) {
+        const valueMap = {};
+        if (!positions || !Array.isArray(positions)) {
+            return valueMap;
+        }
+        for (const pos of positions) {
+            if (pos && pos.InstrumentID && pos.Value) {
+                valueMap[pos.InstrumentID] = (valueMap[pos.InstrumentID] || 0) + pos.Value;
+            }
+        }
+        return valueMap;
+    }
+
+    process(todayPortfolio, yesterdayPortfolio, userId, context) {
+        // This is a historical calculation, requires both days
+        if (!todayPortfolio || !yesterdayPortfolio || !todayPortfolio.AggregatedPositions || !yesterdayPortfolio.AggregatedPositions) {
+            return;
+        }
+
+        // Capture dates from context on the first run
+        if (!this.dates.today && context.todayDateStr && context.yesterdayDateStr) {
+            this.dates.today = context.todayDateStr;
+            this.dates.yesterday = context.yesterdayDateStr;
+        }
+        
+        const yesterdayValues = this._sumAssetValue(yesterdayPortfolio.AggregatedPositions);
+        const todayValues = this._sumAssetValue(todayPortfolio.AggregatedPositions);
+        
+        // Use a set of all unique instruments held across both days
+        const allInstrumentIds = new Set([
+            ...Object.keys(yesterdayValues),
+            ...Object.keys(todayValues)
+        ]);
+
+        for (const instrumentId of allInstrumentIds) {
+            this._initAsset(instrumentId);
+            this.asset_values[instrumentId].day1_value_sum += (yesterdayValues[instrumentId] || 0);
+            this.asset_values[instrumentId].day2_value_sum += (todayValues[instrumentId] || 0);
+        }
+
+        this.user_count++;
+    }
+
+    async getResult() {
+        if (this.user_count === 0 || !this.dates.today) {
+            return {}; // No users processed or dates not found
+        }
+
+        // Load dependencies (prices and mappings) in parallel
+        if (!this.priceMap || !this.mappings) {
+            const [priceData, mappingData] = await Promise.all([
+                loadAllPriceData(),
+                loadInstrumentMappings()
+            ]);
+            this.priceMap = priceData;
+            this.mappings = mappingData;
+        }
+        
+        const finalResults = {};
+        const todayStr = this.dates.today;
+        const yesterdayStr = this.dates.yesterday;
+
+        for (const instrumentId in this.asset_values) {
+            const ticker = this.mappings.instrumentToTicker[instrumentId] || `id_${instrumentId}`;
+            
+            // 1. Calculate average % values
+            const avg_day1_value = this.asset_values[instrumentId].day1_value_sum / this.user_count;
+            const avg_day2_value = this.asset_values[instrumentId].day2_value_sum / this.user_count;
+            
+            // 2. Get the actual price change
+            const priceChangePct = getDailyPriceChange(instrumentId, yesterdayStr, todayStr, this.priceMap);
+
+            if (priceChangePct === null) {
+                // Cannot calculate if price data is missing for either day
+                finalResults[ticker] = {
+                    net_crowd_flow_pct: 0,
+                    error: "Missing price data for calculation."
+                };
+                continue;
+            }
+
+            // 3. Calculate the expected value (the "price-move" effect)
+            // We use avg_day1_value as the base. The cash flow proxy calculation
+            // uses (avg_value - avg_invested) because it's solving for a different unknown.
+            // Here, we are solving for flow *relative to the asset itself*.
+            const expected_day2_value = avg_day1_value * (1 + priceChangePct);
+
+            // 4. Find the signal (the "crowd-flow" effect)
+            const net_crowd_flow_pct = avg_day2_value - expected_day2_value;
+
+            finalResults[ticker] = {
+                net_crowd_flow_pct: net_crowd_flow_pct,
+                avg_value_day1_pct: avg_day1_value,
+                avg_value_day2_pct: avg_day2_value,
+                expected_value_day2_pct: expected_day2_value,
+                price_change_pct: priceChangePct,
+                user_sample_size: this.user_count
+            };
+        }
+        
+        return finalResults;
+    }
+
+    reset() {
+        this.asset_values = {};
+        this.user_count = 0;
+        this.priceMap = null;
+        this.mappings = null;
+        this.dates = {};
+    }
+}
+
+module.exports = AssetCrowdFlow;

package/calculations/capital_flow/deposit_withdrawal_percentage.js

@@ -1,71 +1,118 @@
 /**
- * @fileoverview Estimates the average percentage deposited into accounts between two days.
+ * @fileoverview Estimates a proxy for net crowd cash flow (Deposits vs. Withdrawals)
+ * by aggregating portfolio percentage changes across all users.
+ *
+ * This calculation is based on the formula:
+ * Total_Change = P/L_Effect + Trading_Effect + Cash_Flow_Effect
+ *
+ * Where:
+ * - Total_Change = avg_value_day2 - avg_value_day1
+ * - P/L_Effect = avg_value_day1 - avg_invested_day1
+ * - Trading_Effect = avg_invested_day2 - avg_invested_day1
+ *
+ * We solve for Cash_Flow_Effect, which serves as our proxy.
+ * A negative value indicates a net DEPOSIT (denominator grew, shrinking all %s).
+ * A positive value indicates a net WITHDRAWAL (denominator shrank, inflating all %s).
  */
 
-class DepositPercentage {
+class CrowdCashFlowProxy {
     constructor() {
-        this.totalDepositPercentage = 0;
-        this.userCount = 0;
+        this.total_invested_day1 = 0;
+        this.total_value_day1 = 0;
+        this.total_invested_day2 = 0;
+        this.total_value_day2 = 0;
+        this.user_count = 0;
     }
 
     /**
-     * Calculates total PnL from aggregated or public positions.
-     * @param {object} portfolio - Portfolio data object.
-     * @returns {number|null} Total PnL or null if positions are missing.
+     * Helper to sum a specific field from an AggregatedPositions array.
+     * @param {Array<object>} positions - The AggregatedPositions array.
+     * @param {string} field - The field to sum ('Invested' or 'Value').
+     * @returns {number} The total sum of that field.
      */
-    calculateTotalPnl(portfolio) {
-        // Use the same logic as in ProfitabilityMigration
-        if (portfolio && portfolio.AggregatedPositions) {
-            return portfolio.AggregatedPositions.reduce((sum, pos) => sum + (pos.NetProfit || 0), 0);
-        } else if (portfolio && portfolio.PublicPositions) {
-            // PublicPositions might lack NetProfit, adjust if necessary based on your data
-            return portfolio.PublicPositions.reduce((sum, pos) => sum + (pos.NetProfit || 0), 0);
+    _sumPositions(positions, field) {
+        if (!positions || !Array.isArray(positions)) {
+            return 0;
         }
-        return null;
+        return positions.reduce((sum, pos) => sum + (pos[field] || 0), 0);
     }
 
     process(todayPortfolio, yesterdayPortfolio, userId) {
-        if (!todayPortfolio || !yesterdayPortfolio || !yesterdayPortfolio.PortfolioValue || yesterdayPortfolio.PortfolioValue === 0) {
-            // Need both portfolios and a non-zero yesterday value for calculation
+        // This calculation requires both days' data to compare
+        if (!todayPortfolio || !yesterdayPortfolio || !todayPortfolio.AggregatedPositions || !yesterdayPortfolio.AggregatedPositions) {
             return;
         }
 
-        const valueChange = (todayPortfolio.PortfolioValue || 0) - yesterdayPortfolio.PortfolioValue;
+        const invested_day1 = this._sumPositions(yesterdayPortfolio.AggregatedPositions, 'Invested');
+        const value_day1 = this._sumPositions(yesterdayPortfolio.AggregatedPositions, 'Value');
+        const invested_day2 = this._sumPositions(todayPortfolio.AggregatedPositions, 'Invested');
+        const value_day2 = this._sumPositions(todayPortfolio.AggregatedPositions, 'Value');
 
-        const todayPnl = this.calculateTotalPnl(todayPortfolio);
-        const yesterdayPnl = this.calculateTotalPnl(yesterdayPortfolio);
-
-        // If PnL can't be calculated for either day, we can't reliably estimate cash flow
-        if (todayPnl === null || yesterdayPnl === null) {
+        // Only include users who have some form of positions
+        if (invested_day1 === 0 && invested_day2 === 0 && value_day1 === 0 && value_day2 === 0) {
             return;
         }
 
-        const pnlChange = todayPnl - yesterdayPnl;
-        const cashFlow = valueChange - pnlChange;
-
-        // Check for deposit (positive cash flow)
-        if (cashFlow > 0) {
-            const depositPercentage = (cashFlow / yesterdayPortfolio.PortfolioValue) * 100;
-            this.totalDepositPercentage += depositPercentage;
-            this.userCount++;
-        }
+        this.total_invested_day1 += invested_day1;
+        this.total_value_day1 += value_day1;
+        this.total_invested_day2 += invested_day2;
+        this.total_value_day2 += value_day2;
+        this.user_count++;
     }
 
     getResult() {
-        if (this.userCount === 0) {
-            return {}; // Return empty object if no users contributed
+        if (this.user_count === 0) {
+            return {}; // No users processed, return empty.
         }
 
+        // 1. Calculate the average portfolio for the crowd
+        const avg_invested_day1 = this.total_invested_day1 / this.user_count;
+        const avg_value_day1 = this.total_value_day1 / this.user_count;
+        const avg_invested_day2 = this.total_invested_day2 / this.user_count;
+        const avg_value_day2 = this.total_value_day2 / this.user_count;
+
+        // 2. Isolate the three effects
+        const total_value_change = avg_value_day2 - avg_value_day1;
+        const pl_effect = avg_value_day1 - avg_invested_day1;
+        const trading_effect = avg_invested_day2 - avg_invested_day1;
+
+        // 3. Solve for the Cash Flow Effect
+        // Total_Change = pl_effect + trading_effect + cash_flow_effect
+        const cash_flow_effect = total_value_change - pl_effect - trading_effect;
+
         return {
-            // Calculate the final average directly
-            average_deposit_percentage: this.totalDepositPercentage / this.userCount
+            // The final proxy value.
+            // A negative value signals a net DEPOSIT.
+            // A positive value signals a net WITHDRAWAL.
+            cash_flow_effect_proxy: cash_flow_effect,
+            
+            // Interpretation for the frontend
+            interpretation: "A negative value signals a net crowd deposit. A positive value signals a net crowd withdrawal.",
+            
+            // Debug components
+            components: {
+                total_value_change: total_value_change,
+                pl_effect: pl_effect,
+                trading_effect: trading_effect
+            },
+            // Debug averages
+            averages: {
+                avg_invested_day1: avg_invested_day1,
+                avg_value_day1: avg_value_day1,
+                avg_invested_day2: avg_invested_day2,
+                avg_value_day2: avg_value_day2
+            },
+            user_sample_size: this.user_count
         };
     }
 
     reset() {
-        this.totalDepositPercentage = 0;
-        this.userCount = 0;
+        this.total_invested_day1 = 0;
+        this.total_value_day1 = 0;
+        this.total_invested_day2 = 0;
+        this.total_value_day2 = 0;
+        this.user_count = 0;
     }
 }
 
-module.exports = DepositPercentage;
+module.exports = CrowdCashFlowProxy;

package/index.js

@@ -13,6 +13,7 @@ const path = require('path');
 // --- Utils (Manually Exported) ---
 const firestoreUtils = require('./utils/firestore_utils');
 const mappingProvider = require('./utils/sector_mapping_provider');
+const priceProvider = require('./utils/price_data_provider'); // <-- ADD THIS
 
 // --- Calculations (Dynamically Loaded) ---
 const calculations = requireAll({
@@ -26,6 +27,7 @@ module.exports = {
     calculations,
     utils: {
         ...firestoreUtils,
-        ...mappingProvider
+        ...mappingProvider,
+        ...priceProvider // <-- ADD THIS
     }
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "aiden-shared-calculations-unified",
-  "version": "1.0.10",
+  "version": "1.0.12",
   "description": "Shared calculation modules for the BullTrackers Computation System.",
   "main": "index.js",
   "files": [

package/utils/price_data_provider.js

@@ -0,0 +1,103 @@
+const { Firestore } = require('@google-cloud/firestore');
+const firestore = new Firestore();
+
+// Config
+const PRICE_COLLECTION = 'asset_prices';
+const CACHE_DURATION_MS = 3600000; // 1 hour
+
+// Cache
+let cache = {
+  timestamp: null,
+  priceMap: null, // Will be { instrumentId: { "YYYY-MM-DD": price, ... } }
+};
+
+// In-progress fetch promise
+let fetchPromise = null;
+
+/**
+ * Loads all sharded price data from the `asset_prices` collection.
+ * This is a heavy operation and should be cached.
+ */
+async function loadAllPriceData() {
+  const now = Date.now();
+  if (cache.timestamp && (now - cache.timestamp < CACHE_DURATION_MS)) {
+    return cache.priceMap;
+  }
+
+  if (fetchPromise) {
+    return fetchPromise;
+  }
+
+  fetchPromise = (async () => {
+    console.log('Fetching and caching all asset price data...');
+    const masterPriceMap = {};
+    
+    try {
+      const snapshot = await firestore.collection(PRICE_COLLECTION).get();
+      
+      if (snapshot.empty) {
+        throw new Error(`Price collection '${PRICE_COLLECTION}' is empty.`);
+      }
+
+      // Loop through each shard document (e.g., "shard_0", "shard_1")
+      snapshot.forEach(doc => {
+        const shardData = doc.data();
+        
+        // Loop through each instrumentId in the shard
+        for (const instrumentId in shardData) {
+          // Check if it's a valid instrument entry
+          if (shardData[instrumentId] && shardData[instrumentId].prices) {
+            masterPriceMap[instrumentId] = shardData[instrumentId].prices;
+          }
+        }
+      });
+
+      cache = {
+        timestamp: now,
+        priceMap: masterPriceMap,
+      };
+      
+      console.log(`Successfully cached prices for ${Object.keys(masterPriceMap).length} instruments.`);
+      return masterPriceMap;
+
+    } catch (err) {
+      console.error('CRITICAL: Error loading price data:', err);
+      // On error, return an empty map but don't cache, so a future call can retry.
+      return {};
+    } finally {
+      // Clear the promise so the next call (if cache is stale) triggers a new fetch
+      fetchPromise = null;
+    }
+  })();
+
+  return fetchPromise;
+}
+
+/**
+ * A helper to safely get the price change percentage between two dates.
+ * @param {string} instrumentId - The instrument ID.
+ * @param {string} yesterdayStr - YYYY-MM-DD date string for yesterday.
+ * @param {string} todayStr - YYYY-MM-DD date string for today.
+ * @param {object} priceMap - The master price map from loadAllPriceData().
+ * @returns {number|null} The percentage change (e.g., 0.10 for +10%), or null if data is missing.
+ */
+function getDailyPriceChange(instrumentId, yesterdayStr, todayStr, priceMap) {
+    if (!priceMap || !priceMap[instrumentId]) {
+        return null; // No price data for this instrument
+    }
+    
+    const priceDay1 = priceMap[instrumentId][yesterdayStr];
+    const priceDay2 = priceMap[instrumentId][todayStr];
+
+    if (priceDay1 && priceDay2 && priceDay1 > 0) {
+        return (priceDay2 - priceDay1) / priceDay1;
+    }
+    
+    return null; // Missing one or both dates, or division by zero
+}
+
+
+module.exports = {
+    loadAllPriceData,
+    getDailyPriceChange
+};