aiden-shared-calculations-unified 1.0.64 → 1.0.65

package/calculations/behavioural/historical/smart_money_flow.js

@@ -1,188 +1,197 @@
 /**
- * @fileoverview Tracks the investment flow of "smart money".
+ * @fileoverview Calculation (Pass 3) for smart money flow.
  *
- * --- META REFACTOR (v2) ---
- * This is a streaming meta-calc. It reads historical shards (a side-effect
- * from Pass 1) and streams root portfolio data to calculate its result.
- * It has no direct computational dependencies passed to `process`.
+ * This metric calculates the "Net Crowd Flow Percentage" for
+ * "Smart Money" users, defined as users who were profitable
+ * in 5 of the last 7 days.
+ *
+ * This calculation *depends* on 'user_profitability_tracker'
+ * to identify the cohort.
  */
+const { loadInstrumentMappings } = require('../../../utils/sector_mapping_provider');
 
-const { Firestore } = require('@google-cloud/firestore');
-const firestore = new Firestore();
-const { getInstrumentSectorMap } = require('../../../utils/sector_mapping_provider');
-// NOTE: Corrected relative path for data_loader
-const { loadDataByRefs, getPortfolioPartRefs, loadFullDayMap } = require('../../../../bulltrackers-module/functions/computation-system/utils/data_loader');
+class SmartMoneyFlow {
+    constructor() {
+        // This calculation only aggregates by sector
+        this.sectorData = new Map();
+        this.mappings = null;
+        this.smartMoneyUserIds = null;
+    }
 
+    /**
+     * Defines the output schema for this calculation.
+     * @returns {object} JSON Schema object
+     */
+    static getSchema() {
+        return {
+            "type": "object",
+            "description": "Calculates net capital flow % (price-adjusted) for the 'Smart Money' cohort (profitable 5 of last 7 days), aggregated by sector.",
+            "properties": {
+                "cohort_size": {
+                    "type": "number",
+                    "description": "The number of users identified as being in the Smart Money Cohort."
+                },
+                "sectors": {
+                    "type": "object",
+                    "description": "Price-adjusted net flow per sector.",
+                    "patternProperties": {
+                        // Sector
+                        "^.*$": {
+                            "type": "object",
+                            "properties": {
+                                "net_flow_percentage": { "type": "number" },
+                                "total_invested_today": { "type": "number" },
+                                "total_invested_yesterday": { "type": "number" }
+                            },
+                            "required": ["net_flow_percentage", "total_invested_today", "total_invested_yesterday"]
+                        }
+                    },
+                    "additionalProperties": {
+                        "type": "object",
+                        "properties": {
+                            "net_flow_percentage": { "type": "number" },
+                            "total_invested_today": { "type": "number" },
+                            "total_invested_yesterday": { "type": "number" }
+                        }
+                    }
+                }
+            },
+            "required": ["cohort_size", "sectors"]
+        };
+    }
 
-class SmartMoneyFlow {
-    
     /**
-     * (NEW) Statically declare dependencies.
-     * This calc reads shards from Pass 1, but doesn't take a computed
-     * result as an argument. The manifest ensures it runs in Pass 2.
+     * Statically declare dependencies.
      */
     static getDependencies() {
-        return ['user-profitability-tracker']; // This is just for ordering, not for `fetchedDependencies`
+        return ['user_profitability_tracker'];
     }
-    
-    constructor() {
-        this.smartMoneyUsers = new Set();
-        this.sectorMap = null; 
-        
-        // We now store the portfolio data during process()
-        // (This state is reset for each date by the runner)
-        this.todayPortfolios = {};
-        this.yesterdayPortfolios = {};
+
+    _getPortfolioPositions(portfolio) {
+        return portfolio?.PublicPositions || portfolio?.AggregatedPositions;
+    }
+
+    _initSector(sector) {
+        if (!this.sectorData.has(sector)) {
+            this.sectorData.set(sector, {
+                total_invested_yesterday: 0,
+                total_invested_today: 0,
+                price_change_yesterday: 0,
+            });
+        }
     }
 
     /**
-     * REFACTORED PROCESS METHOD
-     * This is now the *first* method called by the runner.
-     * It caches the portfolios.
+     * Helper to get the cohort IDs from the dependency.
      */
-    async process(dateStr, dependencies, config, fetchedDependencies) {
-        const { logger, db, rootData } = dependencies;
-        const { portfolioRefs } = rootData;
-        logger.log('INFO', '[SmartMoneyFlow] Starting meta-process...');
-        
-        // 1. Load external dependencies (sectors)
-        if (!this.sectorMap) {
-             try {
-                 this.sectorMap = await getInstrumentSectorMap();
-            } catch (error) {
-                logger.log('ERROR', '[SmartMoneyFlow] Failed to load sector map.', { err: error.message });
-                return null; // Abort
-            }
+    _getSmartMoneyCohort(fetchedDependencies) {
+        if (this.smartMoneyUserIds) {
+            return this.smartMoneyUserIds;
         }
         
-        // 2. Load "yesterday's" portfolio data for comparison
-        const yesterdayDate = new Date(dateStr + 'T00:00:00Z');
-        yesterdayDate.setUTCDate(yesterdayDate.getUTCDate() - 1);
-        const yesterdayStr = yesterdayDate.toISOString().slice(0, 10);
-        const yesterdayRefs = await getPortfolioPartRefs(config, dependencies, yesterdayStr);
-        const yesterdayPortfolios = await loadFullDayMap(config, dependencies, yesterdayRefs);
-        logger.log('INFO', `[SmartMoneyFlow] Loaded ${yesterdayRefs.length} part refs for yesterday.`);
-
-        // 3. Identify smart money users by reading Pass 1 shards
-        // This is the dependency.
-        await this.identifySmartMoney(dependencies);
-        if (this.smartMoneyUsers.size === 0) {
-            logger.log('WARN', '[SmartMoneyFlow] No smart money users identified. Aborting.');
-            return { smart_money_flow: {} };
+        const profitabilityData = fetchedDependencies['user_profitability_tracker'];
+        if (!profitabilityData || !profitabilityData.user_details) {
+            return new Set();
         }
 
-        // 4. Stream "today's" portfolio data and process *only smart users*
-        const batchSize = config.partRefBatchSize || 10;
-        const sectorFlow = {};
+        this.smartMoneyUserIds = new Set();
+        for (const [userId, data] of Object.entries(profitabilityData.user_details)) {
+            // Definition: Profitable in 5 of the last 7 days.
+            if (data.profitable_days_7d >= 5) {
+                this.smartMoneyUserIds.add(userId);
+            }
+        }
+        return this.smartMoneyUserIds;
+    }
 
-        for (let i = 0; i < portfolioRefs.length; i += batchSize) {
-            const batchRefs = portfolioRefs.slice(i, i + batchSize);
-            const todayPortfoliosChunk = await loadDataByRefs(config, dependencies, batchRefs);
+    process(todayPortfolio, yesterdayPortfolio, userId, context, todayInsights, yesterdayInsights, fetchedDependencies) {
+        const smartMoneyCohort = this._getSmartMoneyCohort(fetchedDependencies);
+        
+        // This user is not in the "smart money cohort", skip.
+        if (!smartMoneyCohort.has(userId)) {
+            return;
+        }
 
-            for (const uid in todayPortfoliosChunk) {
-                // --- Filter user ---
-                if (!this.smartMoneyUsers.has(uid)) {
-                    continue;
-                }
-                
-                const pToday = todayPortfoliosChunk[uid];
-                const pYesterday = yesterdayPortfolios[uid];
-                
-                if (!pToday || !pYesterday) continue;
+        if (!todayPortfolio || !yesterdayPortfolio) {
+            return;
+        }
 
-                // --- User is in cohort, run logic ---
-                const todaySectorInvestment = this.calculateSectorInvestment(pToday);
-                const yesterdaySectorInvestment = this.calculateSectorInvestment(pYesterday);
+        const yPos = this._getPortfolioPositions(yesterdayPortfolio);
+        const tPos = this._getPortfolioPositions(todayPortfolio);
 
-                const allSectors = new Set([...Object.keys(todaySectorInvestment), ...Object.keys(yesterdaySectorInvestment)]);
-                for (const sector of allSectors) {
-                    const todayAmount = todaySectorInvestment[sector] || 0;
-                    const yesterdayAmount = yesterdaySectorInvestment[sector] || 0;
-                    const change = todayAmount - yesterdayAmount;
+        const yPosMap = new Map(yPos?.map(p => [p.InstrumentID, p]) || []);
+        const tPosMap = new Map(tPos?.map(p => [p.InstrumentID, p]) || []);
 
-                    if (change !== 0) {
-                         if (!sectorFlow[sector]) {
-                            sectorFlow[sector] = 0;
-                        }
-                        sectorFlow[sector] += change;
-                    }
-                }
-            }
-        }
+        const allInstrumentIds = new Set([...yPosMap.keys(), ...tPosMap.keys()]);
         
-        logger.log('INFO', `[SmartMoneyFlow] Processed ${this.smartMoneyUsers.size} smart users.`);
-
-        // 5. Return final result
-        return { smart_money_flow: sectorFlow };
-    }
+        if (!this.mappings) {
+            // Context contains the mappings loaded in Pass 1
+            this.mappings = context.mappings;
+        }
 
-     async identifySmartMoney(dependencies) {
-         const { logger } = dependencies;
-        logger.log('INFO', "[SmartMoneyFlow] Attempting to identify smart money users by reading shards...");
-        try {
-            const shardPromises = [];
-            const NUM_SHARDS = 50; 
-            for (let i = 0; i < NUM_SHARDS; i++) {
-                const docRef = firestore.collection('historical_insights').doc(`user_profitability_shard_${i}`);
-                shardPromises.push(docRef.get());
+        for (const instrumentId of allInstrumentIds) {
+            if (!instrumentId) continue;
+            
+            const yP = yPosMap.get(instrumentId);
+            const tP = tPosMap.get(instrumentId);
+
+            const yInvested = yP?.InvestedAmount || yP?.Amount || 0;
+            const tInvested = tP?.InvestedAmount || tP?.Amount || 0;
+            
+            // Get sector and initialize it
+            const sector = this.mappings.instrumentToSector[instrumentId] || 'Other';
+            this._initSector(sector);
+            const sectorAsset = this.sectorData.get(sector);
+
+            if (yInvested > 0) {
+                const yPriceChange = (yP?.PipsRate || 0) / (yP?.OpenRate || 1);
+                sectorAsset.total_invested_yesterday += yInvested;
+                sectorAsset.price_change_yesterday += yPriceChange * yInvested;
+            }
+            if (tInvested > 0) {
+                sectorAsset.total_invested_today += tInvested;
             }
-            const shardSnapshots = await Promise.all(shardPromises);
-
-            let profitableUsersFound = 0;
-            shardSnapshots.forEach((snap, index) => {
-                if (snap.exists) {
-                    const shardData = snap.data() || {};
-                    for (const userId in shardData) {
-                        const history = shardData[userId];
-                        if (Array.isArray(history) && history.length >= 5) {
-                             const profitableDays = history.slice(-7).filter(d => d && typeof d.pnl === 'number' && d.pnl > 0).length;
-                             if (profitableDays >= 5) {
-                                this.smartMoneyUsers.add(userId);
-                                profitableUsersFound++;
-                            }
-                        }
-                    }
-                } else {
-                     logger.log('WARN', `[SmartMoneyFlow] Profitability shard user_profitability_shard_${index} does not exist.`);
-                }
-            });
-             logger.log('INFO', `[SmartMoneyFlow] Found ${profitableUsersFound} smart money users across all shards.`);
-
-        } catch (error) {
-            logger.log('ERROR', '[SmartMoneyFlow] Error identifying smart money users', { err: error.message });
-            this.smartMoneyUsers.clear();
         }
     }
 
-
-    calculateSectorInvestment(portfolio) {
-        const sectorInvestment = {};
-         if (!this.sectorMap) {
-             console.warn("[SmartMoneyFlow] Cannot calculate sector investment: Sector map not loaded.");
-             return sectorInvestment; 
-         }
-
-        if (portfolio && portfolio.AggregatedPositions && Array.isArray(portfolio.AggregatedPositions)) {
-            for (const pos of portfolio.AggregatedPositions) {
-                 if (pos && typeof pos.InstrumentID !== 'undefined' && typeof pos.Invested === 'number') {
-                    const sector = this.sectorMap[pos.InstrumentID] || 'N/A';
-                    if (!sectorInvestment[sector]) {
-                        sectorInvestment[sector] = 0;
-                    }
-                    sectorInvestment[sector] += pos.Invested;
-                }
+    async getResult(fetchedDependencies) {
+        // Ensure mappings are loaded (can be from context or loaded now)
+        if (!this.mappings) {
+            this.mappings = await loadInstrumentMappings();
+        }
+        
+        // Ensure cohort is calculated at least once
+        const smartMoneyCohort = this._getSmartMoneyCohort(fetchedDependencies);
+        
+        // Calculate Sector Flow
+        const sectorResult = {};
+        for (const [sector, data] of this.sectorData.entries()) {
+            const { total_invested_yesterday, total_invested_today, price_change_yesterday } = data;
+            
+            if (total_invested_yesterday > 0) {
+                const avg_price_change_pct = price_change_yesterday / total_invested_yesterday;
+                const price_contribution = total_invested_yesterday * avg_price_change_pct;
+                const flow_contribution = total_invested_today - (total_invested_yesterday + price_contribution);
+                const net_flow_percentage = (flow_contribution / total_invested_yesterday) * 100;
+                
+                sectorResult[sector] = {
+                    net_flow_percentage: net_flow_percentage,
+                    total_invested_today: total_invested_today,
+                    total_invested_yesterday: total_invested_yesterday
+                };
             }
         }
-        return sectorInvestment;
-    }
 
+        return {
+            cohort_size: smartMoneyCohort.size,
+            sectors: sectorResult
+        };
+    }
 
-    async getResult() { return null; }
     reset() {
-        this.smartMoneyUsers.clear();
-        this.sectorMap = null;
-        this.todayPortfolios = {};
-        this.yesterdayPortfolios = {};
+        this.sectorData.clear();
+        this.mappings = null;
+        this.smartMoneyUserIds = null;
     }
 }
 

package/calculations/capital_flow/historical/crowd-cash-flow-proxy.js

@@ -1,121 +1,127 @@
 /**
- * @fileoverview Estimates a proxy for net crowd cash flow (Deposits vs. Withdrawals)
- * by aggregating portfolio percentage changes across all users.
+ * @fileoverview Calculation (Pass 2) for crowd cash flow proxy.
  *
- * This calculation is based on the formula:
- * Total_Change = P/L_Effect + Trading_Effect + Cash_Flow_Effect
+ * This metric estimates the net cash flow (deposits vs. withdrawals)
+ * of the crowd by analyzing changes in 'Invested' vs. 'Cash' balances
+ * in user portfolios.
  *
- * Where:
- * - Total_Change = avg_value_day2 - avg_value_day1
- * - P/L_Effect = avg_value_day1 - avg_invested_day1
- * - Trading_Effect = avg_invested_day2 - avg_invested_day1
- *
- * We solve for Cash_Flow_Effect, which serves as our proxy.
- * A negative value indicates a net DEPOSIT (denominator grew, shrinking all %s).
- * A positive value indicates a net WITHDRAWAL (denominator shrank, inflating all %s).
- *
- * NOTE: This file is a logical duplicate of deposit_withdrawal_percentage.js
- * to satisfy the hardcoded 'crowd-cash-flow-proxy' dependency in several
- * meta-calculations.
+ * It's a proxy because it doesn't see real bank transactions, but
+ * infers from the total portfolio value vs. the invested amount.
  */
-
 class CrowdCashFlowProxy {
     constructor() {
-        this.total_invested_day1 = 0;
-        this.total_value_day1 = 0;
-        this.total_invested_day2 = 0;
-        this.total_value_day2 = 0;
-        this.user_count = 0;
+        this.total_portfolio_value_yesterday = 0;
+        this.total_invested_yesterday = 0;
+        
+        this.total_portfolio_value_today = 0;
+        this.total_invested_today = 0;
     }
 
     /**
-     * Helper to sum a specific field from an AggregatedPositions array.
-     * @param {Array<object>} positions - The AggregatedPositions array.
-     * @param {string} field - The field to sum ('Invested' or 'Value').
-     * @returns {number} The total sum of that field.
+     * Defines the output schema for this calculation.
+     * @returns {object} JSON Schema object
      */
-    _sumPositions(positions, field) {
-        if (!positions || !Array.isArray(positions)) {
-            return 0;
+    static getSchema() {
+        return {
+            "type": "object",
+            "description": "Estimates net cash flow (deposits/withdrawals) by comparing changes in total portfolio value vs. invested capital.",
+            "properties": {
+                "total_portfolio_value_yesterday": {
+                    "type": "number",
+                    "description": "Sum of all users' total portfolio values from yesterday."
+                },
+                "total_invested_yesterday": {
+                    "type": "number",
+                    "description": "Sum of all users' invested capital from yesterday."
+                },
+                "total_portfolio_value_today": {
+                    "type": "number",
+                    "description": "Sum of all users' total portfolio values from today."
+                },
+                "total_invested_today": {
+                    "type": "number",
+                    "description": "Sum of all users' invested capital from today."
+                },
+                "total_pnl_contribution": {
+                    "type": "number",
+                    "description": "The estimated change in portfolio value attributable to P&L."
+                },
+                "net_cash_flow_proxy": {
+                    "type": "number",
+                    "description": "The estimated net cash flow (Deposits - Withdrawals). Positive indicates net deposits."
+                },
+                "net_cash_flow_proxy_pct": {
+                    "type": "number",
+                    "description": "The net cash flow proxy as a percentage of yesterday's total portfolio value."
+                }
+            },
+            "required": [
+                "total_portfolio_value_yesterday", "total_invested_yesterday",
+                "total_portfolio_value_today", "total_invested_today",
+                "total_pnl_contribution", "net_cash_flow_proxy", "net_cash_flow_proxy_pct"
+            ]
+        };
+    }
+
+    _getPortfolioValues(portfolio) {
+        if (!portfolio || !portfolio.Summary) {
+            return { portfolioValue: 0, invested: 0 };
         }
-        return positions.reduce((sum, pos) => sum + (pos[field] || 0), 0);
+        
+        // Total value (Invested + Cash)
+        const portfolioValue = portfolio.Summary.PortfolioValue || 0;
+        // Total invested in positions
+        const invested = portfolio.Summary.Invested || 0;
+        
+        return { portfolioValue, invested };
     }
 
-    process(todayPortfolio, yesterdayPortfolio, userId) {
-        // This calculation requires both days' data to compare
-        if (!todayPortfolio || !yesterdayPortfolio || !todayPortfolio.AggregatedPositions || !yesterdayPortfolio.AggregatedPositions) {
+    process(todayPortfolio, yesterdayPortfolio) {
+        if (!todayPortfolio || !yesterdayPortfolio) {
             return;
         }
 
-        const invested_day1 = this._sumPositions(yesterdayPortfolio.AggregatedPositions, 'Invested');
-        const value_day1 = this._sumPositions(yesterdayPortfolio.AggregatedPositions, 'Value');
-        const invested_day2 = this._sumPositions(todayPortfolio.AggregatedPositions, 'Invested');
-        const value_day2 = this._sumPositions(todayPortfolio.AggregatedPositions, 'Value');
+        const yValues = this._getPortfolioValues(yesterdayPortfolio);
+        const tValues = this._getPortfolioValues(todayPortfolio);
 
-        // Only include users who have some form of positions
-        if (invested_day1 === 0 && invested_day2 === 0 && value_day1 === 0 && value_day2 === 0) {
-            return;
-        }
+        this.total_portfolio_value_yesterday += yValues.portfolioValue;
+        this.total_invested_yesterday += yValues.invested;
 
-        this.total_invested_day1 += invested_day1;
-        this.total_value_day1 += value_day1;
-        this.total_invested_day2 += invested_day2;
-        this.total_value_day2 += value_day2;
-        this.user_count++;
+        this.total_portfolio_value_today += tValues.portfolioValue;
+        this.total_invested_today += tValues.invested;
     }
 
     getResult() {
-        if (this.user_count === 0) {
-            return {}; // No users processed, return empty.
-        }
-
-        // 1. Calculate the average portfolio for the crowd
-        const avg_invested_day1 = this.total_invested_day1 / this.user_count;
-        const avg_value_day1 = this.total_value_day1 / this.user_count;
-        const avg_invested_day2 = this.total_invested_day2 / this.user_count;
-        const avg_value_day2 = this.total_value_day2 / this.user_count;
+        // P&L is the change in *invested* value, not total value
+        const totalPnl = this.total_invested_today - this.total_invested_yesterday;
 
-        // 2. Isolate the three effects
-        const total_value_change = avg_value_day2 - avg_value_day1;
-        const pl_effect = avg_value_day1 - avg_invested_day1;
-        const trading_effect = avg_invested_day2 - avg_invested_day1;
-
-        // 3. Solve for the Cash Flow Effect
-        // Total_Change = pl_effect + trading_effect + cash_flow_effect
-        const cash_flow_effect = total_value_change - pl_effect - trading_effect;
+        // The change in total value is (P&L + Net Cash Flow)
+        // So, Net Cash Flow = (Total Value Change) - P&L
+        const totalValueChange = this.total_portfolio_value_today - this.total_portfolio_value_yesterday;
+        
+        const netCashFlowProxy = totalValueChange - totalPnl;
+        
+        const netCashFlowProxyPct = (this.total_portfolio_value_yesterday > 0)
+            ? (netCashFlowProxy / this.total_portfolio_value_yesterday) * 100
+            : 0;
 
         return {
-            // The final proxy value.
-            // A negative value signals a net DEPOSIT.
-            // A positive value signals a net WITHDRAWAL.
-            cash_flow_effect_proxy: cash_flow_effect,
-            
-            // Interpretation for the frontend
-            interpretation: "A negative value signals a net crowd deposit. A positive value signals a net crowd withdrawal.",
+            total_portfolio_value_yesterday: this.total_portfolio_value_yesterday,
+            total_invested_yesterday: this.total_invested_yesterday,
+            total_portfolio_value_today: this.total_portfolio_value_today,
+            total_invested_today: this.total_invested_today,
             
-            // Debug components
-            components: {
-                total_value_change: total_value_change,
-                pl_effect: pl_effect,
-                trading_effect: trading_effect
-            },
-            // Debug averages
-            averages: {
-                avg_invested_day1: avg_invested_day1,
-                avg_value_day1: avg_value_day1,
-                avg_invested_day2: avg_invested_day2,
-                avg_value_day2: avg_value_day2
-            },
-            user_sample_size: this.user_count
+            total_pnl_contribution: totalPnl,
+            net_cash_flow_proxy: netCashFlowProxy,
+            net_cash_flow_proxy_pct: netCashFlowProxyPct
         };
     }
 
     reset() {
-        this.total_invested_day1 = 0;
-        this.total_value_day1 = 0;
-        this.total_invested_day2 = 0;
-        this.total_value_day2 = 0;
-        this.user_count = 0;
+        this.total_portfolio_value_yesterday = 0;
+        this.total_invested_yesterday = 0;
+        this.total_portfolio_value_today = 0;
+        this.total_invested_today = 0;
     }
 }