aiden-shared-calculations-unified 1.0.64 → 1.0.66

package/calculations/behavioural/historical/drawdown_response.js

@@ -1,58 +1,136 @@
 /**
- * Analyzes user behavior after a position experiences a >10% drawdown.
+ * @fileoverview Calculation (Pass 2) for drawdown response.
+ *
+ * This metric answers: "How do users behave when a position has a
+ * drawdown of over 10%?"
+ *
+ * It checks all positions from yesterday that were in >10% loss
+ * and tracks whether the user:
+ * 1. Held (position still open today, same size)
+ * 2. Closed (position no longer open today)
+ * 3. Added (position still open, but Invested % is larger)
  */
 class DrawdownResponse {
     constructor() {
-        this.drawdown_events = {
-            held_position: 0,
-            closed_position: 0,
-            added_to_position: 0
+        this.actions = {
+            held: 0,
+            closed: 0,
+            added: 0
         };
+        this.total_in_drawdown = 0;
     }
 
-    process(todayPortfolio, yesterdayPortfolio, userId) {
-        if (!yesterdayPortfolio || !todayPortfolio) {
-            return; // Need both days for comparison
+    /**
+     * Defines the output schema for this calculation.
+     * @returns {object} JSON Schema object
+     */
+    static getSchema() {
+        return {
+            "type": "object",
+            "description": "Analyzes user behavior in response to a >10% position drawdown.",
+            "properties": {
+                "total_positions_in_drawdown": {
+                    "type": "number",
+                    "description": "Total positions from yesterday that were in >10% drawdown."
+                },
+                "action_held_pct": {
+                    "type": "number",
+                    "description": "Percentage of drawdown positions that were held."
+                },
+                "action_closed_pct": {
+                    "type": "number",
+                    "description": "Percentage of drawdown positions that were closed."
+                },
+                "action_added_pct": {
+                    "type": "number",
+                    "description": "Percentage of drawdown positions that were added to."
+                },
+                "raw_counts": {
+                    "type": "object",
+                    "properties": {
+                        "held": { "type": "number" },
+                        "closed": { "type": "number" },
+                        "added": { "type": "number" }
+                    },
+                    "required": ["held", "closed", "added"]
+                }
+            },
+            "required": ["total_positions_in_drawdown", "action_held_pct", "action_closed_pct", "action_added_pct", "raw_counts"]
+        };
+    }
+
+    _getPortfolioMap(portfolio) {
+        // We MUST use PositionID to track specific trades
+        const positions = portfolio?.AggregatedPositions || portfolio?.PublicPositions;
+        if (!positions || !Array.isArray(positions)) {
+            return new Map();
+        }
+        // Map<PositionID, { pnl: number, invested: number }>
+        return new Map(positions.map(p => [p.PositionID, {
+            pnl: (p.NetProfit || 0) / (p.InvestedAmount || p.Amount || 1), // PNL as %
+            invested: p.InvestedAmount || p.Amount || 0
+        }]));
+    }
+
+    process(todayPortfolio, yesterdayPortfolio) {
+        if (!todayPortfolio || !yesterdayPortfolio) {
+            return;
         }
 
-        const yPositions = yesterdayPortfolio.AggregatedPositions || yesterdayPortfolio.PublicPositions;
-        const tPositions = todayPortfolio.AggregatedPositions || todayPortfolio.PublicPositions;
+        const yPosMap = this._getPortfolioMap(yesterdayPortfolio);
+        const tPosMap = this._getPortfolioMap(todayPortfolio);
 
-        if (!yPositions || !Array.isArray(yPositions) || !tPositions || !Array.isArray(tPositions)) {
+        if (yPosMap.size === 0) {
             return;
         }
 
-        // Use PositionID if available (as in original file), fallback to InstrumentID
-        const todayPositions = new Map(tPositions.map(p => [p.PositionID || p.InstrumentID, p]));
-
-        for (const yPos of yPositions) {
-            // FIX: Use the NetProfit field, which is already a percentage.
-            // Your data sample (e.g., -83.6) shows the threshold should be -10.0.
-            const drawdownPercent = yPos.NetProfit || 0;
-            const yPosId = yPos.PositionID || yPos.InstrumentID;
-
-            // Check if this position was in a >10% drawdown yesterday
-            if (drawdownPercent < -10.0) {
-                const todayPos = todayPositions.get(yPosId);
-
-                if (!todayPos) {
-                    // Position was closed
-                    this.drawdown_events.closed_position++;
-                } else if (todayPos.Invested > yPos.Invested) {
-                    // FIX: Use 'Invested' (percentage) to check for increase
-                    // User added money to the losing position
-                    this.drawdown_events.added_to_position++;
+        for (const [yPosId, yPosData] of yPosMap.entries()) {
+            // Check if position was in >10% drawdown
+            if (yPosData.pnl < -0.10) {
+                this.total_in_drawdown++;
+
+                // Now, check what happened today
+                if (!tPosMap.has(yPosId)) {
+                    // 1. Position was closed
+                    this.actions.closed++;
                 } else {
-                    // Position was held (or reduced, but not added to)
-                    this.drawdown_events.held_position++;
+                    const tPosData = tPosMap.get(yPosId);
+                    // 2. Position was added to (check for > 1% increase to avoid noise)
+                    if (tPosData.invested > (yPosData.invested * 1.01)) {
+                        this.actions.added++;
+                    } else {
+                        // 3. Position was held
+                        this.actions.held++;
+                    }
                 }
             }
         }
     }
 
     getResult() {
-        // Return final calculated values
-        return this.drawdown_events;
+        const total = this.total_in_drawdown;
+        if (total === 0) {
+            return {
+                total_positions_in_drawdown: 0,
+                action_held_pct: 0,
+                action_closed_pct: 0,
+                action_added_pct: 0,
+                raw_counts: this.actions
+            };
+        }
+
+        return {
+            total_positions_in_drawdown: total,
+            action_held_pct: (this.actions.held / total) * 100,
+            action_closed_pct: (this.actions.closed / total) * 100,
+            action_added_pct: (this.actions.added / total) * 100,
+            raw_counts: this.actions
+        };
+    }
+
+    reset() {
+        this.actions = { held: 0, closed: 0, added: 0 };
+        this.total_in_drawdown = 0;
     }
 }
 

package/calculations/behavioural/historical/dumb-cohort-flow.js

@@ -1,218 +1,238 @@
 /**
- * @fileoverview Calculates "Net Crowd Flow" and "Sector Rotation"
- * *only* for the "Dumb Cohort" (Bottom 20% of Investor Scores).
+ * @fileoverview Calculation (Pass 3) for dumb cohort flow.
  *
- * --- META REFACTOR (v2) ---
- * This calculation is `type: "meta"` and expects its dependencies
- * (the user-investment-profile results) to be fetched by the pass runner.
- * It then streams root portfolio data.
+ * This metric calculates the "Net Crowd Flow Percentage" for the
+ * "Dumb Cohort" (bottom 20% of investors).
+ *
+ * This calculation *depends* on 'user_profitability_tracker'
+ * to identify the cohort.
  */
-
-const { Firestore } = require('@google-cloud/firestore');
-const firestore = new Firestore();
-const { loadAllPriceData, getDailyPriceChange } = require('../../../utils/price_data_provider');
-const { loadInstrumentMappings, getInstrumentSectorMap } = require('../../../utils/sector_mapping_provider');
-// NOTE: Corrected relative path for data_loader
-const { loadDataByRefs, getPortfolioPartRefs, loadFullDayMap } = require('../../../../bulltrackers-module/functions/computation-system/utils/data_loader');
-
-const COHORT_PERCENTILE = 0.2; // Bottom 20%
-const PROFILE_CALC_ID = 'user-investment-profile'; // The calc to read IS scores from
+const { loadInstrumentMappings } = require('../../../utils/sector_mapping_provider');
 
 class DumbCohortFlow {
+    constructor() {
+        this.assetData = new Map();
+        this.sectorData = new Map();
+        this.mappings = null;
+        this.dumbCohortUserIds = null;
+    }
 
     /**
-     * (NEW) Statically declare dependencies.
+     * Defines the output schema for this calculation.
+     * @returns {object} JSON Schema object
      */
-    static getDependencies() {
-        return [PROFILE_CALC_ID];
-    }
+    static getSchema() {
+        const flowSchema = {
+            "type": "object",
+            "properties": {
+                "net_flow_percentage": { "type": "number" },
+                "total_invested_today": { "type": "number" },
+                "total_invested_yesterday": { "type": "number" }
+            },
+            "required": ["net_flow_percentage", "total_invested_today", "total_invested_yesterday"]
+        };
 
-    constructor() {
-        // Meta-calc, no constructor state needed
+        return {
+            "type": "object",
+            "description": "Calculates net capital flow % (price-adjusted) for the 'Dumb Cohort' (bottom 20% users), aggregated by asset and sector.",
+            "properties": {
+                "cohort_size": {
+                    "type": "number",
+                    "description": "The number of users identified as being in the Dumb Cohort."
+                },
+                "assets": {
+                    "type": "object",
+                    "description": "Price-adjusted net flow per asset.",
+                    "patternProperties": { "^.*$": flowSchema }, // Ticker
+                    "additionalProperties": flowSchema
+                },
+                "sectors": {
+                    "type": "object",
+                    "description": "Price-adjusted net flow per sector.",
+                    "patternProperties": { "^.*$": flowSchema }, // Sector
+                    "additionalProperties": flowSchema
+                }
+            },
+            "required": ["cohort_size", "assets", "sectors"]
+        };
     }
 
     /**
-     * Loads the Investor Scores from the fetched dependency.
+     * Statically declare dependencies.
      */
-    _loadCohort(logger, fetchedDependencies) {
-        logger.log('INFO', '[DumbCohortFlow] Loading Investor Scores from fetched dependency...');
-        
-        const profileData = fetchedDependencies[PROFILE_CALC_ID];
-        
-        if (!profileData || !profileData.daily_investor_scores || Object.keys(profileData.daily_investor_scores).length === 0) {
-            logger.log('WARN', `[DumbCohortFlow] Cannot find dependency in fetched data: ${PROFILE_CALC_ID}. Cohort will not be built.`);
-            return null; // Return null to signal failure
-        }
-            
-        const scores = profileData.daily_investor_scores;
-        const allScores = Object.entries(scores).map(([userId, score]) => ({ userId, score }));
-        allScores.sort((a, b) => a.score - b.score);
-        
-        const thresholdIndex = Math.floor(allScores.length * COHORT_PERCENTILE);
-        const thresholdScore = allScores[thresholdIndex]?.score || 0; // Get 20th percentile score
-        
-        const dumbCohortIds = new Set(
-            allScores.filter(s => s.score <= thresholdScore).map(s => s.userId) // Get users *at or below*
-        );
-        
-        logger.log('INFO', `[DumbCohortFlow] Cohort built. ${dumbCohortIds.size} users at or below ${thresholdScore.toFixed(2)} (20th percentile).`);
-        return dumbCohortIds;
+    static getDependencies() {
+        return ['user_profitability_tracker'];
     }
 
-    // --- Asset Flow Helpers (unchanged) ---
-    _initAsset(asset_values, instrumentId) {
-        if (!asset_values[instrumentId]) {
-            asset_values[instrumentId] = { day1_value_sum: 0, day2_value_sum: 0 };
-        }
+    _getPortfolioPositions(portfolio) {
+        return portfolio?.PublicPositions || portfolio?.AggregatedPositions;
     }
-    _sumAssetValue(positions) {
-        const valueMap = {};
-        if (!positions || !Array.isArray(positions)) return valueMap;
-        for (const pos of positions) {
-            if (pos && pos.InstrumentID && pos.Value) {
-                valueMap[pos.InstrumentID] = (valueMap[pos.InstrumentID] || 0) + pos.Value;
-            }
+
+    _initAsset(instrumentId) {
+        if (!this.assetData.has(instrumentId)) {
+            this.assetData.set(instrumentId, {
+                total_invested_yesterday: 0,
+                total_invested_today: 0,
+                price_change_yesterday: 0,
+            });
         }
-        return valueMap;
     }
-    _accumulateSectorInvestment(portfolio, target, sectorMap) {
-        if (portfolio && portfolio.AggregatedPositions) {
-            for (const pos of portfolio.AggregatedPositions) {
-                const sector = sectorMap[pos.InstrumentID] || 'N/A';
-                target[sector] = (target[sector] || 0) + (pos.Invested || pos.Amount || 0);
-            }
+    
+    _initSector(sector) {
+        if (!this.sectorData.has(sector)) {
+            this.sectorData.set(sector, {
+                total_invested_yesterday: 0,
+                total_invested_today: 0,
+                price_change_yesterday: 0,
+            });
         }
     }
 
     /**
-     * REFACTORED PROCESS METHOD
-     * @param {string} dateStr The date to run the analysis for (e.g., "2025-10-31").
-     * @param {object} dependencies The shared dependencies (db, logger, rootData, etc.).
-     * @param {object} config The computation system configuration.
-     * @param {object} fetchedDependencies In-memory results from previous passes.
-     * @returns {Promise<object|null>} The analysis result or null.
+     * Helper to get the cohort IDs from the dependency.
      */
-    async process(dateStr, dependencies, config, fetchedDependencies) {
-        const { logger, db, rootData, calculationUtils } = dependencies;
-        const { portfolioRefs } = rootData;
-        logger.log('INFO', '[DumbCohortFlow] Starting meta-process...');
+    _getDumbCohort(fetchedDependencies) {
+        if (this.dumbCohortUserIds) {
+            return this.dumbCohortUserIds;
+        }
         
-        // 1. Load Cohort from in-memory dependency
-        const dumbCohortIds = this._loadCohort(logger, fetchedDependencies);
-        if (!dumbCohortIds) {
-            return null; // Dependency failed
+        const profitabilityData = fetchedDependencies['user_profitability_tracker'];
+        if (!profitabilityData || !profitabilityData.ranked_users) {
+            return new Set();
         }
 
-        // 2. Load external dependencies (prices, sectors)
-        const [priceMap, mappings, sectorMap] = await Promise.all([
-            loadAllPriceData(),
-            loadInstrumentMappings(),
-            getInstrumentSectorMap()
-        ]);
-        if (!priceMap || !mappings || !sectorMap || Object.keys(priceMap).length === 0) {
-            logger.log('ERROR', '[DumbCohortFlow] Failed to load critical price/mapping/sector data. Aborting.');
-            return null;
-        }
+        const rankedUsers = profitabilityData.ranked_users;
+        const cohortSize = Math.floor(rankedUsers.length * 0.20);
+        
+        // The 'ranked_users' are sorted ascending by P&L, so bottom 20% is the first 20%.
+        this.dumbCohortUserIds = new Set(rankedUsers.slice(0, cohortSize).map(u => u.userId));
+        return this.dumbCohortUserIds;
+    }
 
-        // 3. Load "yesterday's" portfolio data for comparison
-        const yesterdayDate = new Date(dateStr + 'T00:00:00Z');
-        yesterdayDate.setUTCDate(yesterdayDate.getUTCDate() - 1);
-        const yesterdayStr = yesterdayDate.toISOString().slice(0, 10);
-        const yesterdayRefs = await getPortfolioPartRefs(config, dependencies, yesterdayStr);
-        const yesterdayPortfolios = await loadFullDayMap(config, dependencies, yesterdayRefs);
-        logger.log('INFO', `[DumbCohortFlow] Loaded ${yesterdayRefs.length} part refs for yesterday.`);
-
-        // 4. Stream "today's" portfolio data and process
-        const asset_values = {};
-        const todaySectorInvestment = {};
-        const yesterdaySectorInvestment = {};
-        let user_count = 0;
-
-        const batchSize = config.partRefBatchSize || 10;
-        for (let i = 0; i < portfolioRefs.length; i += batchSize) {
-            const batchRefs = portfolioRefs.slice(i, i + batchSize);
-            const todayPortfoliosChunk = await loadDataByRefs(config, dependencies, batchRefs);
-
-            for (const uid in todayPortfoliosChunk) {
-                
-                if (!dumbCohortIds.has(uid)) continue; // --- Filter user ---
-                
-                const pToday = todayPortfoliosChunk[uid];
-                const pYesterday = yesterdayPortfolios[uid];
-                
-                if (!pToday || !pYesterday || !pToday.AggregatedPositions || !pYesterday.AggregatedPositions) {
-                    continue;
-                }
-                
-                // 4a. RUN ASSET FLOW LOGIC
-                const yesterdayValues = this._sumAssetValue(pYesterday.AggregatedPositions);
-                const todayValues = this._sumAssetValue(pToday.AggregatedPositions);
-                const allInstrumentIds = new Set([...Object.keys(yesterdayValues), ...Object.keys(todayValues)]);
-                
-                for (const instrumentId of allInstrumentIds) {
-                    this._initAsset(asset_values, instrumentId);
-                    asset_values[instrumentId].day1_value_sum += (yesterdayValues[instrumentId] || 0);
-                    asset_values[instrumentId].day2_value_sum += (todayValues[instrumentId] || 0);
-                }
+    process(todayPortfolio, yesterdayPortfolio, userId, context, todayInsights, yesterdayInsights, fetchedDependencies) {
+        const dumbCohort = this._getDumbCohort(fetchedDependencies);
+        
+        // This user is not in the "dumb cohort", skip.
+        if (!dumbCohort.has(userId)) {
+            return;
+        }
 
-                // 4b. RUN SECTOR ROTATION LOGIC
-                this._accumulateSectorInvestment(pToday, todaySectorInvestment, sectorMap);
-                this._accumulateSectorInvestment(pYesterday, yesterdaySectorInvestment, sectorMap);
-                user_count++;
-            }
+        if (!todayPortfolio || !yesterdayPortfolio) {
+            return;
         }
+
+        const yPos = this._getPortfolioPositions(yesterdayPortfolio);
+        const tPos = this._getPortfolioPositions(todayPortfolio);
+
+        const yPosMap = new Map(yPos?.map(p => [p.InstrumentID, p]) || []);
+        const tPosMap = new Map(tPos?.map(p => [p.InstrumentID, p]) || []);
+
+        const allInstrumentIds = new Set([...yPosMap.keys(), ...tPosMap.keys()]);
         
-        logger.log('INFO', `[DumbCohortFlow] Processed ${user_count} users in cohort.`);
-        
-        // --- 5. GETRESULT LOGIC ---
-        if (user_count === 0) {
-            logger.warn('[DumbCohortFlow] No users processed for dumb cohort. Returning null.');
-            return null;
+        if (!this.mappings) {
+            // Context contains the mappings loaded in Pass 1
+            this.mappings = context.mappings;
         }
 
-        // 5a. Calculate Asset Flow
-        const finalAssetFlow = {};
-        for (const instrumentId in asset_values) {
-            const ticker = mappings.instrumentToTicker[instrumentId] || `id_${instrumentId}`;
-            const avg_day1_value = asset_values[instrumentId].day1_value_sum / user_count;
-            const avg_day2_value = asset_values[instrumentId].day2_value_sum / user_count;
-            const priceChangePct = getDailyPriceChange(instrumentId, yesterdayStr, dateStr, priceMap);
+        for (const instrumentId of allInstrumentIds) {
+            if (!instrumentId) continue;
+            
+            this._initAsset(instrumentId);
+            const asset = this.assetData.get(instrumentId);
 
-            if (priceChangePct === null) continue;
+            const yP = yPosMap.get(instrumentId);
+            const tP = tPosMap.get(instrumentId);
 
-            const expected_day2_value = avg_day1_value * (1 + priceChangePct);
-            const net_crowd_flow_pct = avg_day2_value - expected_day2_value;
+            const yInvested = yP?.InvestedAmount || yP?.Amount || 0;
+            const tInvested = tP?.InvestedAmount || tP?.Amount || 0;
+            
+            // Get sector and initialize it
+            const sector = this.mappings.instrumentToSector[instrumentId] || 'Other';
+            this._initSector(sector);
+            const sectorAsset = this.sectorData.get(sector);
 
-            finalAssetFlow[ticker] = {
-                net_crowd_flow_pct: net_crowd_flow_pct,
-                avg_value_day1_pct: avg_day1_value,
-                avg_value_day2_pct: avg_day2_value
-            };
+            if (yInvested > 0) {
+                const yPriceChange = (yP?.PipsRate || 0) / (yP?.OpenRate || 1);
+                
+                asset.total_invested_yesterday += yInvested;
+                asset.price_change_yesterday += yPriceChange * yInvested;
+                
+                sectorAsset.total_invested_yesterday += yInvested;
+                sectorAsset.price_change_yesterday += yPriceChange * yInvested;
+            }
+            if (tInvested > 0) {
+                asset.total_invested_today += tInvested;
+                sectorAsset.total_invested_today += tInvested;
+            }
         }
-        
-        // 5b. Calculate Sector Rotation
-        const finalSectorRotation = {};
-        const allSectors = new Set([...Object.keys(todaySectorInvestment), ...Object.keys(yesterdaySectorInvestment)]);
-        for (const sector of allSectors) {
-            const todayAmount = todaySectorInvestment[sector] || 0;
-            const yesterdayAmount = yesterdaySectorInvestment[sector] || 0;
-            finalSectorRotation[sector] = todayAmount - yesterdayAmount;
+    }
+    
+    _calculateFlow(dataMap) {
+        const result = {};
+        for (const [key, data] of dataMap.entries()) {
+            const { total_invested_yesterday, total_invested_today, price_change_yesterday } = data;
+            
+            if (total_invested_yesterday > 0) {
+                const avg_price_change_pct = price_change_yesterday / total_invested_yesterday;
+                const price_contribution = total_invested_yesterday * avg_price_change_pct;
+                const flow_contribution = total_invested_today - (total_invested_yesterday + price_contribution);
+                const net_flow_percentage = (flow_contribution / total_invested_yesterday) * 100;
+                
+                result[key] = {
+                    net_flow_percentage: net_flow_percentage,
+                    total_invested_today: total_invested_today,
+                    total_invested_yesterday: total_invested_yesterday
+                };
+            }
         }
+        return result;
+    }
 
-        if (Object.keys(finalAssetFlow).length === 0) {
-            logger.warn('[DumbCohortFlow] No asset flow calculated (likely all price data missing). Returning null.');
-            return null;
+    async getResult(fetchedDependencies) {
+        // Ensure mappings are loaded (can be from context or loaded now)
+        if (!this.mappings) {
+            this.mappings = await loadInstrumentMappings();
         }
+        
+        // Ensure cohort is calculated at least once
+        const dumbCohort = this._getDumbCohort(fetchedDependencies);
+
+        // 1. Calculate Asset Flow
+        const assetResult = {};
+        for (const [instrumentId, data] of this.assetData.entries()) {
+            const ticker = this.mappings.instrumentToTicker[instrumentId] || `id_${instrumentId}`;
+            const { total_invested_yesterday, total_invested_today, price_change_yesterday } = data;
+            
+            if (total_invested_yesterday > 0) {
+                const avg_price_change_pct = price_change_yesterday / total_invested_yesterday;
+                const price_contribution = total_invested_yesterday * avg_price_change_pct;
+                const flow_contribution = total_invested_today - (total_invested_yesterday + price_contribution);
+                const net_flow_percentage = (flow_contribution / total_invested_yesterday) * 100;
+                
+                assetResult[ticker] = {
+                    net_flow_percentage: net_flow_percentage,
+                    total_invested_today: total_invested_today,
+                    total_invested_yesterday: total_invested_yesterday
+                };
+            }
+        }
+        
+        // 2. Calculate Sector Flow
+        const sectorResult = this._calculateFlow(this.sectorData);
 
         return {
-            asset_flow: finalAssetFlow,
-            sector_rotation: finalSectorRotation,
-            user_sample_size: user_count
+            cohort_size: dumbCohort.size,
+            assets: assetResult,
+            sectors: sectorResult
         };
     }
 
-    async getResult() { return null; }
-    reset() { }
+    reset() {
+        this.assetData.clear();
+        this.sectorData.clear();
+        this.mappings = null;
+        this.dumbCohortUserIds = null;
+    }
 }
 
 module.exports = DumbCohortFlow;