aiden-shared-calculations-unified 1.0.64 → 1.0.66

package/calculations/behavioural/historical/smart-cohort-flow.js

@@ -1,218 +1,238 @@
 /**
- * @fileoverview Calculates "Net Crowd Flow" and "Sector Rotation"
- * *only* for the "Smart Cohort" (Top 20% of Investor Scores).
+ * @fileoverview Calculation (Pass 3) for smart cohort flow.
  *
- * --- META REFACTOR (v2) ---
- * This calculation is `type: "meta"` and expects its dependencies
- * (the user-investment-profile results) to be fetched by the pass runner.
- * It then streams root portfolio data.
+ * This metric calculates the "Net Crowd Flow Percentage" for the
+ * "Smart Cohort" (top 20% of investors).
+ *
+ * This calculation *depends* on 'user_profitability_tracker'
+ * to identify the cohort.
  */
-
-const { Firestore } = require('@google-cloud/firestore');
-const firestore = new Firestore();
-const { loadAllPriceData, getDailyPriceChange } = require('../../../utils/price_data_provider');
-const { loadInstrumentMappings, getInstrumentSectorMap } = require('../../../utils/sector_mapping_provider');
-// NOTE: Corrected relative path for data_loader
-const { loadDataByRefs, getPortfolioPartRefs, loadFullDayMap } = require('../../../../bulltrackers-module/functions/computation-system/utils/data_loader');
-
-const COHORT_PERCENTILE = 0.8; // Top 20%
-const PROFILE_CALC_ID = 'user-investment-profile'; // The calc to read IS scores from
+const { loadInstrumentMappings } = require('../../../utils/sector_mapping_provider');
 
 class SmartCohortFlow {
+    constructor() {
+        this.assetData = new Map();
+        this.sectorData = new Map();
+        this.mappings = null;
+        this.smartCohortUserIds = null;
+    }
 
     /**
-     * (NEW) Statically declare dependencies.
+     * Defines the output schema for this calculation.
+     * @returns {object} JSON Schema object
      */
-    static getDependencies() {
-        return [PROFILE_CALC_ID];
-    }
-    
-    constructor() {
-        // Meta-calc, no constructor state needed
+    static getSchema() {
+        const flowSchema = {
+            "type": "object",
+            "properties": {
+                "net_flow_percentage": { "type": "number" },
+                "total_invested_today": { "type": "number" },
+                "total_invested_yesterday": { "type": "number" }
+            },
+            "required": ["net_flow_percentage", "total_invested_today", "total_invested_yesterday"]
+        };
+
+        return {
+            "type": "object",
+            "description": "Calculates net capital flow % (price-adjusted) for the 'Smart Cohort' (top 20% users), aggregated by asset and sector.",
+            "properties": {
+                "cohort_size": {
+                    "type": "number",
+                    "description": "The number of users identified as being in the Smart Cohort."
+                },
+                "assets": {
+                    "type": "object",
+                    "description": "Price-adjusted net flow per asset.",
+                    "patternProperties": { "^.*$": flowSchema }, // Ticker
+                    "additionalProperties": flowSchema
+                },
+                "sectors": {
+                    "type": "object",
+                    "description": "Price-adjusted net flow per sector.",
+                    "patternProperties": { "^.*$": flowSchema }, // Sector
+                    "additionalProperties": flowSchema
+                }
+            },
+            "required": ["cohort_size", "assets", "sectors"]
+        };
     }
 
     /**
-     * Loads the Investor Scores from the fetched dependency.
+     * Statically declare dependencies.
      */
-    _loadCohort(logger, fetchedDependencies) {
-        logger.log('INFO', '[SmartCohortFlow] Loading Investor Scores from fetched dependency...');
-        
-        const profileData = fetchedDependencies[PROFILE_CALC_ID];
-        
-        if (!profileData || !profileData.daily_investor_scores || Object.keys(profileData.daily_investor_scores).length === 0) {
-            logger.log('WARN', `[SmartCohortFlow] Cannot find dependency in fetched data: ${PROFILE_CALC_ID}. Cohort will not be built.`);
-            return null; // Return null to signal failure
-        }
-            
-        const scores = profileData.daily_investor_scores;
-        const allScores = Object.entries(scores).map(([userId, score]) => ({ userId, score }));
-        allScores.sort((a, b) => a.score - b.score);
-        
-        const thresholdIndex = Math.floor(allScores.length * COHORT_PERCENTILE);
-        const thresholdScore = allScores[thresholdIndex]?.score || 999;
-        
-        const smartCohortIds = new Set(
-            allScores.filter(s => s.score >= thresholdScore).map(s => s.userId)
-        );
-        
-        logger.log('INFO', `[SmartCohortFlow] Cohort built. ${smartCohortIds.size} users at or above ${thresholdScore.toFixed(2)} (80th percentile).`);
-        return smartCohortIds;
+    static getDependencies() {
+        return ['user_profitability_tracker'];
     }
 
-    // --- Asset Flow Helpers (unchanged) ---
-    _initAsset(asset_values, instrumentId) {
-        if (!asset_values[instrumentId]) {
-            asset_values[instrumentId] = { day1_value_sum: 0, day2_value_sum: 0 };
-        }
+    _getPortfolioPositions(portfolio) {
+        return portfolio?.PublicPositions || portfolio?.AggregatedPositions;
     }
-    _sumAssetValue(positions) {
-        const valueMap = {};
-        if (!positions || !Array.isArray(positions)) return valueMap;
-        for (const pos of positions) {
-            if (pos && pos.InstrumentID && pos.Value) {
-                valueMap[pos.InstrumentID] = (valueMap[pos.InstrumentID] || 0) + pos.Value;
-            }
+
+    _initAsset(instrumentId) {
+        if (!this.assetData.has(instrumentId)) {
+            this.assetData.set(instrumentId, {
+                total_invested_yesterday: 0,
+                total_invested_today: 0,
+                price_change_yesterday: 0,
+            });
         }
-        return valueMap;
     }
-    _accumulateSectorInvestment(portfolio, target, sectorMap) {
-        if (portfolio && portfolio.AggregatedPositions) {
-            for (const pos of portfolio.AggregatedPositions) {
-                const sector = sectorMap[pos.InstrumentID] || 'N/A';
-                target[sector] = (target[sector] || 0) + (pos.Invested || pos.Amount || 0);
-            }
+    
+    _initSector(sector) {
+        if (!this.sectorData.has(sector)) {
+            this.sectorData.set(sector, {
+                total_invested_yesterday: 0,
+                total_invested_today: 0,
+                price_change_yesterday: 0,
+            });
         }
     }
 
     /**
-     * REFACTORED PROCESS METHOD
-     * @param {string} dateStr The date to run the analysis for (e.g., "2025-10-31").
-     * @param {object} dependencies The shared dependencies (db, logger, rootData, etc.).
-     * @param {object} config The computation system configuration.
-     * @param {object} fetchedDependencies In-memory results from previous passes.
-     * @returns {Promise<object|null>} The analysis result or null.
+     * Helper to get the cohort IDs from the dependency.
      */
-    async process(dateStr, dependencies, config, fetchedDependencies) {
-        const { logger, db, rootData, calculationUtils } = dependencies;
-        const { portfolioRefs } = rootData;
-        logger.log('INFO', '[SmartCohortFlow] Starting meta-process...');
+    _getSmartCohort(fetchedDependencies) {
+        if (this.smartCohortUserIds) {
+            return this.smartCohortUserIds;
+        }
         
-        // 1. Load Cohort from in-memory dependency
-        const smartCohortIds = this._loadCohort(logger, fetchedDependencies);
-        if (!smartCohortIds) {
-            return null; // Dependency failed
+        const profitabilityData = fetchedDependencies['user_profitability_tracker'];
+        if (!profitabilityData || !profitabilityData.ranked_users) {
+            return new Set();
         }
 
-        // 2. Load external dependencies (prices, sectors)
-        const [priceMap, mappings, sectorMap] = await Promise.all([
-            loadAllPriceData(),
-            loadInstrumentMappings(),
-            getInstrumentSectorMap()
-        ]);
-        if (!priceMap || !mappings || !sectorMap || Object.keys(priceMap).length === 0) {
-            logger.log('ERROR', '[SmartCohortFlow] Failed to load critical price/mapping/sector data. Aborting.');
-            return null; 
-        }
+        const rankedUsers = profitabilityData.ranked_users;
+        const cohortSize = Math.floor(rankedUsers.length * 0.20);
+        
+        // The 'ranked_users' are sorted ascending by P&L, so top 20% is the *last* 20%.
+        this.smartCohortUserIds = new Set(rankedUsers.slice(-cohortSize).map(u => u.userId));
+        return this.smartCohortUserIds;
+    }
 
-        // 3. Load "yesterday's" portfolio data for comparison
-        const yesterdayDate = new Date(dateStr + 'T00:00:00Z');
-        yesterdayDate.setUTCDate(yesterdayDate.getUTCDate() - 1);
-        const yesterdayStr = yesterdayDate.toISOString().slice(0, 10);
-        const yesterdayRefs = await getPortfolioPartRefs(config, dependencies, yesterdayStr);
-        const yesterdayPortfolios = await loadFullDayMap(config, dependencies, yesterdayRefs);
-        logger.log('INFO', `[SmartCohortFlow] Loaded ${yesterdayRefs.length} part refs for yesterday.`);
-
-        // 4. Stream "today's" portfolio data and process
-        const asset_values = {};
-        const todaySectorInvestment = {};
-        const yesterdaySectorInvestment = {};
-        let user_count = 0;
-
-        const batchSize = config.partRefBatchSize || 10;
-        for (let i = 0; i < portfolioRefs.length; i += batchSize) {
-            const batchRefs = portfolioRefs.slice(i, i + batchSize);
-            const todayPortfoliosChunk = await loadDataByRefs(config, dependencies, batchRefs);
-
-            for (const uid in todayPortfoliosChunk) {
-                
-                if (!smartCohortIds.has(uid)) continue; // --- Filter user ---
-                
-                const pToday = todayPortfoliosChunk[uid];
-                const pYesterday = yesterdayPortfolios[uid];
-                
-                if (!pToday || !pYesterday || !pToday.AggregatedPositions || !pYesterday.AggregatedPositions) {
-                    continue;
-                }
-                
-                // 4a. RUN ASSET FLOW LOGIC
-                const yesterdayValues = this._sumAssetValue(pYesterday.AggregatedPositions);
-                const todayValues = this._sumAssetValue(pToday.AggregatedPositions);
-                const allInstrumentIds = new Set([...Object.keys(yesterdayValues), ...Object.keys(todayValues)]);
-                
-                for (const instrumentId of allInstrumentIds) {
-                    this._initAsset(asset_values, instrumentId);
-                    asset_values[instrumentId].day1_value_sum += (yesterdayValues[instrumentId] || 0);
-                    asset_values[instrumentId].day2_value_sum += (todayValues[instrumentId] || 0);
-                }
+    process(todayPortfolio, yesterdayPortfolio, userId, context, todayInsights, yesterdayInsights, fetchedDependencies) {
+        const smartCohort = this._getSmartCohort(fetchedDependencies);
+        
+        // This user is not in the "smart cohort", skip.
+        if (!smartCohort.has(userId)) {
+            return;
+        }
 
-                // 4b. RUN SECTOR ROTATION LOGIC
-                this._accumulateSectorInvestment(pToday, todaySectorInvestment, sectorMap);
-                this._accumulateSectorInvestment(pYesterday, yesterdaySectorInvestment, sectorMap);
-                user_count++;
-            }
+        if (!todayPortfolio || !yesterdayPortfolio) {
+            return;
         }
+
+        const yPos = this._getPortfolioPositions(yesterdayPortfolio);
+        const tPos = this._getPortfolioPositions(todayPortfolio);
+
+        const yPosMap = new Map(yPos?.map(p => [p.InstrumentID, p]) || []);
+        const tPosMap = new Map(tPos?.map(p => [p.InstrumentID, p]) || []);
+
+        const allInstrumentIds = new Set([...yPosMap.keys(), ...tPosMap.keys()]);
         
-        logger.log('INFO', `[SmartCohortFlow] Processed ${user_count} users in cohort.`);
-        
-        // --- 5. GETRESULT LOGIC ---
-        if (user_count === 0) {
-            logger.warn('[SmartCohortFlow] No users processed for smart cohort. Returning null.');
-            return null;
+        if (!this.mappings) {
+            // Context contains the mappings loaded in Pass 1
+            this.mappings = context.mappings;
         }
 
-        // 5a. Calculate Asset Flow
-        const finalAssetFlow = {};
-        for (const instrumentId in asset_values) {
-            const ticker = mappings.instrumentToTicker[instrumentId] || `id_${instrumentId}`;
-            const avg_day1_value = asset_values[instrumentId].day1_value_sum / user_count;
-            const avg_day2_value = asset_values[instrumentId].day2_value_sum / user_count;
-            const priceChangePct = getDailyPriceChange(instrumentId, yesterdayStr, dateStr, priceMap);
+        for (const instrumentId of allInstrumentIds) {
+            if (!instrumentId) continue;
+            
+            this._initAsset(instrumentId);
+            const asset = this.assetData.get(instrumentId);
 
-            if (priceChangePct === null) continue;
+            const yP = yPosMap.get(instrumentId);
+            const tP = tPosMap.get(instrumentId);
 
-            const expected_day2_value = avg_day1_value * (1 + priceChangePct);
-            const net_crowd_flow_pct = avg_day2_value - expected_day2_value;
+            const yInvested = yP?.InvestedAmount || yP?.Amount || 0;
+            const tInvested = tP?.InvestedAmount || tP?.Amount || 0;
+            
+            // Get sector and initialize it
+            const sector = this.mappings.instrumentToSector[instrumentId] || 'Other';
+            this._initSector(sector);
+            const sectorAsset = this.sectorData.get(sector);
 
-            finalAssetFlow[ticker] = {
-                net_crowd_flow_pct: net_crowd_flow_pct,
-                avg_value_day1_pct: avg_day1_value,
-                avg_value_day2_pct: avg_day2_value
-            };
+            if (yInvested > 0) {
+                const yPriceChange = (yP?.PipsRate || 0) / (yP?.OpenRate || 1);
+                
+                asset.total_invested_yesterday += yInvested;
+                asset.price_change_yesterday += yPriceChange * yInvested;
+                
+                sectorAsset.total_invested_yesterday += yInvested;
+                sectorAsset.price_change_yesterday += yPriceChange * yInvested;
+            }
+            if (tInvested > 0) {
+                asset.total_invested_today += tInvested;
+                sectorAsset.total_invested_today += tInvested;
+            }
         }
-        
-        // 5b. Calculate Sector Rotation
-        const finalSectorRotation = {};
-        const allSectors = new Set([...Object.keys(todaySectorInvestment), ...Object.keys(yesterdaySectorInvestment)]);
-        for (const sector of allSectors) {
-            const todayAmount = todaySectorInvestment[sector] || 0;
-            const yesterdayAmount = yesterdaySectorInvestment[sector] || 0;
-            finalSectorRotation[sector] = todayAmount - yesterdayAmount;
+    }
+    
+    _calculateFlow(dataMap) {
+        const result = {};
+        for (const [key, data] of dataMap.entries()) {
+            const { total_invested_yesterday, total_invested_today, price_change_yesterday } = data;
+            
+            if (total_invested_yesterday > 0) {
+                const avg_price_change_pct = price_change_yesterday / total_invested_yesterday;
+                const price_contribution = total_invested_yesterday * avg_price_change_pct;
+                const flow_contribution = total_invested_today - (total_invested_yesterday + price_contribution);
+                const net_flow_percentage = (flow_contribution / total_invested_yesterday) * 100;
+                
+                result[key] = {
+                    net_flow_percentage: net_flow_percentage,
+                    total_invested_today: total_invested_today,
+                    total_invested_yesterday: total_invested_yesterday
+                };
+            }
         }
+        return result;
+    }
 
-        if (Object.keys(finalAssetFlow).length === 0) {
-            logger.warn('[SmartCohortFlow] No asset flow calculated (likely all price data missing). Returning null.');
-            return null;
+    async getResult(fetchedDependencies) {
+        // Ensure mappings are loaded (can be from context or loaded now)
+        if (!this.mappings) {
+            this.mappings = await loadInstrumentMappings();
+        }
+        
+        // Ensure cohort is calculated at least once
+        const smartCohort = this._getSmartCohort(fetchedDependencies);
+
+        // 1. Calculate Asset Flow
+        const assetResult = {};
+        for (const [instrumentId, data] of this.assetData.entries()) {
+            const ticker = this.mappings.instrumentToTicker[instrumentId] || `id_${instrumentId}`;
+            const { total_invested_yesterday, total_invested_today, price_change_yesterday } = data;
+            
+            if (total_invested_yesterday > 0) {
+                const avg_price_change_pct = price_change_yesterday / total_invested_yesterday;
+                const price_contribution = total_invested_yesterday * avg_price_change_pct;
+                const flow_contribution = total_invested_today - (total_invested_yesterday + price_contribution);
+                const net_flow_percentage = (flow_contribution / total_invested_yesterday) * 100;
+                
+                assetResult[ticker] = {
+                    net_flow_percentage: net_flow_percentage,
+                    total_invested_today: total_invested_today,
+                    total_invested_yesterday: total_invested_yesterday
+                };
+            }
         }
+        
+        // 2. Calculate Sector Flow
+        const sectorResult = this._calculateFlow(this.sectorData);
 
         return {
-            asset_flow: finalAssetFlow,
-            sector_rotation: finalSectorRotation,
-            user_sample_size: user_count
+            cohort_size: smartCohort.size,
+            assets: assetResult,
+            sectors: sectorResult
         };
     }
 
-    async getResult() { return null; }
-    reset() { }
+    reset() {
+        this.assetData.clear();
+        this.sectorData.clear();
+        this.mappings = null;
+        this.smartCohortUserIds = null;
+    }
 }
 
 module.exports = SmartCohortFlow;