aiden-shared-calculations-unified 1.0.35 → 1.0.37

package/calculations/behavioural/historical/dumb-cohort-flow.js

@@ -1,250 +1,218 @@
-/**
- * @fileoverview Calculates "Net Crowd Flow" and "Sector Rotation"
- * *only* for the "Dumb Cohort" (Bottom 20% of Investor Scores).
- *
- * This calc depends on 'user-investment-profile.js' being run first for the same day.
- */
-
-const { Firestore } = require('@google-cloud/firestore');
-const firestore = new Firestore();
-const { loadAllPriceData, getDailyPriceChange } = require('../../../utils/price_data_provider');
-const { loadInstrumentMappings, getInstrumentSectorMap } = require('../../../utils/sector_mapping_provider');
-
-const COHORT_PERCENTILE = 0.2; // Bottom 20%
-const PROFILE_CALC_ID = 'user-investment-profile'; // The calc to read IS scores from
-
-class DumbCohortFlow {
-    constructor() {
-        // Asset Flow
-        this.asset_values = {}; // { instrumentId: { day1_value_sum: 0, day2_value_sum: 0 } }
-        // Sector Rotation
-        this.todaySectorInvestment = {};
-        this.yesterdaySectorInvestment = {};
-        
-        // --- START MODIFICATION ---
-        this.dumbCohortIds = null; // Set to null. Will be a Set on success.
-        // --- END MODIFICATION ---
-        
-        this.user_count = 0; // Number of *cohort* users
-        this.priceMap = null;
-        this.mappings = null;
-        this.sectorMap = null;
-        this.dates = {};
-    }
-
-    /**
-     * Loads the Investor Scores, calculates the cohort threshold, and builds the Set of user IDs.
-     */
-    async _loadCohort(context, dependencies) {
-        const { db, logger } = dependencies;
-        logger.log('INFO', '[DumbCohortFlow] Loading Investor Scores to build cohort...');
-        
-        try {
-            const scoreMapRef = db.collection(context.config.resultsCollection).doc(context.todayDateStr)
-                .collection(context.config.resultsSubcollection).doc('behavioural')
-                .collection(context.config.computationsSubcollection).doc(PROFILE_CALC_ID);
-                
-            const doc = await scoreMapRef.get();
-            
-            // --- START MODIFICATION ---
-            // Check for doc, data, and that the scores map isn't empty
-            if (!doc.exists || !doc.data().daily_investor_scores || Object.keys(doc.data().daily_investor_scores).length === 0) {
-                logger.log('WARN', '[DumbCohortFlow] Cannot find dependency: daily_investor_scores. Cohort will not be built. Returning null on getResult.');
-                // Keep this.dumbCohortIds = null
-                return; // Abort
-            }
-            // --- END MODIFICATION ---
-            
-            const scores = doc.data().daily_investor_scores;
-            const allScores = Object.entries(scores).map(([userId, score]) => ({ userId, score }));
-            allScores.sort((a, b) => a.score - b.score);
-            
-            const thresholdIndex = Math.floor(allScores.length * COHORT_PERCENTILE);
-            const thresholdScore = allScores[thresholdIndex]?.score || 0; // Get 20th percentile score
-            
-            // --- START MODIFICATION ---
-            // Successfully loaded, now create the Set
-            this.dumbCohortIds = new Set(
-                allScores.filter(s => s.score <= thresholdScore).map(s => s.userId) // Get users *at or below*
-            );
-            // --- END MODIFICATION ---
-            
-            logger.log('INFO', `[DumbCohortFlow] Cohort built. ${this.dumbCohortIds.size} users at or below ${thresholdScore.toFixed(2)} (20th percentile).`);
-            
-        } catch (e) {
-            logger.log('ERROR', '[DumbCohortFlow] Failed to load cohort.', { error: e.message });
-            // Keep this.dumbCohortIds = null on error
-        }
-    }
-
-    // --- Asset Flow Helpers ---
-    _initAsset(instrumentId) {
-        if (!this.asset_values[instrumentId]) {
-            this.asset_values[instrumentId] = { day1_value_sum: 0, day2_value_sum: 0 };
-        }
-    }
-    _sumAssetValue(positions) {
-        const valueMap = {};
-        if (!positions || !Array.isArray(positions)) return valueMap;
-        for (const pos of positions) {
-            if (pos && pos.InstrumentID && pos.Value) {
-                valueMap[pos.InstrumentID] = (valueMap[pos.InstrumentID] || 0) + pos.Value;
-            }
-        }
-        return valueMap;
-    }
-    // --- Sector Rotation Helper ---
-    _accumulateSectorInvestment(portfolio, target) {
-        if (portfolio && portfolio.AggregatedPositions) {
-            for (const pos of portfolio.AggregatedPositions) {
-                const sector = this.sectorMap[pos.InstrumentID] || 'N/A';
-                target[sector] = (target[sector] || 0) + (pos.Invested || pos.Amount || 0);
-            }
-        }
-    }
-
-    /**
-     * PROCESS: Runs daily for each user.
-     */
-    async process(todayPortfolio, yesterdayPortfolio, userId, context) {
-        // 1. Load cohort on first run
-        if (!this.dumbCohortIds) {
-            await this._loadCohort(context, context.dependencies);
-            this.dates.today = context.todayDateStr;
-            this.dates.yesterday = context.yesterdayDateStr;
-        }
-
-        // 2. Filter user
-        // --- START MODIFICATION ---
-        // If cohort failed to load, this.dumbCohortIds will be null, and this check will fail correctly.
-        if (!this.dumbCohortIds || !this.dumbCohortIds.has(userId) || !todayPortfolio || !yesterdayPortfolio || !todayPortfolio.AggregatedPositions || !yesterdayPortfolio.AggregatedPositions) {
-            return;
-        }
-        // --- END MODIFICATION ---
-        
-        // 3. User is in the cohort, load maps if needed
-        if (!this.sectorMap) {
-            this.sectorMap = await getInstrumentSectorMap();
-        }
-
-        // --- 4. RUN ASSET FLOW LOGIC ---
-        const yesterdayValues = this._sumAssetValue(yesterdayPortfolio.AggregatedPositions);
-        const todayValues = this._sumAssetValue(todayPortfolio.AggregatedPositions);
-        const allInstrumentIds = new Set([...Object.keys(yesterdayValues), ...Object.keys(todayValues)]);
-        
-        for (const instrumentId of allInstrumentIds) {
-            this._initAsset(instrumentId);
-            this.asset_values[instrumentId].day1_value_sum += (yesterdayValues[instrumentId] || 0);
-            this.asset_values[instrumentId].day2_value_sum += (todayValues[instrumentId] || 0);
-        }
-
-        // --- 5. RUN SECTOR ROTATION LOGIC ---
-        this._accumulateSectorInvestment(todayPortfolio, this.todaySectorInvestment);
-        this._accumulateSectorInvestment(yesterdayPortfolio, this.yesterdaySectorInvestment);
-
-        this.user_count++;
-    }
-
-    /**
-     * GETRESULT: Aggregates and returns the flow data for the cohort.
-     */
-    async getResult() {
-        // --- START MODIFICATION ---
-        // If cohort IDs were never loaded due to dependency failure, return null.
-        if (this.dumbCohortIds === null) {
-            console.warn('[DumbCohortFlow] Skipping getResult because dependency (user-investment-profile) failed to load.');
-            return null;
-        }
-        
-        // If cohort loaded but no users were processed, also return null (or an empty object, but null is safer for backfill)
-        if (this.user_count === 0 || !this.dates.today) {
-            console.warn('[DumbCohortFlow] No users processed for dumb cohort. Returning null.');
-            return null;
-        }
-        // --- END MODIFICATION ---
-
-        // 1. Load dependencies
-        if (!this.priceMap || !this.mappings) {
-            // --- START MODIFICATION ---
-            // Add error handling for this load, and check for empty priceMap
-            try {
-                const [priceData, mappingData] = await Promise.all([
-                    loadAllPriceData(),
-                    loadInstrumentMappings()
-                ]);
-                this.priceMap = priceData;
-                this.mappings = mappingData;
-
-                if (!this.priceMap || Object.keys(this.priceMap).length === 0) {
-                     console.error('[DumbCohortFlow] CRITICAL: Price map is empty or failed to load. Aborting calculation to allow backfill.');
-                    return null; // Return null to trigger backfill
-                }
-            } catch (e) {
-                 console.error('[DumbCohortFlow] Failed to load price/mapping dependencies:', e);
-                 return null;
-            }
-            // --- END MODIFICATION ---
-        }
-
-        // --- 2. Calculate Asset Flow ---
-        const finalAssetFlow = {};
-        const todayStr = this.dates.today;
-        const yesterdayStr = this.dates.yesterday;
-
-        for (const instrumentId in this.asset_values) {
-            const ticker = this.mappings.instrumentToTicker[instrumentId] || `id_${instrumentId}`;
-            const avg_day1_value = this.asset_values[instrumentId].day1_value_sum / this.user_count;
-            const avg_day2_value = this.asset_values[instrumentId].day2_value_sum / this.user_count;
-            const priceChangePct = getDailyPriceChange(instrumentId, yesterdayStr, todayStr, this.priceMap);
-
-            if (priceChangePct === null) continue; // Skip if price data missing
-
-            const expected_day2_value = avg_day1_value * (1 + priceChangePct);
-            const net_crowd_flow_pct = avg_day2_value - expected_day2_value;
-
-            finalAssetFlow[ticker] = {
-                net_crowd_flow_pct: net_crowd_flow_pct,
-                avg_value_day1_pct: avg_day1_value,
-                avg_value_day2_pct: avg_day2_value
-            };
-        }
-        
-        // --- 3. Calculate Sector Rotation ---
-        const finalSectorRotation = {};
-        const allSectors = new Set([...Object.keys(this.todaySectorInvestment), ...Object.keys(this.yesterdaySectorInvestment)]);
-        for (const sector of allSectors) {
-            const todayAmount = this.todaySectorInvestment[sector] || 0;
-            const yesterdayAmount = this.yesterdaySectorInvestment[sector] || 0;
-            finalSectorRotation[sector] = todayAmount - yesterdayAmount;
-        }
-        
-        // --- START MODIFICATION ---
-        // If no asset flow was calculated (e.g., all price data missing), fail
-        if (Object.keys(finalAssetFlow).length === 0) {
-            console.warn('[DumbCohortFlow] No asset flow calculated (likely all price data missing). Returning null.');
-            return null;
-        }
-        // --- END MODIFICATION ---
-
-        // 4. Return combined result
-        return {
-            asset_flow: finalAssetFlow,
-            sector_rotation: finalSectorRotation,
-            user_sample_size: this.user_count
-        };
-    }
-
-    reset() { 
-        this.asset_values = {};
-        this.todaySectorInvestment = {};
-        this.yesterdaySectorInvestment = {};
-        this.dumbCohortIds = null;
-        this.user_count = 0;
-        this.priceMap = null;
-        this.mappings = null;
-        this.sectorMap = null;
-        this.dates = {};
-    }
-}
-
+/**
+ * @fileoverview Calculates "Net Crowd Flow" and "Sector Rotation"
+ * *only* for the "Dumb Cohort" (Bottom 20% of Investor Scores).
+ *
+ * --- META REFACTOR ---
+ * This calculation is now `type: "meta"` to consume in-memory dependencies.
+ * It runs ONCE per day, receives the in-memory cache, and must
+ * perform its own user data streaming.
+ */
+
+const { Firestore } = require('@google-cloud/firestore');
+const firestore = new Firestore();
+const { loadAllPriceData, getDailyPriceChange } = require('../../../utils/price_data_provider');
+const { loadInstrumentMappings, getInstrumentSectorMap } = require('../../../utils/sector_mapping_provider');
+const { loadDataByRefs } = require('../../../../bulltrackers-module/functions/computation-system/utils/data_loader'); // Adjust path as needed
+
+const COHORT_PERCENTILE = 0.2; // Bottom 20%
+const PROFILE_CALC_ID = 'user-investment-profile'; // The calc to read IS scores from
+
+class DumbCohortFlow {
+    constructor() {
+        // Meta-calc, no constructor state needed
+    }
+
+    /**
+     * Loads the Investor Scores, calculates the cohort threshold, and builds the Set of user IDs.
+     * --- MODIFIED: Reads from in-memory 'computedDependencies' ---
+     */
+    _loadCohort(logger, computedDependencies) {
+        logger.log('INFO', '[DumbCohortFlow] Loading Investor Scores from in-memory cache...');
+        
+        const profileData = computedDependencies[PROFILE_CALC_ID];
+        
+        if (!profileData || !profileData.daily_investor_scores || Object.keys(profileData.daily_investor_scores).length === 0) {
+            logger.log('WARN', `[DumbCohortFlow] Cannot find dependency in-memory: ${PROFILE_CALC_ID}. Cohort will not be built.`);
+            return null; // Return null to signal failure
+        }
+            
+        const scores = profileData.daily_investor_scores;
+        const allScores = Object.entries(scores).map(([userId, score]) => ({ userId, score }));
+        allScores.sort((a, b) => a.score - b.score);
+        
+        const thresholdIndex = Math.floor(allScores.length * COHORT_PERCENTILE);
+        const thresholdScore = allScores[thresholdIndex]?.score || 0; // Get 20th percentile score
+        
+        const dumbCohortIds = new Set(
+            allScores.filter(s => s.score <= thresholdScore).map(s => s.userId) // Get users *at or below*
+        );
+        
+        logger.log('INFO', `[DumbCohortFlow] Cohort built. ${dumbCohortIds.size} users at or below ${thresholdScore.toFixed(2)} (20th percentile).`);
+        return dumbCohortIds;
+    }
+
+    // --- Asset Flow Helpers (unchanged) ---
+    _initAsset(asset_values, instrumentId) {
+        if (!asset_values[instrumentId]) {
+            asset_values[instrumentId] = { day1_value_sum: 0, day2_value_sum: 0 };
+        }
+    }
+    _sumAssetValue(positions) {
+        const valueMap = {};
+        if (!positions || !Array.isArray(positions)) return valueMap;
+        for (const pos of positions) {
+            if (pos && pos.InstrumentID && pos.Value) {
+                valueMap[pos.InstrumentID] = (valueMap[pos.InstrumentID] || 0) + pos.Value;
+            }
+        }
+        return valueMap;
+    }
+    // --- Sector Rotation Helper (unchanged) ---
+    _accumulateSectorInvestment(portfolio, target, sectorMap) {
+        if (portfolio && portfolio.AggregatedPositions) {
+            for (const pos of portfolio.AggregatedPositions) {
+                const sector = sectorMap[pos.InstrumentID] || 'N/A';
+                target[sector] = (target[sector] || 0) + (pos.Invested || pos.Amount || 0);
+            }
+        }
+    }
+
+    /**
+     * PROCESS: META REFACTOR
+     * This now runs ONCE, loads all data, streams users, and returns one big result.
+     */
+    async process(dateStr, dependencies, config, computedDependencies) {
+        const { logger, db, rootData, calculationUtils } = dependencies;
+        const { portfolioRefs } = rootData;
+        logger.log('INFO', '[DumbCohortFlow] Starting meta-process...');
+        
+        // 1. Load Cohort from in-memory dependency
+        const dumbCohortIds = this._loadCohort(logger, computedDependencies);
+        if (!dumbCohortIds) {
+            return null; // Dependency failed
+        }
+
+        // 2. Load external dependencies (prices, sectors)
+        const [priceMap, mappings, sectorMap] = await Promise.all([
+            loadAllPriceData(),
+            loadInstrumentMappings(),
+            getInstrumentSectorMap()
+        ]);
+        if (!priceMap || !mappings || !sectorMap || Object.keys(priceMap).length === 0) {
+            logger.log('ERROR', '[DumbCohortFlow] Failed to load critical price/mapping/sector data. Aborting.');
+            return null; // Return null to trigger backfill
+        }
+
+        // 3. Load "yesterday's" portfolio data for comparison
+        const yesterdayDate = new Date(dateStr + 'T00:00:00Z');
+        yesterdayDate.setUTCDate(yesterdayDate.getUTCDate() - 1);
+        const yesterdayStr = yesterdayDate.toISOString().slice(0, 10);
+        const yesterdayRefs = await calculationUtils.getPortfolioPartRefs(config, dependencies, yesterdayStr);
+        const yesterdayPortfolios = await loadFullDayMap(config, dependencies, yesterdayRefs);
+        logger.log('INFO', `[DumbCohortFlow] Loaded ${yesterdayRefs.length} part refs for yesterday.`);
+
+        // 4. Stream "today's" portfolio data and process
+        
+        // --- Local state for this run ---
+        const asset_values = {};
+        const todaySectorInvestment = {};
+        const yesterdaySectorInvestment = {};
+        let user_count = 0;
+        // --- End Local state ---
+
+        const batchSize = config.partRefBatchSize || 10;
+        for (let i = 0; i < portfolioRefs.length; i += batchSize) {
+            const batchRefs = portfolioRefs.slice(i, i + batchSize);
+            const todayPortfoliosChunk = await loadDataByRefs(config, dependencies, batchRefs);
+
+            for (const uid in todayPortfoliosChunk) {
+                
+                // --- Filter user ---
+                if (!dumbCohortIds.has(uid)) {
+                    continue;
+                }
+                
+                const pToday = todayPortfoliosChunk[uid];
+                const pYesterday = yesterdayPortfolios[uid];
+                
+                if (!pToday || !pYesterday || !pToday.AggregatedPositions || !pYesterday.AggregatedPositions) {
+                    continue;
+                }
+                
+                // --- User is in cohort, run logic ---
+                
+                // 4a. RUN ASSET FLOW LOGIC
+                const yesterdayValues = this._sumAssetValue(pYesterday.AggregatedPositions);
+                const todayValues = this._sumAssetValue(pToday.AggregatedPositions);
+                const allInstrumentIds = new Set([...Object.keys(yesterdayValues), ...Object.keys(todayValues)]);
+                
+                for (const instrumentId of allInstrumentIds) {
+                    this._initAsset(asset_values, instrumentId);
+                    asset_values[instrumentId].day1_value_sum += (yesterdayValues[instrumentId] || 0);
+                    asset_values[instrumentId].day2_value_sum += (todayValues[instrumentId] || 0);
+                }
+
+                // 4b. RUN SECTOR ROTATION LOGIC
+                this._accumulateSectorInvestment(pToday, todaySectorInvestment, sectorMap);
+                this._accumulateSectorInvestment(pYesterday, yesterdaySectorInvestment, sectorMap);
+
+                user_count++;
+            }
+        }
+        
+        logger.log('INFO', `[DumbCohortFlow] Processed ${user_count} users in cohort.`);
+        
+        // --- 5. GETRESULT LOGIC IS NOW INSIDE PROCESS ---
+        
+        if (user_count === 0) {
+            logger.warn('[DumbCohortFlow] No users processed for dumb cohort. Returning null.');
+            return null;
+        }
+
+        // 5a. Calculate Asset Flow
+        const finalAssetFlow = {};
+        for (const instrumentId in asset_values) {
+            const ticker = mappings.instrumentToTicker[instrumentId] || `id_${instrumentId}`;
+            const avg_day1_value = asset_values[instrumentId].day1_value_sum / user_count;
+            const avg_day2_value = asset_values[instrumentId].day2_value_sum / user_count;
+            const priceChangePct = getDailyPriceChange(instrumentId, yesterdayStr, dateStr, priceMap);
+
+            if (priceChangePct === null) continue;
+
+            const expected_day2_value = avg_day1_value * (1 + priceChangePct);
+            const net_crowd_flow_pct = avg_day2_value - expected_day2_value;
+
+            finalAssetFlow[ticker] = {
+                net_crowd_flow_pct: net_crowd_flow_pct,
+                avg_value_day1_pct: avg_day1_value,
+                avg_value_day2_pct: avg_day2_value
+            };
+        }
+        
+        // 5b. Calculate Sector Rotation
+        const finalSectorRotation = {};
+        const allSectors = new Set([...Object.keys(todaySectorInvestment), ...Object.keys(yesterdaySectorInvestment)]);
+        for (const sector of allSectors) {
+            const todayAmount = todaySectorInvestment[sector] || 0;
+            const yesterdayAmount = yesterdaySectorInvestment[sector] || 0;
+            finalSectorRotation[sector] = todayAmount - yesterdayAmount;
+        }
+
+        if (Object.keys(finalAssetFlow).length === 0) {
+            logger.warn('[DumbCohortFlow] No asset flow calculated (likely all price data missing). Returning null.');
+            return null;
+        }
+
+        // 6. Return combined result
+        return {
+            asset_flow: finalAssetFlow,
+            sector_rotation: finalSectorRotation,
+            user_sample_size: user_count
+        };
+    }
+
+    async getResult() { return null; }
+    reset() { }
+}
+
 module.exports = DumbCohortFlow;

package/calculations/behavioural/historical/gain_response.js

@@ -1,58 +1,58 @@
-/**
- * Analyzes user behavior after a position experiences a >10% gain.
- */
-class GainResponse {
-    constructor() {
-        this.gain_events = {
-            held_position: 0,
-            closed_position: 0,
-            reduced_position: 0 // e.g., took partial profit
-        };
-    }
-
-    process(todayPortfolio, yesterdayPortfolio, userId) {
-        if (!yesterdayPortfolio || !todayPortfolio) {
-            return; // Need both days for comparison
-        }
-
-        const yPositions = yesterdayPortfolio.AggregatedPositions || yesterdayPortfolio.PublicPositions;
-        const tPositions = todayPortfolio.AggregatedPositions || todayPortfolio.PublicPositions;
-
-        if (!yPositions || !Array.isArray(yPositions) || !tPositions || !Array.isArray(tPositions)) {
-            return;
-        }
-
-        // Use PositionID if available (as in original file), fallback to InstrumentID
-        const todayPositions = new Map(tPositions.map(p => [p.PositionID || p.InstrumentID, p]));
-
-        for (const yPos of yPositions) {
-            // FIX: Use the NetProfit field, which is already a percentage.
-            // Your data sample (e.g., 23.5) shows the threshold should be 10.0.
-            const gainPercent = yPos.NetProfit || 0;
-            const yPosId = yPos.PositionID || yPos.InstrumentID;
-
-            // Check if this position was in a >10% gain yesterday
-            if (gainPercent > 10.0) {
-                const todayPos = todayPositions.get(yPosId);
-
-                if (!todayPos) {
-                    // Position was closed (took full profit)
-                    this.gain_events.closed_position++;
-                } else if (todayPos.Invested < yPos.Invested) {
-                    // FIX: Use 'Invested' (percentage) to check for reduction
-                    // User reduced the position (took partial profit)
-                    this.gain_events.reduced_position++;
-                } else {
-                    // Position was held (or added to)
-                    this.gain_events.held_position++;
-                }
-            }
-        }
-    }
-
-    getResult() {
-        return this.gain_events;
-    }
-}
-
+/**
+ * Analyzes user behavior after a position experiences a >10% gain.
+ */
+class GainResponse {
+    constructor() {
+        this.gain_events = {
+            held_position: 0,
+            closed_position: 0,
+            reduced_position: 0 // e.g., took partial profit
+        };
+    }
+
+    process(todayPortfolio, yesterdayPortfolio, userId) {
+        if (!yesterdayPortfolio || !todayPortfolio) {
+            return; // Need both days for comparison
+        }
+
+        const yPositions = yesterdayPortfolio.AggregatedPositions || yesterdayPortfolio.PublicPositions;
+        const tPositions = todayPortfolio.AggregatedPositions || todayPortfolio.PublicPositions;
+
+        if (!yPositions || !Array.isArray(yPositions) || !tPositions || !Array.isArray(tPositions)) {
+            return;
+        }
+
+        // Use PositionID if available (as in original file), fallback to InstrumentID
+        const todayPositions = new Map(tPositions.map(p => [p.PositionID || p.InstrumentID, p]));
+
+        for (const yPos of yPositions) {
+            // FIX: Use the NetProfit field, which is already a percentage.
+            // Your data sample (e.g., 23.5) shows the threshold should be 10.0.
+            const gainPercent = yPos.NetProfit || 0;
+            const yPosId = yPos.PositionID || yPos.InstrumentID;
+
+            // Check if this position was in a >10% gain yesterday
+            if (gainPercent > 10.0) {
+                const todayPos = todayPositions.get(yPosId);
+
+                if (!todayPos) {
+                    // Position was closed (took full profit)
+                    this.gain_events.closed_position++;
+                } else if (todayPos.Invested < yPos.Invested) {
+                    // FIX: Use 'Invested' (percentage) to check for reduction
+                    // User reduced the position (took partial profit)
+                    this.gain_events.reduced_position++;
+                } else {
+                    // Position was held (or added to)
+                    this.gain_events.held_position++;
+                }
+            }
+        }
+    }
+
+    getResult() {
+        return this.gain_events;
+    }
+}
+
 module.exports = GainResponse;