aiden-shared-calculations-unified 1.0.36 → 1.0.38

package/calculations/behavioural/historical/dumb-cohort-flow.js

@@ -2,85 +2,59 @@
  * @fileoverview Calculates "Net Crowd Flow" and "Sector Rotation"
  * *only* for the "Dumb Cohort" (Bottom 20% of Investor Scores).
  *
- * This calc depends on 'user-investment-profile.js' being run first for the same day.
+ * --- META REFACTOR ---
+ * This calculation is now `type: "meta"` to consume in-memory dependencies.
+ * It runs ONCE per day, receives the in-memory cache, and must
+ * perform its own user data streaming.
  */
 
 const { Firestore } = require('@google-cloud/firestore');
 const firestore = new Firestore();
 const { loadAllPriceData, getDailyPriceChange } = require('../../../utils/price_data_provider');
 const { loadInstrumentMappings, getInstrumentSectorMap } = require('../../../utils/sector_mapping_provider');
+const { loadDataByRefs } = require('../../../../bulltrackers-module/functions/computation-system/utils/data_loader'); // Adjust path as needed
 
 const COHORT_PERCENTILE = 0.2; // Bottom 20%
 const PROFILE_CALC_ID = 'user-investment-profile'; // The calc to read IS scores from
 
 class DumbCohortFlow {
     constructor() {
-        // Asset Flow
-        this.asset_values = {}; // { instrumentId: { day1_value_sum: 0, day2_value_sum: 0 } }
-        // Sector Rotation
-        this.todaySectorInvestment = {};
-        this.yesterdaySectorInvestment = {};
-        
-        // --- START MODIFICATION ---
-        this.dumbCohortIds = null; // Set to null. Will be a Set on success.
-        // --- END MODIFICATION ---
-        
-        this.user_count = 0; // Number of *cohort* users
-        this.priceMap = null;
-        this.mappings = null;
-        this.sectorMap = null;
-        this.dates = {};
+        // Meta-calc, no constructor state needed
     }
 
     /**
      * Loads the Investor Scores, calculates the cohort threshold, and builds the Set of user IDs.
+     * --- MODIFIED: Reads from in-memory 'computedDependencies' ---
      */
-    async _loadCohort(context, dependencies) {
-        const { db, logger } = dependencies;
-        logger.log('INFO', '[DumbCohortFlow] Loading Investor Scores to build cohort...');
+    _loadCohort(logger, computedDependencies) {
+        logger.log('INFO', '[DumbCohortFlow] Loading Investor Scores from in-memory cache...');
         
-        try {
-            const scoreMapRef = db.collection(context.config.resultsCollection).doc(context.todayDateStr)
-                .collection(context.config.resultsSubcollection).doc('behavioural')
-                .collection(context.config.computationsSubcollection).doc(PROFILE_CALC_ID);
-                
-            const doc = await scoreMapRef.get();
-            
-            // --- START MODIFICATION ---
-            // Check for doc, data, and that the scores map isn't empty
-            if (!doc.exists || !doc.data().daily_investor_scores || Object.keys(doc.data().daily_investor_scores).length === 0) {
-                logger.log('WARN', '[DumbCohortFlow] Cannot find dependency: daily_investor_scores. Cohort will not be built. Returning null on getResult.');
-                // Keep this.dumbCohortIds = null
-                return; // Abort
-            }
-            // --- END MODIFICATION ---
-            
-            const scores = doc.data().daily_investor_scores;
-            const allScores = Object.entries(scores).map(([userId, score]) => ({ userId, score }));
-            allScores.sort((a, b) => a.score - b.score);
-            
-            const thresholdIndex = Math.floor(allScores.length * COHORT_PERCENTILE);
-            const thresholdScore = allScores[thresholdIndex]?.score || 0; // Get 20th percentile score
-            
-            // --- START MODIFICATION ---
-            // Successfully loaded, now create the Set
-            this.dumbCohortIds = new Set(
-                allScores.filter(s => s.score <= thresholdScore).map(s => s.userId) // Get users *at or below*
-            );
-            // --- END MODIFICATION ---
-            
-            logger.log('INFO', `[DumbCohortFlow] Cohort built. ${this.dumbCohortIds.size} users at or below ${thresholdScore.toFixed(2)} (20th percentile).`);
-            
-        } catch (e) {
-            logger.log('ERROR', '[DumbCohortFlow] Failed to load cohort.', { error: e.message });
-            // Keep this.dumbCohortIds = null on error
+        const profileData = computedDependencies[PROFILE_CALC_ID];
+        
+        if (!profileData || !profileData.daily_investor_scores || Object.keys(profileData.daily_investor_scores).length === 0) {
+            logger.log('WARN', `[DumbCohortFlow] Cannot find dependency in-memory: ${PROFILE_CALC_ID}. Cohort will not be built.`);
+            return null; // Return null to signal failure
         }
+            
+        const scores = profileData.daily_investor_scores;
+        const allScores = Object.entries(scores).map(([userId, score]) => ({ userId, score }));
+        allScores.sort((a, b) => a.score - b.score);
+        
+        const thresholdIndex = Math.floor(allScores.length * COHORT_PERCENTILE);
+        const thresholdScore = allScores[thresholdIndex]?.score || 0; // Get 20th percentile score
+        
+        const dumbCohortIds = new Set(
+            allScores.filter(s => s.score <= thresholdScore).map(s => s.userId) // Get users *at or below*
+        );
+        
+        logger.log('INFO', `[DumbCohortFlow] Cohort built. ${dumbCohortIds.size} users at or below ${thresholdScore.toFixed(2)} (20th percentile).`);
+        return dumbCohortIds;
     }
 
-    // --- Asset Flow Helpers ---
-    _initAsset(instrumentId) {
-        if (!this.asset_values[instrumentId]) {
-            this.asset_values[instrumentId] = { day1_value_sum: 0, day2_value_sum: 0 };
+    // --- Asset Flow Helpers (unchanged) ---
+    _initAsset(asset_values, instrumentId) {
+        if (!asset_values[instrumentId]) {
+            asset_values[instrumentId] = { day1_value_sum: 0, day2_value_sum: 0 };
         }
     }
     _sumAssetValue(positions) {
@@ -93,111 +67,117 @@ class DumbCohortFlow {
         }
         return valueMap;
     }
-    // --- Sector Rotation Helper ---
-    _accumulateSectorInvestment(portfolio, target) {
+    // --- Sector Rotation Helper (unchanged) ---
+    _accumulateSectorInvestment(portfolio, target, sectorMap) {
         if (portfolio && portfolio.AggregatedPositions) {
             for (const pos of portfolio.AggregatedPositions) {
-                const sector = this.sectorMap[pos.InstrumentID] || 'N/A';
+                const sector = sectorMap[pos.InstrumentID] || 'N/A';
                 target[sector] = (target[sector] || 0) + (pos.Invested || pos.Amount || 0);
             }
         }
     }
 
     /**
-     * PROCESS: Runs daily for each user.
+     * PROCESS: META REFACTOR
+     * This now runs ONCE, loads all data, streams users, and returns one big result.
      */
-    async process(todayPortfolio, yesterdayPortfolio, userId, context) {
-        // 1. Load cohort on first run
-        if (!this.dumbCohortIds) {
-            await this._loadCohort(context, context.dependencies);
-            this.dates.today = context.todayDateStr;
-            this.dates.yesterday = context.yesterdayDateStr;
-        }
-
-        // 2. Filter user
-        // --- START MODIFICATION ---
-        // If cohort failed to load, this.dumbCohortIds will be null, and this check will fail correctly.
-        if (!this.dumbCohortIds || !this.dumbCohortIds.has(userId) || !todayPortfolio || !yesterdayPortfolio || !todayPortfolio.AggregatedPositions || !yesterdayPortfolio.AggregatedPositions) {
-            return;
-        }
-        // --- END MODIFICATION ---
+    async process(dateStr, dependencies, config, computedDependencies) {
+        const { logger, db, rootData, calculationUtils } = dependencies;
+        const { portfolioRefs } = rootData;
+        logger.log('INFO', '[DumbCohortFlow] Starting meta-process...');
         
-        // 3. User is in the cohort, load maps if needed
-        if (!this.sectorMap) {
-            this.sectorMap = await getInstrumentSectorMap();
+        // 1. Load Cohort from in-memory dependency
+        const dumbCohortIds = this._loadCohort(logger, computedDependencies);
+        if (!dumbCohortIds) {
+            return null; // Dependency failed
         }
 
-        // --- 4. RUN ASSET FLOW LOGIC ---
-        const yesterdayValues = this._sumAssetValue(yesterdayPortfolio.AggregatedPositions);
-        const todayValues = this._sumAssetValue(todayPortfolio.AggregatedPositions);
-        const allInstrumentIds = new Set([...Object.keys(yesterdayValues), ...Object.keys(todayValues)]);
-        
-        for (const instrumentId of allInstrumentIds) {
-            this._initAsset(instrumentId);
-            this.asset_values[instrumentId].day1_value_sum += (yesterdayValues[instrumentId] || 0);
-            this.asset_values[instrumentId].day2_value_sum += (todayValues[instrumentId] || 0);
+        // 2. Load external dependencies (prices, sectors)
+        const [priceMap, mappings, sectorMap] = await Promise.all([
+            loadAllPriceData(),
+            loadInstrumentMappings(),
+            getInstrumentSectorMap()
+        ]);
+        if (!priceMap || !mappings || !sectorMap || Object.keys(priceMap).length === 0) {
+            logger.log('ERROR', '[DumbCohortFlow] Failed to load critical price/mapping/sector data. Aborting.');
+            return null; // Return null to trigger backfill
         }
 
-        // --- 5. RUN SECTOR ROTATION LOGIC ---
-        this._accumulateSectorInvestment(todayPortfolio, this.todaySectorInvestment);
-        this._accumulateSectorInvestment(yesterdayPortfolio, this.yesterdaySectorInvestment);
+        // 3. Load "yesterday's" portfolio data for comparison
+        const yesterdayDate = new Date(dateStr + 'T00:00:00Z');
+        yesterdayDate.setUTCDate(yesterdayDate.getUTCDate() - 1);
+        const yesterdayStr = yesterdayDate.toISOString().slice(0, 10);
+        const yesterdayRefs = await calculationUtils.getPortfolioPartRefs(config, dependencies, yesterdayStr);
+        const yesterdayPortfolios = await loadFullDayMap(config, dependencies, yesterdayRefs);
+        logger.log('INFO', `[DumbCohortFlow] Loaded ${yesterdayRefs.length} part refs for yesterday.`);
 
-        this.user_count++;
-    }
+        // 4. Stream "today's" portfolio data and process
+        
+        // --- Local state for this run ---
+        const asset_values = {};
+        const todaySectorInvestment = {};
+        const yesterdaySectorInvestment = {};
+        let user_count = 0;
+        // --- End Local state ---
+
+        const batchSize = config.partRefBatchSize || 10;
+        for (let i = 0; i < portfolioRefs.length; i += batchSize) {
+            const batchRefs = portfolioRefs.slice(i, i + batchSize);
+            const todayPortfoliosChunk = await loadDataByRefs(config, dependencies, batchRefs);
+
+            for (const uid in todayPortfoliosChunk) {
+                
+                // --- Filter user ---
+                if (!dumbCohortIds.has(uid)) {
+                    continue;
+                }
+                
+                const pToday = todayPortfoliosChunk[uid];
+                const pYesterday = yesterdayPortfolios[uid];
+                
+                if (!pToday || !pYesterday || !pToday.AggregatedPositions || !pYesterday.AggregatedPositions) {
+                    continue;
+                }
+                
+                // --- User is in cohort, run logic ---
+                
+                // 4a. RUN ASSET FLOW LOGIC
+                const yesterdayValues = this._sumAssetValue(pYesterday.AggregatedPositions);
+                const todayValues = this._sumAssetValue(pToday.AggregatedPositions);
+                const allInstrumentIds = new Set([...Object.keys(yesterdayValues), ...Object.keys(todayValues)]);
+                
+                for (const instrumentId of allInstrumentIds) {
+                    this._initAsset(asset_values, instrumentId);
+                    asset_values[instrumentId].day1_value_sum += (yesterdayValues[instrumentId] || 0);
+                    asset_values[instrumentId].day2_value_sum += (todayValues[instrumentId] || 0);
+                }
 
-    /**
-     * GETRESULT: Aggregates and returns the flow data for the cohort.
-     */
-    async getResult() {
-        // --- START MODIFICATION ---
-        // If cohort IDs were never loaded due to dependency failure, return null.
-        if (this.dumbCohortIds === null) {
-            console.warn('[DumbCohortFlow] Skipping getResult because dependency (user-investment-profile) failed to load.');
-            return null;
+                // 4b. RUN SECTOR ROTATION LOGIC
+                this._accumulateSectorInvestment(pToday, todaySectorInvestment, sectorMap);
+                this._accumulateSectorInvestment(pYesterday, yesterdaySectorInvestment, sectorMap);
+
+                user_count++;
+            }
         }
         
-        // If cohort loaded but no users were processed, also return null (or an empty object, but null is safer for backfill)
-        if (this.user_count === 0 || !this.dates.today) {
-            console.warn('[DumbCohortFlow] No users processed for dumb cohort. Returning null.');
+        logger.log('INFO', `[DumbCohortFlow] Processed ${user_count} users in cohort.`);
+        
+        // --- 5. GETRESULT LOGIC IS NOW INSIDE PROCESS ---
+        
+        if (user_count === 0) {
+            logger.warn('[DumbCohortFlow] No users processed for dumb cohort. Returning null.');
             return null;
         }
-        // --- END MODIFICATION ---
-
-        // 1. Load dependencies
-        if (!this.priceMap || !this.mappings) {
-            // --- START MODIFICATION ---
-            // Add error handling for this load, and check for empty priceMap
-            try {
-                const [priceData, mappingData] = await Promise.all([
-                    loadAllPriceData(),
-                    loadInstrumentMappings()
-                ]);
-                this.priceMap = priceData;
-                this.mappings = mappingData;
-
-                if (!this.priceMap || Object.keys(this.priceMap).length === 0) {
-                     console.error('[DumbCohortFlow] CRITICAL: Price map is empty or failed to load. Aborting calculation to allow backfill.');
-                    return null; // Return null to trigger backfill
-                }
-            } catch (e) {
-                 console.error('[DumbCohortFlow] Failed to load price/mapping dependencies:', e);
-                 return null;
-            }
-            // --- END MODIFICATION ---
-        }
 
-        // --- 2. Calculate Asset Flow ---
+        // 5a. Calculate Asset Flow
         const finalAssetFlow = {};
-        const todayStr = this.dates.today;
-        const yesterdayStr = this.dates.yesterday;
-
-        for (const instrumentId in this.asset_values) {
-            const ticker = this.mappings.instrumentToTicker[instrumentId] || `id_${instrumentId}`;
-            const avg_day1_value = this.asset_values[instrumentId].day1_value_sum / this.user_count;
-            const avg_day2_value = this.asset_values[instrumentId].day2_value_sum / this.user_count;
-            const priceChangePct = getDailyPriceChange(instrumentId, yesterdayStr, todayStr, this.priceMap);
+        for (const instrumentId in asset_values) {
+            const ticker = mappings.instrumentToTicker[instrumentId] || `id_${instrumentId}`;
+            const avg_day1_value = asset_values[instrumentId].day1_value_sum / user_count;
+            const avg_day2_value = asset_values[instrumentId].day2_value_sum / user_count;
+            const priceChangePct = getDailyPriceChange(instrumentId, yesterdayStr, dateStr, priceMap);
 
-            if (priceChangePct === null) continue; // Skip if price data missing
+            if (priceChangePct === null) continue;
 
             const expected_day2_value = avg_day1_value * (1 + priceChangePct);
             const net_crowd_flow_pct = avg_day2_value - expected_day2_value;
@@ -209,42 +189,30 @@ class DumbCohortFlow {
             };
         }
         
-        // --- 3. Calculate Sector Rotation ---
+        // 5b. Calculate Sector Rotation
         const finalSectorRotation = {};
-        const allSectors = new Set([...Object.keys(this.todaySectorInvestment), ...Object.keys(this.yesterdaySectorInvestment)]);
+        const allSectors = new Set([...Object.keys(todaySectorInvestment), ...Object.keys(yesterdaySectorInvestment)]);
         for (const sector of allSectors) {
-            const todayAmount = this.todaySectorInvestment[sector] || 0;
-            const yesterdayAmount = this.yesterdaySectorInvestment[sector] || 0;
+            const todayAmount = todaySectorInvestment[sector] || 0;
+            const yesterdayAmount = yesterdaySectorInvestment[sector] || 0;
             finalSectorRotation[sector] = todayAmount - yesterdayAmount;
         }
-        
-        // --- START MODIFICATION ---
-        // If no asset flow was calculated (e.g., all price data missing), fail
+
         if (Object.keys(finalAssetFlow).length === 0) {
-            console.warn('[DumbCohortFlow] No asset flow calculated (likely all price data missing). Returning null.');
+            logger.warn('[DumbCohortFlow] No asset flow calculated (likely all price data missing). Returning null.');
             return null;
         }
-        // --- END MODIFICATION ---
 
-        // 4. Return combined result
+        // 6. Return combined result
         return {
             asset_flow: finalAssetFlow,
             sector_rotation: finalSectorRotation,
-            user_sample_size: this.user_count
+            user_sample_size: user_count
         };
     }
 
-    reset() { 
-        this.asset_values = {};
-        this.todaySectorInvestment = {};
-        this.yesterdaySectorInvestment = {};
-        this.dumbCohortIds = null;
-        this.user_count = 0;
-        this.priceMap = null;
-        this.mappings = null;
-        this.sectorMap = null;
-        this.dates = {};
-    }
+    async getResult() { return null; }
+    reset() { }
 }
 
 module.exports = DumbCohortFlow;