aiden-shared-calculations-unified 1.0.82 → 1.0.84

package/calculations/gem/skilled-cohort-flow.js

@@ -1,241 +1,252 @@
 /**
- * @fileoverview Calculation (Pass 2) for "Skilled Cohort" flow.
- *
- * This metric calculates the "Net Crowd Flow Percentage" for the
- * "Skilled Cohort" (top 20% by long-term skill score).
- *
- * This calculation *depends* on 'cohort-skill-definition'
- * to identify the cohort.
+ * @fileoverview GEM Product Line (Pass 2)
+ * --- FIX ---
+ * - Added defensive checks in '_loadDependencies' to prevent crash
+ * when dependencies fail to load (e.g., due to worker bug).
+ * - **FIXED:** Restored logic to calculate 'avg_position_change_pct'
+ * to resolve the hardcoded '0' value. This involves
+ * re-adding fields to '_initFlowData' and 'process'.
  */
-const { loadInstrumentMappings } = require('../../utils/sector_mapping_provider');
-
 
 class SkilledCohortFlow {
     constructor() {
-        this.assetData = new Map();
-        this.sectorData = new Map();
-        this.mappings = null;
-        this.skilledCohortUserIds = null;
-    }
-
-    /**
-     * Defines the output schema for this calculation.
-     * @returns {object} JSON Schema object
-     */
-    static getSchema() {
-        const flowSchema = {
-            "type": "object",
-            "properties": {
-                "net_flow_percentage": { "type": "number" },
-                "total_invested_today": { "type": "number" },
-                "total_invested_yesterday": { "type": "number" }
-            },
-            "required": ["net_flow_percentage", "total_invested_today", "total_invested_yesterday"]
-        };
-
-        return {
-            "type": "object",
-            "description": "Calculates net capital flow % (price-adjusted) for the 'Skilled Cohort' (top 20% by skill score), aggregated by asset and sector.",
-            "properties": {
-                "cohort_size": {
-                    "type": "number",
-                    "description": "The number of users identified as being in the Skilled Cohort."
-                },
-                "assets": {
-                    "type": "object",
-                    "description": "Price-adjusted net flow per asset.",
-                    "patternProperties": { "^.*$": flowSchema }, // Ticker
-                    "additionalProperties": flowSchema
-                },
-                "sectors": {
-                    "type": "object",
-                    "description": "Price-adjusted net flow per sector.",
-                    "patternProperties": { "^.*$": flowSchema }, // Sector
-                    "additionalProperties": flowSchema
-                }
-            },
-            "required": ["cohort_size", "assets", "sectors"]
-        };
+        this.assetFlows = new Map();
+        this.cohortMap = new Map();
+        this.tickerMap = null;
+        this.dependenciesLoaded = false;
+        this.priceChangeMap = null; 
     }
 
-    /**
-     * Statically defines all metadata for the manifest builder.
-     */
     static getMetadata() {
         return {
             type: 'standard',
             rootDataDependencies: ['portfolio'],
-            isHistorical: true, // Compares today vs yesterday portfolio
+            isHistorical: true, 
             userType: 'all',
             category: 'gem'
         };
     }
 
-    /**
-     * Statically declare dependencies.
-     */
     static getDependencies() {
-        return ['cohort-skill-definition']; // Pass 1
+        return [
+            'cohort-skill-definition',     // from gem (Pass 1)
+            'instrument-price-change-1d' // from core (Pass 1)
+        ];
+    }
+
+    static getSchema() {
+        const tickerSchema = {
+            "type": "object",
+            "properties": {
+                "net_flow_pct": { "type": "number" },
+                "net_flow_contribution": { "type": "number" },
+                "avg_position_change_pct": { "type": "number" },
+                "user_count": { "type": "number" }
+            },
+            "required": ["net_flow_pct", "net_flow_contribution", "avg_position_change_pct", "user_count"]
+        };
+
+        return {
+            "type": "object",
+            "description": "Calculates capital flow and conviction change for the 'Skilled' cohort.",
+            "patternProperties": { "^.*$": tickerSchema },
+            "additionalProperties": tickerSchema
+        };
     }
 
     _getPortfolioPositions(portfolio) {
-        return portfolio?.PublicPositions || portfolio?.AggregatedPositions;
+        // --- FIX: Support both Normal (Aggregated) and Speculator (Public) ---
+        return portfolio?.AggregatedPositions || portfolio?.PublicPositions;
     }
 
-    _initAsset(instrumentId) {
-        if (!this.assetData.has(instrumentId)) {
-            this.assetData.set(instrumentId, {
+    // --- THIS IS THE FIX (Part 1) ---
+    _initFlowData(instrumentId) {
+        if (!this.assetFlows.has(instrumentId)) {
+            this.assetFlows.set(instrumentId, {
                 total_invested_yesterday: 0,
                 total_invested_today: 0,
                 price_change_yesterday: 0,
+                // Restore fields needed for avg_position_change_pct
+                total_pos_size_yesterday: 0, 
+                total_pos_size_today: 0, 
+                user_count_yesterday: 0, 
+                user_count_today: 0
             });
         }
     }
-    
-    _initSector(sector) {
-        if (!this.sectorData.has(sector)) {
-            this.sectorData.set(sector, {
-                total_invested_yesterday: 0,
-                total_invested_today: 0,
-                price_change_yesterday: 0,
-            });
+    // --- END FIX (Part 1) ---
+
+    _loadDependencies(fetchedDependencies) {
+        if (this.dependenciesLoaded) return;
+
+        if (!fetchedDependencies) {
+            // This will stop the test and explain the root cause.
+            throw new Error(
+                `[skilled-cohort-flow] CRITICAL ERROR: fetchedDependencies object was UNDEFINED. 
+                This means a dependency test (like 'cohort-skill-definition' or 'instrument-price-change-1d') 
+                failed to run or produced an invalid result.`
+            );
         }
-    }
 
-    _getSkilledCohort(fetchedDependencies) {
-        if (this.skilledCohortUserIds) {
-            return this.skilledCohortUserIds;
-        }
-        
-        // FIX: Use normalized dependency name
         const cohortData = fetchedDependencies['cohort-skill-definition'];
-        if (!cohortData || !cohortData.skilled_user_ids) {
-            return new Set();
+        if (!cohortData) {
+             throw new Error(
+                `[skilled-cohort-flow] DEPENDENCY ERROR: 'cohort-skill-definition' was missing. 
+                Check logs for errors in that calculation.`
+             );
+        }
+
+        // This logic is correct for 'cohort-skill-definition' output
+        if (cohortData && cohortData.skilled_user_ids) {
+            (cohortData.skilled_user_ids || []).forEach(uid => this.cohortMap.set(String(uid), 'skilled'));
+        }
+
+        this.priceChangeMap = fetchedDependencies['instrument-price-change-1d'];
+        if (!this.priceChangeMap) {
+             throw new Error(
+                `[skilled-cohort-flow] DEPENDENCY ERROR: 'instrument-price-change-1d' was missing. 
+                This is likely due to the worker bug for 'price' dependencies.`
+             );
         }
 
-        this.skilledCohortUserIds = new Set(cohortData.skilled_user_ids);
-        return this.skilledCohortUserIds;
+        this.dependenciesLoaded = true;
     }
 
+    // --- THIS IS THE FIX (Part 2) ---
     process(todayPortfolio, yesterdayPortfolio, userId, context, todayInsights, yesterdayInsights, fetchedDependencies) {
-        const skilledCohort = this._getSkilledCohort(fetchedDependencies);
-        
-        if (!skilledCohort.has(userId)) {
-            return; // This user is not in the "skilled cohort", skip.
+        // Load dependencies *before* checking cohort
+        this._loadDependencies(fetchedDependencies);
+
+        if (!this.tickerMap) {
+            this.tickerMap = context.instrumentToTicker;
         }
 
-        if (!todayPortfolio || !yesterdayPortfolio) {
+        const cohortName = this.cohortMap.get(String(userId)); // Ensure string ID
+        if (cohortName !== 'skilled') {
+            return; // Not in skilled cohort
+        }
+
+        if (!todayPortfolio || !yesterdayPortfolio || !this.priceChangeMap || !this.tickerMap) {
             return;
         }
 
-        // Logic from here is identical to smart-cohort-flow.js
         const yPos = this._getPortfolioPositions(yesterdayPortfolio);
         const tPos = this._getPortfolioPositions(todayPortfolio);
-        const yPosMap = new Map(yPos?.map(p => [p.InstrumentID, p]) || []);
-        const tPosMap = new Map(tPos?.map(p => [p.InstrumentID, p]) || []);
+        if (!yPos || !tPos) return;
+
+        const yPosMap = new Map(yPos.map(p => [p.InstrumentID, p]));
+        const tPosMap = new Map(tPos.map(p => [p.InstrumentID, p]));
         const allInstrumentIds = new Set([...yPosMap.keys(), ...tPosMap.keys()]);
-        
-        if (!this.mappings) {
-            this.mappings = context.mappings;
-        }
 
         for (const instrumentId of allInstrumentIds) {
             if (!instrumentId) continue;
-            
-            this._initAsset(instrumentId);
-            const asset = this.assetData.get(instrumentId);
+
+            this._initFlowData(instrumentId);
+            const asset = this.assetFlows.get(instrumentId);
+
             const yP = yPosMap.get(instrumentId);
             const tP = tPosMap.get(instrumentId);
+
+            // Get $ flow
+            const yInvested = yP?.Invested || yP?.Amount || 0; 
+            const tInvested = tP?.Invested || tP?.Amount || 0;
             
-            // Use 'Invested' for % portfolio, not 'Amount'
-            const yInvested = yP?.Invested || 0; 
-            const tInvested = tP?.Invested || 0;
-            
-            const sector = this.mappings.instrumentToSector[instrumentId] || 'Other';
-            this._initSector(sector);
-            const sectorAsset = this.sectorData.get(sector);
+            // Get position size (same as $ flow for these schemas)
+            const ySize = yP?.Invested || yP?.Amount || 0;
+            const tSize = tP?.Invested || tP?.Amount || 0;
 
             if (yInvested > 0) {
-                // Use P&L % from portfolio 'NetProfit'
-                const yPriceChange = (yP?.NetProfit || 0) / (yP?.Invested || 1); 
-                
                 asset.total_invested_yesterday += yInvested;
-                asset.price_change_yesterday += yPriceChange * yInvested;
+
+                const ticker = this.tickerMap[instrumentId];
+                const yPriceChange_pct = (ticker && this.priceChangeMap[ticker]) 
+                                            ? this.priceChangeMap[ticker].price_change_1d_pct 
+                                            : 0;
+
+                const yPriceChange_decimal = (yPriceChange_pct || 0) / 100.0;
+                asset.price_change_yesterday += yPriceChange_decimal * yInvested;
                 
-                sectorAsset.total_invested_yesterday += yInvested;
-                sectorAsset.price_change_yesterday += yPriceChange * yInvested;
+                // Track yesterday's size and user count
+                asset.total_pos_size_yesterday += ySize;
+                asset.user_count_yesterday++;
             }
             if (tInvested > 0) {
                 asset.total_invested_today += tInvested;
-                sectorAsset.total_invested_today += tInvested;
-            }
-        }
-    }
-    
-    _calculateFlow(dataMap) {
-        const result = {};
-        for (const [key, data] of dataMap.entries()) {
-            const { total_invested_yesterday, total_invested_today, price_change_yesterday } = data;
-            
-            if (total_invested_yesterday > 0) {
-                const avg_price_change_pct = price_change_yesterday / total_invested_yesterday;
-                const price_contribution = total_invested_yesterday * avg_price_change_pct;
-                const flow_contribution = total_invested_today - (total_invested_yesterday + price_contribution);
-                const net_flow_percentage = (flow_contribution / total_invested_yesterday) * 100;
+                asset.user_count_today++; // Count users holding today
                 
-                result[key] = {
-                    net_flow_percentage: net_flow_percentage,
-                    total_invested_today: total_invested_today,
-                    total_invested_yesterday: total_invested_yesterday
-                };
+                // Track today's size
+                asset.total_pos_size_today += tSize;
             }
         }
-        return result;
     }
+    // --- END FIX (Part 2) ---
 
-    async getResult(fetchedDependencies) {
-        if (!this.mappings) {
-            this.mappings = await loadInstrumentMappings();
+    // --- THIS IS THE FIX (Part 3) ---
+    async getResult() {
+        if (!this.tickerMap) {
+            return {}; 
         }
-        
-        const skilledCohort = this._getSkilledCohort(fetchedDependencies);
-
-        // 1. Calculate Asset Flow
-        const assetResult = {};
-        for (const [instrumentId, data] of this.assetData.entries()) {
-            const ticker = this.mappings.instrumentToTicker[instrumentId] || `id_${instrumentId}`;
-            const { total_invested_yesterday, total_invested_today, price_change_yesterday } = data;
-            
+
+        const finalResult = {};
+
+        for (const [instrumentId, data] of this.assetFlows.entries()) {
+            const ticker = this.tickerMap[instrumentId];
+            if (!ticker) continue;
+
+            const { 
+                total_invested_yesterday, total_invested_today, price_change_yesterday,
+                // Destructure the restored fields
+                total_pos_size_yesterday, total_pos_size_today,
+                user_count_yesterday, user_count_today
+            } = data;
+
+            let net_flow_percentage = 0;
+            let flow_contribution = 0;
+
             if (total_invested_yesterday > 0) {
-                const avg_price_change_pct = price_change_yesterday / total_invested_yesterday;
-                const price_contribution = total_invested_yesterday * avg_price_change_pct;
-                const flow_contribution = total_invested_today - (total_invested_yesterday + price_contribution);
-                const net_flow_percentage = (flow_contribution / total_invested_yesterday) * 100;
-                
-                assetResult[ticker] = {
-                    net_flow_percentage: net_flow_percentage,
-                    total_invested_today: total_invested_today,
-                    total_invested_yesterday: total_invested_yesterday
+                const avg_price_change_decimal = (price_change_yesterday === 0) ? 0 : (price_change_yesterday / total_invested_yesterday);
+                const price_adjusted_yesterday_value = total_invested_yesterday * (1 + avg_price_change_decimal);
+
+                flow_contribution = total_invested_today - price_adjusted_yesterday_value;
+                net_flow_percentage = (flow_contribution / total_invested_yesterday) * 100;
+            } else if (total_invested_today > 0) {
+                flow_contribution = total_invested_today;
+                net_flow_percentage = Infinity;
+            }
+
+            // Restore the calculation logic for conviction
+            let avg_position_change_pct = 0; 
+            if (user_count_yesterday > 0 && user_count_today > 0) {
+                const avg_pos_y = total_pos_size_yesterday / user_count_yesterday;
+                const avg_pos_t = total_pos_size_today / user_count_today;
+                if (avg_pos_y > 0) {
+                    avg_position_change_pct = ((avg_pos_t - avg_pos_y) / avg_pos_y) * 100;
+                }
+            } else if (user_count_today > 0) {
+                // Pure inflow, conviction is max
+                avg_position_change_pct = Infinity;
+            }
+
+            if (isFinite(net_flow_percentage) && isFinite(flow_contribution)) {
+                finalResult[ticker] = {
+                    net_flow_pct: net_flow_percentage,
+                    net_flow_contribution: flow_contribution,
+                    avg_position_change_pct: avg_position_change_pct, // Now correctly calculated
+                    user_count: user_count_today
                 };
             }
         }
-        
-        // 2. Calculate Sector Flow
-        const sectorResult = this._calculateFlow(this.sectorData);
 
-        return {
-            cohort_size: skilledCohort.size,
-            assets: assetResult,
-            sectors: sectorResult
-        };
+        return finalResult;
     }
+    // --- END FIX (Part 3) ---
 
     reset() {
-        this.assetData.clear();
-        this.sectorData.clear();
-        this.mappings = null;
-        this.skilledCohortUserIds = null;
+        this.assetFlows.clear();
+        this.cohortMap.clear();
+        this.tickerMap = null;
+        this.dependenciesLoaded = false;
+        this.priceChangeMap = null;
     }
 }
-
 module.exports = SkilledCohortFlow;

package/calculations/gem/skilled-unskilled-divergence.js

@@ -1,153 +1,113 @@
 /**
- * @fileoverview Calculation (Pass 3) for skilled-unskilled divergence.
+ * @fileoverview GEM Product Line (Pass 3)
  *
- * This metric answers: "What divergence signals (e.g., capitulation,
- * euphoria) can be found by comparing the net asset and sector flow
- * of the 'Skilled Cohort' vs. the 'Unskilled Cohort'?"
- *
- * It *depends* on 'skilled-cohort-flow' and 'unskilled-cohort-flow'.
+ * This 'meta' calculation answers: "What is the net flow
+ * divergence between Skilled and Unskilled cohorts?"
  */
 class SkilledUnskilledDivergence {
+
+    // --- STANDARD 2: ADDED ---
     constructor() {
-        // No per-user processing
+        this.result = {};
     }
 
+    /** Statically defines metadata */
+    static getMetadata() {
+        return {
+            type: 'meta',
+            rootDataDependencies: [],
+            isHistorical: false,
+            userType: 'n/a',
+            category: 'gem'
+        };
+    }
+
+    /** Statically declare dependencies */
+    static getDependencies() {
+        return [
+            'skilled-cohort-flow',    // from gem (Pass 2)
+            'unskilled-cohort-flow'   // from gem (Pass 2)
+        ];
+    }
+    
     /**
      * Defines the output schema for this calculation.
-     * @returns {object} JSON Schema object
      */
     static getSchema() {
-        const signalSchema = {
+        const tickerSchema = {
             "type": "object",
             "properties": {
-                "status": {
-                    "type": "string",
-                    "enum": ["Capitulation", "Euphoria", "Confirmation (Buy)", "Confirmation (Sell)", "Divergence (Skilled Buy)", "Divergence (Skilled Sell)", "Neutral"]
-                },
                 "skilled_flow_pct": { "type": "number" },
-                "unskilled_flow_pct": { "type": "number" }
+                "unskilled_flow_pct": { "type": "number" },
+                "flow_divergence_score": { "type": "number" }
             },
-            "required": ["status", "skilled_flow_pct", "unskilled_flow_pct"]
+            "required": ["skilled_flow_pct", "unskilled_flow_pct", "flow_divergence_score"]
         };
-
+        
         return {
             "type": "object",
-            "description": "Generates divergence signals by comparing net flow of 'Skilled' vs. 'Unskilled' cohorts, by asset and sector.",
-            "properties": {
-                "assets": {
-                    "type": "object",
-                    "description": "Divergence signals per asset.",
-                    "patternProperties": { "^.*$": signalSchema }, // Ticker
-                    "additionalProperties": signalSchema
-                },
-                "sectors": {
-                    "type": "object",
-                    "description": "Divergence signals per sector.",
-                    "patternProperties": { "^.*$": signalSchema }, // Sector
-                    "additionalProperties": signalSchema
-                }
-            },
-            "required": ["assets", "sectors"]
-        };
-    }
-
-    /**
-     * Statically defines all metadata for the manifest builder.
-     */
-    static getMetadata() {
-        return {
-            type: 'meta',
-            rootDataDependencies: [],
-            isHistorical: false,
-            userType: 'n/a',
-            category: 'gem'
+            "description": "Tracks the net % flow divergence between skilled and unskilled cohorts.",
+            "patternProperties": { "^.*$": tickerSchema },
+            "additionalProperties": tickerSchema
         };
     }
 
     /**
-     * Statically declare dependencies.
+     * This is a 'meta' calculation. It runs once.
      */
-    static getDependencies() {
-        return [
-            'skilled-cohort-flow', // Pass 2
-            'unskilled-cohort-flow'   // Pass 2
-        ];
-    }
+    // --- STANDARD 1: UPDATED SIGNATURE (added rootData) ---
+    async process(dateStr, rootData, dependencies, config, fetchedDependencies) {
+        const { logger } = dependencies;
+        
+        const skilledFlow = fetchedDependencies['skilled-cohort-flow'];
+        const unskilledFlow = fetchedDependencies['unskilled-cohort-flow'];
 
-    process() {
-        // No-op
-    }
-    
-    _calculateDivergence(skilledFlow, unskilledFlow) {
-        const result = {};
         if (!skilledFlow || !unskilledFlow) {
-            return result;
+            logger.log('WARN', `[gem/skilled-unskilled-divergence] Missing dependencies for ${dateStr}.`);
+            // --- STANDARD 2: DO NOT RETURN ---
+            this.result = {};
+            return;
         }
-        
-        const allKeys = new Set([...Object.keys(skilledFlow), ...Object.keys(unskilledFlow)]);
-        const THRESHOLD = 1.0; // Min flow % to register as a signal
 
-        for (const key of allKeys) {
-            const sFlow = skilledFlow[key]?.net_flow_percentage || 0;
-            const dFlow = unskilledFlow[key]?.net_flow_percentage || 0;
+        const allTickers = new Set([
+            ...Object.keys(skilledFlow),
+            ...Object.keys(unskilledFlow)
+        ]);
 
-            let status = 'Neutral';
+        const result = {};
 
-            // Both buying
-            if (sFlow > THRESHOLD && dFlow > THRESHOLD) {
-                status = 'Confirmation (Buy)';
-            }
-            // Both selling
-            else if (sFlow < -THRESHOLD && dFlow < -THRESHOLD) {
-                status = 'Confirmation (Sell)';
-            }
-            // Skilled buying, Unskilled selling
-            else if (sFlow > THRESHOLD && dFlow < -THRESHOLD) {
-                status = 'Capitulation'; // Skilled buying the dip from unskilled
-            }
-            // Skilled selling, Unskilled buying
-            else if (sFlow < -THRESHOLD && dFlow > THRESHOLD) {
-                status = 'Euphoria'; // Skilled selling into unskilled fomo
-            }
-            // Skilled buying, Unskilled neutral
-            else if (sFlow > THRESHOLD && Math.abs(dFlow) < THRESHOLD) {
-                status = 'Divergence (Skilled Buy)';
-            }
-            // Skilled selling, Unskilled neutral
-            else if (sFlow < -THRESHOLD && Math.abs(dFlow) < THRESHOLD) {
-                status = 'Divergence (Skilled Sell)';
-            }
+        for (const ticker of allTickers) {
+            const skilledData = skilledFlow[ticker];
+            const unskilledData = unskilledFlow[ticker];
             
-            result[key] = {
-                status: status,
-                skilled_flow_pct: sFlow,
-                unskilled_flow_pct: dFlow
+            // Get net_flow_pct
+            const skilled_flow_pct = skilledData?.net_flow_pct || 0;
+            const unskilled_flow_pct = unskilledData?.net_flow_pct || 0;
+            
+            // Only include assets that are actively being traded
+            if (skilled_flow_pct === 0 && unskilled_flow_pct === 0) {
+                continue;
+            }
+
+            result[ticker] = {
+                skilled_flow_pct: skilled_flow_pct,
+                unskilled_flow_pct: unskilled_flow_pct,
+                flow_divergence_score: skilled_flow_pct - unskilled_flow_pct
             };
         }
-        return result;
+        
+        // --- STANDARD 2: SET STATE, DO NOT RETURN ---
+        this.result = result;
     }
 
-    /**
-     * This is a 'meta' calculation.
-     * @param {object} fetchedDependencies - Results from Pass 2.
-     */
-    getResult(fetchedDependencies) {
-        // FIX: Use normalized dependency names
-        const skilledFlowData = fetchedDependencies['skilled-cohort-flow'];
-        const unskilledFlowData = fetchedDependencies['unskilled-cohort-flow'];
-
-        const assetResult = this._calculateDivergence(skilledFlowData?.assets, unskilledFlowData?.assets);
-        const sectorResult = this._calculateDivergence(skilledFlowData?.sectors, unskilledFlowData?.sectors);
-
-        return {
-            assets: assetResult,
-            sectors: sectorResult
-        };
+    // --- STANDARD 2: ADDED ---
+    async getResult(fetchedDependencies) {
+        return this.result;
     }
 
+    // --- STANDARD 2: ADDED ---
     reset() {
-        // No state
+        this.result = {};
     }
 }
-
 module.exports = SkilledUnskilledDivergence;