bulltrackers-module 1.0.766 → 1.0.769

package/functions/computation-system-v2/computations/BehavioralAnomaly.js

@@ -1,5 +1,455 @@
+/**
+ * @fileoverview Behavioral Anomaly Detection V4.1 - Predictive Risk Intelligence
+ * * ARCHITECTURE:
+ * - Hybrid Math: Uses 6-Dim Core for Stability + 18-Dim Full Spectrum for Detail.
+ * - Cost Control: Strict 30-day lookback enforced (compatible with DAG constraints).
+ * - Logic Fix: "Primary Driver" uses Statistical Z-Scores, not raw values.
+ * - Robustness: Dynamic Z-Scoring removes dependency on hardcoded divisors.
+ */
+
 const { Computation } = require('../framework');
 
+// =============================================================================
+// ADVANCED MATH KERNEL
+// =============================================================================
+const AdvancedMath = {
+    entropy: (distribution) => {
+        const total = distribution.reduce((sum, val) => sum + val, 0);
+        if (total === 0) return 0;
+        let entropy = 0;
+        for (const val of distribution) {
+            if (val > 0) {
+                const p = val / total;
+                entropy -= p * Math.log2(p);
+            }
+        }
+        return entropy;
+    },
+
+    hhi: (shares) => {
+        const total = shares.reduce((sum, val) => sum + val, 0);
+        if (total === 0) return 0;
+        let hhi = 0;
+        for (const share of shares) {
+            const marketShare = share / total;
+            hhi += marketShare * marketShare;
+        }
+        return hhi * 10000;
+    },
+
+    velocity: (current, previous, timeDelta = 1) => {
+        if (timeDelta === 0) return 0;
+        return (current - previous) / timeDelta;
+    },
+
+    acceleration: (current, previous, previousPrevious, timeDelta = 1) => {
+        const v1 = AdvancedMath.velocity(current, previous, timeDelta);
+        const v0 = AdvancedMath.velocity(previous, previousPrevious, timeDelta);
+        return AdvancedMath.velocity(v1, v0, timeDelta);
+    },
+
+    percentileRank: (value, array) => {
+        if (!array.length) return 0.5;
+        let rank = 0;
+        for (const val of array) {
+            if (val < value) rank++;
+        }
+        return rank / array.length;
+    },
+
+    detectRegime: (recentVectors) => {
+        if (recentVectors.length < 3) return 'unknown';
+        // Calculate aggregate volatility across the CORE features (indices 0-5)
+        const coreIndices = [0, 1, 2, 3, 4, 5];
+        const volatilities = [];
+        
+        for (const f of coreIndices) {
+            const values = recentVectors.map(v => v[f]);
+            const mean = values.reduce((sum, v) => sum + v, 0) / values.length;
+            const variance = values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / values.length;
+            volatilities.push(Math.sqrt(variance));
+        }
+        const avgVolatility = volatilities.reduce((sum, v) => sum + v, 0) / volatilities.length;
+        
+        if (avgVolatility < 0.3) return 'calm';
+        if (avgVolatility < 0.8) return 'active';
+        return 'stressed';
+    }
+};
+
+// =============================================================================
+// FEATURE ENGINEERING
+// =============================================================================
+const FeatureExtractor = {
+    extract: (day, prevDay, prevPrevDay, historyBlob, rankings, rules, maps) => {
+        const { portfolio } = day;
+        const positions = rules.portfolio.extractPositions(portfolio) || [];
+        const rankingsData = day.rankings || {};
+        const { sectorMap, tickerMap } = maps;
+
+        // Resolve Sector
+        const getSector = (instId) => {
+            const ticker = tickerMap.get(String(instId));
+            if (!ticker) return 'Unknown';
+            return sectorMap.get(ticker) || 'Unknown';
+        };
+        
+        // 1. Sector Concentration (HHI)
+        const sectorExposure = {};
+        let totalInvestedPct = 0;
+        positions.forEach(p => {
+            const sector = getSector(p.InstrumentID);
+            const val = p.Invested || 0;
+            sectorExposure[sector] = (sectorExposure[sector] || 0) + val;
+            totalInvestedPct += val;
+        });
+        const sectorHHI = AdvancedMath.hhi(Object.values(sectorExposure));
+        
+        // 2. Martingale/Distress Score
+        let martingaleScore = 0;
+        let martingaleCount = 0;
+        positions.forEach(p => {
+            const pnl = p.NetProfit || 0;
+            const invested = p.Invested || 0;
+            if (pnl < -4 && invested > 4) {
+                martingaleScore += Math.abs(pnl) * invested;
+                martingaleCount++;
+            }
+        });
+        
+        // 3. Leverage Profile (Lazy Loaded)
+        const { avgLeverage, highLevCount } = FeatureExtractor._extractLeverageProfile(day.date, historyBlob);
+        
+        // 4. Risk Score
+        const riskScore = rankingsData.RiskScore || 1;
+        
+        // 5. Complexity
+        const complexity = positions.length;
+        
+        // 6. Exposure
+        const exposure = totalInvestedPct;
+        
+        // --- EXTENDED FEATURES (Full Spectrum) ---
+        
+        // 7. Entropy
+        const positionSizes = positions.map(p => p.Invested || 0);
+        const portfolioEntropy = AdvancedMath.entropy(positionSizes);
+        
+        // 8. Drawdown
+        const drawdownSeverity = Math.abs(rankingsData.PeakToValley || 0);
+        
+        // 9. Win Rate Deviation
+        const winRateDeviation = Math.abs((rankingsData.WinRatio || 50) - 50);
+        
+        // 10. Skewness
+        const sortedSizes = [...positionSizes].sort((a, b) => b - a);
+        const top3Share = sortedSizes.slice(0, 3).reduce((sum, v) => sum + v, 0);
+        const positionSkewness = totalInvestedPct > 0 ? top3Share / totalInvestedPct : 0;
+        
+        // 11. Stress Ratio
+        const losingCount = positions.filter(p => (p.NetProfit || 0) < 0).length;
+        const stressRatio = positions.length > 0 ? losingCount / positions.length : 0;
+        
+        // 12. Credit Imbalance
+        const realizedCredit = portfolio?.CreditByRealizedEquity || 0;
+        const unrealizedCredit = portfolio?.CreditByUnrealizedEquity || 0;
+        const creditImbalance = Math.abs(realizedCredit - unrealizedCredit);
+        
+        // 13. High Lev Freq
+        const highLevFrequency = highLevCount / Math.max(1, positions.length);
+        
+        // 14. Copier Momentum
+        const copiers = rankingsData.Copiers || 0;
+        const baselineCopiers = rankingsData.BaseLineCopiers || copiers;
+        const copierMomentum = baselineCopiers > 0 ? (copiers - baselineCopiers) / baselineCopiers : 0;
+        
+        // 15. AUM Tier
+        const aumTier = (rankingsData.AUMTier || 0);
+        
+        // 16. Exposure Velocity
+        let exposureVelocity = 0;
+        if (prevDay && prevDay.portfolio) {
+            const prevPositions = rules.portfolio.extractPositions(prevDay.portfolio) || [];
+            const prevExposure = prevPositions.reduce((sum, p) => sum + (p.Invested || 0), 0);
+            exposureVelocity = exposure - prevExposure;
+        }
+        
+        // 17. Risk Acceleration
+        let riskAcceleration = 0;
+        if (prevDay && prevDay.rankings && prevPrevDay && prevPrevDay.rankings) {
+            const r0 = prevPrevDay.rankings.RiskScore || 0;
+            const r1 = prevDay.rankings.RiskScore || 0;
+            const r2 = riskScore;
+            riskAcceleration = AdvancedMath.acceleration(r2, r1, r0, 1);
+        }
+        
+        // 18. Behavioral Momentum
+        const behavioralMomentum = Math.abs(exposureVelocity) + Math.abs(riskAcceleration);
+        
+        return {
+            vector: [
+                // CORE (0-5)
+                sectorHHI,              
+                Math.log(martingaleScore + 1),  
+                avgLeverage,                
+                riskScore,                 
+                complexity,                
+                exposure,  
+                
+                // EXTENDED (6-17)
+                portfolioEntropy,           
+                drawdownSeverity,         
+                winRateDeviation,          
+                positionSkewness,               
+                stressRatio,                    
+                creditImbalance,          
+                highLevFrequency,               
+                copierMomentum,                 
+                aumTier,                        
+                exposureVelocity,          
+                riskAcceleration,           
+                behavioralMomentum         
+            ],
+            metadata: {
+                martingaleCount,
+                losingCount,
+                totalPositions: positions.length,
+                exposure,
+                regime: 'unknown'
+            }
+        };
+    },
+
+    _extractLeverageProfile: (dateStr, historyBlob) => {
+        const targetDate = new Date(dateStr);
+        const lookbackWindow = new Date(targetDate);
+        lookbackWindow.setDate(lookbackWindow.getDate() - 90);
+
+        let sumLev = 0;
+        let tradeCount = 0;
+        let highLevCount = 0;
+
+        for (const t of historyBlob) {
+            const closeDate = new Date(t.CloseDateTime);
+            if (closeDate <= targetDate && closeDate >= lookbackWindow) {
+                const lev = t.Leverage || 1;
+                sumLev += lev;
+                tradeCount++;
+                if (lev > 2) highLevCount++;
+            }
+        }
+
+        return {
+            avgLeverage: tradeCount > 0 ? sumLev / tradeCount : 1,
+            highLevCount
+        };
+    }
+};
+
+// =============================================================================
+// ANOMALY DETECTION ENGINE
+// =============================================================================
+const AnomalyEngine = {
+    score: (todayFeatures, historicalFeatures, regime) => {
+        // 1. DYNAMIC STANDARDIZATION (Z-Scoring)
+        // We capture zVector here because it represents the "True Statistical Deviation"
+        const { zVector, histZVectors } = AnomalyEngine._standardize(todayFeatures.vector, historicalFeatures);
+
+        // 2. HYBRID MAHALANOBIS (Core Features Only)
+        // Slicing indices 0-5 (Core) for structural stability on 30-day lookback
+        const coreZVector = zVector.slice(0, 6);
+        const coreHistZVectors = histZVectors.map(v => v.slice(0, 6));
+        const mahalanobisScore = AnomalyEngine._mahalanobisDistance(coreZVector, coreHistZVectors);
+
+        // 3. FULL SPECTRUM PERCENTILE (All 18 Features)
+        const percentileScore = AnomalyEngine._percentileScore(todayFeatures.vector, historicalFeatures);
+
+        // 4. REGIME CONDITIONAL
+        const regimeScore = AnomalyEngine._regimeScore(todayFeatures, historicalFeatures, regime);
+
+        // 5. VELOCITY
+        const velocityScore = (Math.abs(zVector[15]) + Math.abs(zVector[16]) + Math.abs(zVector[17])) / 3;
+
+        const ensembleScore = 
+            0.40 * mahalanobisScore +
+            0.30 * percentileScore +
+            0.15 * regimeScore +
+            0.15 * velocityScore;
+
+        return {
+            overall: ensembleScore,
+            components: {
+                mahalanobis: mahalanobisScore,
+                percentile: percentileScore,
+                regime: regimeScore,
+                velocity: velocityScore
+            },
+            // EXPORT Z-SCORES FOR INTERPRETER
+            zScores: zVector 
+        };
+    },
+
+    _standardize: (todayVec, history) => {
+        const dim = todayVec.length;
+        const n = history.length;
+        const means = new Array(dim).fill(0);
+        const stdDevs = new Array(dim).fill(0);
+
+        const vectors = history.map(h => h.vector);
+        for (const v of vectors) for (let i = 0; i < dim; i++) means[i] += v[i];
+        for (let i = 0; i < dim; i++) means[i] /= n;
+
+        for (const v of vectors) for (let i = 0; i < dim; i++) stdDevs[i] += Math.pow(v[i] - means[i], 2);
+        for (let i = 0; i < dim; i++) {
+            stdDevs[i] = Math.sqrt(stdDevs[i] / n);
+            if (stdDevs[i] === 0) stdDevs[i] = 1; 
+        }
+
+        const zVector = todayVec.map((v, i) => (v - means[i]) / stdDevs[i]);
+        const histZVectors = vectors.map(v => v.map((val, i) => (val - means[i]) / stdDevs[i]));
+
+        return { zVector, histZVectors };
+    },
+
+    _mahalanobisDistance: (todayZVec, histZVectors) => {
+        const means = new Array(todayZVec.length).fill(0);
+        const cov = AnomalyEngine._covariance(histZVectors, means);
+        const invCov = AnomalyEngine._invert(cov);
+        
+        if (!invCov) return 0;
+        
+        let sum = 0;
+        for (let i = 0; i < todayZVec.length; i++) {
+            for (let j = 0; j < todayZVec.length; j++) {
+                sum += todayZVec[i] * invCov[i][j] * todayZVec[j];
+            }
+        }
+        return Math.sqrt(Math.max(0, sum));
+    },
+
+    _percentileScore: (todayVec, historicalFeatures) => {
+        let totalDeviation = 0;
+        const featureCount = todayVec.length;
+        for (let i = 0; i < featureCount; i++) {
+            const historicalValues = historicalFeatures.map(f => f.vector[i]);
+            const percentile = AdvancedMath.percentileRank(todayVec[i], historicalValues);
+            totalDeviation += Math.abs(percentile - 0.5) * 2;
+        }
+        return totalDeviation / featureCount;
+    },
+
+    _regimeScore: (todayFeatures, historicalFeatures, regime) => {
+        const sameRegimeFeatures = historicalFeatures.filter(f => f.metadata.regime === regime);
+        const comparisonSet = sameRegimeFeatures.length < 5 ? historicalFeatures : sameRegimeFeatures;
+        return AnomalyEngine._percentileScore(todayFeatures.vector, comparisonSet);
+    },
+
+    _covariance: (vectors, means) => {
+        const dim = vectors[0].length;
+        const n = vectors.length;
+        const matrix = Array(dim).fill(0).map(() => Array(dim).fill(0));
+        for (const v of vectors) {
+            for (let i = 0; i < dim; i++) {
+                for (let j = 0; j < dim; j++) {
+                    matrix[i][j] += (v[i] - means[i]) * (v[j] - means[j]);
+                }
+            }
+        }
+        // Ridge Regularization
+        return matrix.map((row, i) => row.map((val, j) => (val / (n - 1)) + (i === j ? 1e-3 : 0)));
+    },
+
+    _invert: (M) => {
+        try {
+            const n = M.length;
+            const A = M.map(row => [...row]);
+            const I = M.map((_, i) => M.map((__, j) => (i === j ? 1 : 0)));
+            for (let i = 0; i < n; i++) {
+                let pivot = A[i][i];
+                if (Math.abs(pivot) < 1e-9) pivot = 1e-9;
+                for (let j = 0; j < n; j++) { A[i][j] /= pivot; I[i][j] /= pivot; }
+                for (let k = 0; k < n; k++) {
+                    if (k !== i) {
+                        const f = A[k][i];
+                        for (let j = 0; j < n; j++) { A[k][j] -= f * A[i][j]; I[k][j] -= f * I[i][j]; }
+                    }
+                }
+            }
+            return I;
+        } catch (e) { return null; }
+    }
+};
+
+// =============================================================================
+// PREDICTIVE ENGINE & INTERPRETER
+// =============================================================================
+const PredictiveEngine = {
+    forecastRisk: (todayFeatures, historicalFeatures, anomalyScore) => {
+        const momentum = todayFeatures.vector[17]; 
+        const riskAccel = todayFeatures.vector[16];
+        
+        let baseProbability = 1 / (1 + Math.exp(-(anomalyScore - 3))); 
+        
+        if (momentum > 1.0) baseProbability = Math.min(0.99, baseProbability * 1.2);
+        if (riskAccel > 1.0) baseProbability = Math.min(0.99, baseProbability * 1.15);
+        
+        return {
+            probability7d: baseProbability,
+            confidence: historicalFeatures.length >= 25 ? 'high' : 'medium',
+            trajectory: momentum > 0 ? 'escalating' : 'stable'
+        };
+    }
+};
+
+const SemanticInterpreter = {
+    interpret: (scores, features, prediction, regime, featureNames) => {
+        // ROBUST LOGIC: Use the Z-Scores passed from AnomalyEngine.
+        const zScores = scores.zScores || [];
+        
+        let maxIdx = 0;
+        let maxZ = 0;
+        
+        // Find feature with highest ABSOLUTE Z-score (Statistically most significant)
+        zScores.forEach((z, i) => {
+            if (Math.abs(z) > maxZ) {
+                maxZ = Math.abs(z);
+                maxIdx = i;
+            }
+        });
+        
+        const primaryDriver = featureNames[maxIdx];
+        const driverZ = zScores[maxIdx];
+        const rawValue = features.vector[maxIdx];
+        
+        let description = '';
+        let severity = 'low';
+        
+        if (scores.overall > 5.0) { severity = 'critical'; description = `🚨 CRITICAL: Severe anomaly in ${regime} regime. `; }
+        else if (scores.overall > 4.0) { severity = 'high'; description = `⚠️ HIGH RISK: Significant deviation. `; }
+        else if (scores.overall > 3.0) { severity = 'medium'; description = `⚡ MODERATE: Unusual pattern. `; }
+        else { description = `ℹ️ NOTICE: Minor shift. `; }
+        
+        const direction = driverZ > 0 ? "increased" : "decreased";
+        
+        description += `Driven by ${primaryDriver}. `;
+        description += `Value ${direction} to ${rawValue.toFixed(2)} (${driverZ > 0 ? '+' : ''}${maxZ.toFixed(1)}σ). `;
+        
+        if (prediction.probability7d > 0.7) description += `High escalation risk (${(prediction.probability7d*100).toFixed(0)}%).`;
+        
+        return {
+            description,
+            severity,
+            primaryDriver,
+            driverValue: rawValue.toFixed(2),
+            driverSigma: (driverZ > 0 ? '+' : '-') + maxZ.toFixed(1) + 'σ',
+            regime
+        };
+    }
+};
+
+// =============================================================================
+// MAIN COMPUTATION
+// =============================================================================
 class BehavioralAnomaly extends Computation {
 
     static getConfig() {
@@ -10,271 +460,153 @@ class BehavioralAnomaly extends Computation {
             isHistorical: true,
             
             requires: {
+                // COST CONTROL: 30-day limit
                 'portfolio_snapshots': {
-                    lookback: 60,
+                    lookback: 30, 
                     mandatory: true,
                     fields: ['user_id', 'portfolio_data', 'date']
                 },
                 'pi_rankings': {
-                    lookback: 60,
-                    mandatory: true,
+                    lookback: 30,
+                    mandatory: true, 
                     fields: ['pi_id', 'rankings_data', 'date']
                 },
                 'trade_history_snapshots': {
-                    lookback: 60,
-                    mandatory: false, // Not all users have trade history every day
+                    lookback: 0,
+                    mandatory: false,
                     fields: ['user_id', 'history_data', 'date']
-                }
+                },
+                'ticker_mappings': { mandatory: false, fields: ['instrument_id', 'ticker'] },
+                'sector_mappings': { mandatory: false, fields: ['symbol', 'sector'] },
+                'pi_master_list': { mandatory: false, fields: ['cid', 'username'] }
             },
 
             storage: {
                 bigquery: true,
-                firestore: {
-                    enabled: true,
-                    path: 'alerts/{date}/BehavioralAnomaly/{entityId}',
-                    merge: true
+                firestore: { 
+                    enabled: true, 
+                    path: 'alerts/{date}/BehavioralAnomaly/{entityId}', 
+                    merge: true 
                 }
             },
-
-            // LEGACY METADATA PRESERVED
+            
             userType: 'POPULAR_INVESTOR',
             alert: {
-                id: 'behavioralAnomaly',
-                frontendName: 'Behavioral Anomaly',
-                description: 'Alert when a Popular Investor deviates significantly from their baseline behavior',
-                messageTemplate: 'Behavioral Alert for {piUsername}: {primaryDriver} Deviation ({driverSignificance}) detected.',
-                severity: 'high',
-                configKey: 'behavioralAnomaly',
-                isDynamic: true,
-                thresholds: [
-                    {
-                        key: 'anomalyScoreThreshold',
-                        type: 'number',
-                        label: 'Anomaly Sensitivity',
-                        description: 'Alert when anomaly score exceeds this threshold (higher = less sensitive)',
-                        default: 3.5,
-                        min: 2.0,
-                        max: 5.0,
-                        step: 0.5,
-                        unit: 'σ (standard deviations)'
-                    }
-                ],
-                conditions: [
-                    {
-                        key: 'watchedDrivers',
-                        type: 'array',
-                        label: 'Watched Behavior Drivers',
-                        description: 'Only alert for these specific behavioral factors (empty = all)',
-                        default: [],
-                        options: [
-                            { value: 'Concentration (HHI)', label: 'Portfolio Concentration' },
-                            { value: 'Martingale Behavior', label: 'Martingale Behavior' },
-                            { value: 'Capacity Strain', label: 'Capacity Strain' },
-                            { value: 'Risk Score', label: 'Risk Score Changes' }
-                        ]
-                    }
-                ],
-                resultKeys: ['triggered', 'anomalyScore', 'primaryDriver', 'driverSignificance', 'baselineDays']
+                id: 'behavioral_anomaly_v4',
+                frontendName: 'Behavioral Risk Intelligence',
+                severity: 'high', 
+                isDynamic: true
             }
         };
     }
 
-    // --- Math Helper (Self-Contained) ---
-    static LinearAlgebra = {
-        mean(data) {
-            const sum = data.reduce((a, b) => a + b, 0);
-            return sum / data.length;
-        },
-        covarianceMatrix(vectors) {
-            const n = vectors.length;
-            const dim = vectors[0].length;
-            const means = Array(dim).fill(0).map((_, i) => this.mean(vectors.map(v => v[i])));
-            const matrix = Array(dim).fill(0).map(() => Array(dim).fill(0));
-
-            for (let i = 0; i < dim; i++) {
-                for (let j = 0; j < dim; j++) {
-                    let sum = 0;
-                    for (let k = 0; k < n; k++) {
-                        sum += (vectors[k][i] - means[i]) * (vectors[k][j] - means[j]);
-                    }
-                    matrix[i][j] = sum / (n - 1);
-                }
-            }
-            return { matrix, means };
-        },
-        invertMatrix(M) {
-            // Simple 4x4 inversion or generic Gaussian elimination
-            // For brevity, assuming 4x4 or small dimensions suitable for JS
-            // (Implementation omitted for brevity, essentially standard Gaussian elimination)
-            // Simplified check for diagonal/stability
-             if(M.length === 0) return null;
-             // Placeholder for full inversion logic - in production use a library
-             // For this migration, we check for zero variance on diagonal
-             for(let i=0; i<M.length; i++) if(M[i][i] === 0) return null;
-             return M; // Mock return: In real usage, implement numeric.js or similar
-        },
-        mahalanobisDistance(vector, means, inverseCov) {
-            // D^2 = (x - u)^T * S^-1 * (x - u)
-            // Simplified Euclidean for fallback if inversion fails, or full calc
-            // For this output, we assume full calculation logic exists here.
-            let diff = vector.map((v, i) => v - means[i]);
-            let sum = 0;
-            // Mock calculation
-            return Math.sqrt(diff.reduce((acc, val) => acc + (val * val), 0)); 
-        }
-    };
-
-    calculateHHI(portfolioData) {
-        const positions = this.rules.portfolio.extractPositions(portfolioData);
-        if (!positions || positions.length === 0) return 0;
-
-        let sumSquares = 0;
-        const totalValue = this.rules.portfolio.calculateTotalValue(positions);
-
-        positions.forEach(p => {
-            // Calculate actual % weight based on Value
-            const val = this.rules.portfolio.getValue(p);
-            const weight = totalValue > 0 ? (val / totalValue) * 100 : 0;
-            sumSquares += (weight * weight);
-        });
-        return sumSquares;
-    }
-
-    calculateMartingaleScore(historyData) {
-        const trades = this.rules.trades.extractTrades(historyData);
-        if (!trades || trades.length < 2) return 0;
-        
-        // Sort by close date
-        const sorted = [...trades].sort((a, b) => 
-            (this.rules.trades.getCloseDate(a) || 0) - (this.rules.trades.getCloseDate(b) || 0)
-        );
-        const recent = sorted.slice(-30);
-
-        let lossEvents = 0;
-        let martingaleResponses = 0;
-
-        for (let i = 0; i < recent.length - 1; i++) {
-            const current = recent[i];
-            const next = recent[i+1];
+    async process(context) {
+        const { data, entityId, date, rules, references } = context;
 
-            if (this.rules.trades.getNetProfit(current) < 0) {
-                lossEvents++;
-                const currentLev = this.rules.trades.getLeverage(current);
-                const nextLev = this.rules.trades.getLeverage(next);
-                if (nextLev > currentLev) martingaleResponses++;
-            }
+        // 1. Map Building
+        const tickerMap = new Map();
+        if (references.ticker_mappings) {
+            Object.values(references.ticker_mappings).forEach(r => tickerMap.set(String(r.instrument_id), r.ticker));
         }
-        return lossEvents > 0 ? (martingaleResponses / lossEvents) : 0;
-    }
-
-    getDailyVector(portfolioData, rankingsData, historyData) {
-        const hhi = this.calculateHHI(portfolioData);
-        const mScore = this.calculateMartingaleScore(historyData);
-        
-        let strain = 0;
-        let risk = 1;
-
-        if (rankingsData) {
-            const copiers = this.rules.rankings.getCopiers(rankingsData);
-            const aum = this.rules.rankings.getAUM(rankingsData);
-            strain = aum > 0 ? (copiers / (aum / 1000)) : 0;
-            risk = this.rules.rankings.getRiskScore(rankingsData);
+        const sectorMap = new Map();
+        if (references.sector_mappings) {
+            Object.values(references.sector_mappings).forEach(r => sectorMap.set(r.symbol, r.sector));
         }
+        const maps = { tickerMap, sectorMap };
+
+        // 2. Data Alignment
+        const toDateStr = (d) => d && d.value ? d.value : (d instanceof Date ? d.toISOString().slice(0, 10) : String(d));
+        const getRows = (dataset) => {
+            if (!dataset) return [];
+            if (dataset[entityId]) return Array.isArray(dataset[entityId]) ? dataset[entityId] : [dataset[entityId]];
+            if (Array.isArray(dataset)) return dataset.filter(r => String(r.user_id || r.pi_id || r.cid) === String(entityId));
+            return [];
+        };
 
-        return [hhi, mScore, strain, risk];
-    }
+        const portfolios = getRows(data.portfolio_snapshots);
+        const rankings = getRows(data.pi_rankings);
+        const historyRows = getRows(data.trade_history_snapshots);
+        const currentHistoryBlob = historyRows.length > 0 ? (historyRows[0].history_data?.PublicHistoryPositions || []) : [];
+        const identityRows = getRows(data.pi_master_list);
+        const username = identityRows.length > 0 ? identityRows[0].username : "Unknown";
+
+        const dailyData = new Map();
+        portfolios.forEach(p => {
+            const d = toDateStr(p.date);
+            if (!dailyData.has(d)) dailyData.set(d, {});
+            dailyData.get(d).portfolio = p.portfolio_data;
+            dailyData.get(d).date = d;
+        });
+        rankings.forEach(r => {
+            const d = toDateStr(r.date);
+            if (dailyData.has(d)) dailyData.get(d).rankings = r.rankings_data;
+        });
 
-    async process(context) {
-        const { data, entityId, rules } = context;
-        this.rules = rules; // Attach rules for helper access
+        if (!dailyData.has(date)) return;
 
-        const portfolios = data['portfolio_snapshots'] || [];
-        const rankings = data['pi_rankings'] || [];
-        const histories = data['trade_history_snapshots'] || [];
+        // 3. Historical Extraction (30 days)
+        const historicalFeatures = [];
+        const lookbackDate = new Date(date);
+        lookbackDate.setDate(lookbackDate.getDate() - 30); 
 
-        // Sort data by date ascending for baseline building
-        portfolios.sort((a, b) => new Date(a.date) - new Date(b.date));
+        let prevDay = null;
+        let prevPrevDay = null;
         
-        const trainingVectors = [];
-        const MIN_DATAPOINTS = 15;
-
-        // Build Baseline
-        for (const portRow of portfolios) {
-            const date = portRow.date;
+        for (let d = new Date(lookbackDate); d < new Date(date); d.setDate(d.getDate() + 1)) {
+            const dStr = d.toISOString().slice(0, 10);
+            const dayData = dailyData.get(dStr);
             
-            // Match ranking and history for this date
-            const rankRow = rankings.find(r => r.date === date);
-            const histRow = histories.find(h => h.date === date);
-
-            const portData = rules.portfolio.extractPortfolioData(portRow);
-            const rankData = rules.rankings.extractRankingsData(rankRow);
-            
-            // Note: History logic simplified; V1 filtered full history, 
-            // here we rely on the daily snapshots which usually contain accumulated history
-            const histData = histRow; 
-
-            if (portData) {
-                const vec = this.getDailyVector(portData, rankData, histData);
-                trainingVectors.push(vec);
+            if (dayData && dayData.portfolio) {
+                const features = FeatureExtractor.extract(dayData, prevDay, prevPrevDay, currentHistoryBlob, dayData.rankings, rules, maps);
+                historicalFeatures.push(features);
+                prevPrevDay = prevDay;
+                prevDay = dayData;
             }
         }
 
-        if (trainingVectors.length < MIN_DATAPOINTS) {
-            this.setResult(entityId, { status: 'INSUFFICIENT_BASELINE', dataPoints: trainingVectors.length });
-            return;
+        if (historicalFeatures.length < 15) {
+             this.setResult(entityId, { triggered: false, status: 'INSUFFICIENT_HISTORY' });
+             return;
         }
 
-        // Compute Stats
-        const stats = BehavioralAnomaly.LinearAlgebra.covarianceMatrix(trainingVectors);
-        // Note: Real implementation needs a robust inversion. 
-        // If variance is 0, inversion fails.
-        const inverseCov = BehavioralAnomaly.LinearAlgebra.invertMatrix(stats.matrix);
-
-        if (!inverseCov) {
-            this.setResult(entityId, { status: 'STABLE_STATE', info: 'Variance too low' });
-            return;
-        }
+        // 4. Today's Features
+        const todayData = dailyData.get(date);
+        const todayFeatures = FeatureExtractor.extract(todayData, prevDay, prevPrevDay, currentHistoryBlob, todayData.rankings, rules, maps);
 
-        // Today's Vector (Last element in sorted array)
-        const todayPortRow = portfolios[portfolios.length - 1];
-        const todayRankRow = rankings.find(r => r.date === todayPortRow.date);
-        const todayHistRow = histories.find(h => h.date === todayPortRow.date);
-
-        const todayVector = this.getDailyVector(
-            rules.portfolio.extractPortfolioData(todayPortRow),
-            rules.rankings.extractRankingsData(todayRankRow),
-            todayHistRow
-        );
-
-        const distance = BehavioralAnomaly.LinearAlgebra.mahalanobisDistance(todayVector, stats.means, inverseCov);
-        const IS_ANOMALY = distance > 3.5;
-
-        const result = {
-            triggered: IS_ANOMALY,
-            anomalyScore: parseFloat(distance.toFixed(2)),
-            baselineDays: trainingVectors.length
-        };
-
-        if (IS_ANOMALY) {
-            const featureNames = ['Concentration (HHI)', 'Martingale Behavior', 'Capacity Strain', 'Risk Score'];
-            let maxZ = 0;
-            let primaryDriver = 'Unknown';
-            
-            todayVector.forEach((val, i) => {
-                const stdDev = Math.sqrt(stats.matrix[i][i]);
-                const z = stdDev > 0 ? Math.abs((val - stats.means[i]) / stdDev) : 0;
-                if (z > maxZ) {
-                    maxZ = z;
-                    primaryDriver = featureNames[i];
-                }
-            });
+        // 5. Detection & Scoring
+        const recentVectors = historicalFeatures.slice(-5).map(f => f.vector);
+        const regime = AdvancedMath.detectRegime(recentVectors);
+        historicalFeatures.forEach(f => f.metadata.regime = regime); 
+        
+        const scores = AnomalyEngine.score(todayFeatures, historicalFeatures, regime);
+        const prediction = PredictiveEngine.forecastRisk(todayFeatures, historicalFeatures, scores.overall);
 
-            result.primaryDriver = primaryDriver;
-            result.driverSignificance = `${maxZ.toFixed(1)}σ`;
-            result.vectors = { current: todayVector, baselineMeans: stats.means };
-        }
+        const featureNames = [
+            'Sector HHI', 'Martingale', 'Avg Leverage', 'Risk Score', 'Complexity', 'Exposure', 
+            'Entropy', 'Drawdown', 'Win Rate Dev', 'Skewness', 'Stress Ratio', 'Credit Imbal', 
+            'High Lev Freq', 'Copier Mom', 'AUM Tier', 'Exp Velocity', 'Risk Accel', 'Behav Mom'
+        ];
 
-        this.setResult(entityId, result);
+        const interpretation = SemanticInterpreter.interpret(scores, todayFeatures, prediction, regime, featureNames);
+        
+        const THRESHOLD = 3.5;
+        const triggered = scores.overall > THRESHOLD || prediction.probability7d > 0.75;
+
+        this.setResult(entityId, {
+            triggered,
+            score: Number(scores.overall.toFixed(2)),
+            severity: interpretation.severity,
+            regime,
+            description: interpretation.description,
+            username,
+            driver: interpretation.primaryDriver,
+            driverValue: interpretation.driverValue,
+            prediction: {
+                probability: (prediction.probability7d * 100).toFixed(1) + '%',
+                trajectory: prediction.trajectory
+            },
+            features: todayFeatures.vector.map(v => Number(v.toFixed(3)))
+        });
     }
 }