bulltrackers-module 1.0.768 → 1.0.769

package/functions/computation-system-v2/computations/BehavioralAnomaly.js

@@ -1,15 +1,455 @@
 /**
- * @fileoverview Behavioral Anomaly Detection (V2)
- * Detects significant deviations in investor behavior compared to their 60-day baseline.
- * * MATH: Uses Mahalanobis Distance to detect outliers in multi-dimensional space.
- * FEATURES:
- * 1. Concentration (HHI): Are they suddenly putting all eggs in one basket?
- * 2. Martingale (Loss Chasing): Are they increasing leverage after losses?
- * 3. Capacity Strain: Is their AUM growing faster than their copy count implies (risk of slippage)?
- * 4. Risk Score: Sudden spikes in risk.
+ * @fileoverview Behavioral Anomaly Detection V4.1 - Predictive Risk Intelligence
+ * * ARCHITECTURE:
+ * - Hybrid Math: Uses 6-Dim Core for Stability + 18-Dim Full Spectrum for Detail.
+ * - Cost Control: Strict 30-day lookback enforced (compatible with DAG constraints).
+ * - Logic Fix: "Primary Driver" uses Statistical Z-Scores, not raw values.
+ * - Robustness: Dynamic Z-Scoring removes dependency on hardcoded divisors.
  */
+
 const { Computation } = require('../framework');
 
+// =============================================================================
+// ADVANCED MATH KERNEL
+// =============================================================================
+const AdvancedMath = {
+    entropy: (distribution) => {
+        const total = distribution.reduce((sum, val) => sum + val, 0);
+        if (total === 0) return 0;
+        let entropy = 0;
+        for (const val of distribution) {
+            if (val > 0) {
+                const p = val / total;
+                entropy -= p * Math.log2(p);
+            }
+        }
+        return entropy;
+    },
+
+    hhi: (shares) => {
+        const total = shares.reduce((sum, val) => sum + val, 0);
+        if (total === 0) return 0;
+        let hhi = 0;
+        for (const share of shares) {
+            const marketShare = share / total;
+            hhi += marketShare * marketShare;
+        }
+        return hhi * 10000;
+    },
+
+    velocity: (current, previous, timeDelta = 1) => {
+        if (timeDelta === 0) return 0;
+        return (current - previous) / timeDelta;
+    },
+
+    acceleration: (current, previous, previousPrevious, timeDelta = 1) => {
+        const v1 = AdvancedMath.velocity(current, previous, timeDelta);
+        const v0 = AdvancedMath.velocity(previous, previousPrevious, timeDelta);
+        return AdvancedMath.velocity(v1, v0, timeDelta);
+    },
+
+    percentileRank: (value, array) => {
+        if (!array.length) return 0.5;
+        let rank = 0;
+        for (const val of array) {
+            if (val < value) rank++;
+        }
+        return rank / array.length;
+    },
+
+    detectRegime: (recentVectors) => {
+        if (recentVectors.length < 3) return 'unknown';
+        // Calculate aggregate volatility across the CORE features (indices 0-5)
+        const coreIndices = [0, 1, 2, 3, 4, 5];
+        const volatilities = [];
+        
+        for (const f of coreIndices) {
+            const values = recentVectors.map(v => v[f]);
+            const mean = values.reduce((sum, v) => sum + v, 0) / values.length;
+            const variance = values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / values.length;
+            volatilities.push(Math.sqrt(variance));
+        }
+        const avgVolatility = volatilities.reduce((sum, v) => sum + v, 0) / volatilities.length;
+        
+        if (avgVolatility < 0.3) return 'calm';
+        if (avgVolatility < 0.8) return 'active';
+        return 'stressed';
+    }
+};
+
+// =============================================================================
+// FEATURE ENGINEERING
+// =============================================================================
+const FeatureExtractor = {
+    extract: (day, prevDay, prevPrevDay, historyBlob, rankings, rules, maps) => {
+        const { portfolio } = day;
+        const positions = rules.portfolio.extractPositions(portfolio) || [];
+        const rankingsData = day.rankings || {};
+        const { sectorMap, tickerMap } = maps;
+
+        // Resolve Sector
+        const getSector = (instId) => {
+            const ticker = tickerMap.get(String(instId));
+            if (!ticker) return 'Unknown';
+            return sectorMap.get(ticker) || 'Unknown';
+        };
+        
+        // 1. Sector Concentration (HHI)
+        const sectorExposure = {};
+        let totalInvestedPct = 0;
+        positions.forEach(p => {
+            const sector = getSector(p.InstrumentID);
+            const val = p.Invested || 0;
+            sectorExposure[sector] = (sectorExposure[sector] || 0) + val;
+            totalInvestedPct += val;
+        });
+        const sectorHHI = AdvancedMath.hhi(Object.values(sectorExposure));
+        
+        // 2. Martingale/Distress Score
+        let martingaleScore = 0;
+        let martingaleCount = 0;
+        positions.forEach(p => {
+            const pnl = p.NetProfit || 0;
+            const invested = p.Invested || 0;
+            if (pnl < -4 && invested > 4) {
+                martingaleScore += Math.abs(pnl) * invested;
+                martingaleCount++;
+            }
+        });
+        
+        // 3. Leverage Profile (Lazy Loaded)
+        const { avgLeverage, highLevCount } = FeatureExtractor._extractLeverageProfile(day.date, historyBlob);
+        
+        // 4. Risk Score
+        const riskScore = rankingsData.RiskScore || 1;
+        
+        // 5. Complexity
+        const complexity = positions.length;
+        
+        // 6. Exposure
+        const exposure = totalInvestedPct;
+        
+        // --- EXTENDED FEATURES (Full Spectrum) ---
+        
+        // 7. Entropy
+        const positionSizes = positions.map(p => p.Invested || 0);
+        const portfolioEntropy = AdvancedMath.entropy(positionSizes);
+        
+        // 8. Drawdown
+        const drawdownSeverity = Math.abs(rankingsData.PeakToValley || 0);
+        
+        // 9. Win Rate Deviation
+        const winRateDeviation = Math.abs((rankingsData.WinRatio || 50) - 50);
+        
+        // 10. Skewness
+        const sortedSizes = [...positionSizes].sort((a, b) => b - a);
+        const top3Share = sortedSizes.slice(0, 3).reduce((sum, v) => sum + v, 0);
+        const positionSkewness = totalInvestedPct > 0 ? top3Share / totalInvestedPct : 0;
+        
+        // 11. Stress Ratio
+        const losingCount = positions.filter(p => (p.NetProfit || 0) < 0).length;
+        const stressRatio = positions.length > 0 ? losingCount / positions.length : 0;
+        
+        // 12. Credit Imbalance
+        const realizedCredit = portfolio?.CreditByRealizedEquity || 0;
+        const unrealizedCredit = portfolio?.CreditByUnrealizedEquity || 0;
+        const creditImbalance = Math.abs(realizedCredit - unrealizedCredit);
+        
+        // 13. High Lev Freq
+        const highLevFrequency = highLevCount / Math.max(1, positions.length);
+        
+        // 14. Copier Momentum
+        const copiers = rankingsData.Copiers || 0;
+        const baselineCopiers = rankingsData.BaseLineCopiers || copiers;
+        const copierMomentum = baselineCopiers > 0 ? (copiers - baselineCopiers) / baselineCopiers : 0;
+        
+        // 15. AUM Tier
+        const aumTier = (rankingsData.AUMTier || 0);
+        
+        // 16. Exposure Velocity
+        let exposureVelocity = 0;
+        if (prevDay && prevDay.portfolio) {
+            const prevPositions = rules.portfolio.extractPositions(prevDay.portfolio) || [];
+            const prevExposure = prevPositions.reduce((sum, p) => sum + (p.Invested || 0), 0);
+            exposureVelocity = exposure - prevExposure;
+        }
+        
+        // 17. Risk Acceleration
+        let riskAcceleration = 0;
+        if (prevDay && prevDay.rankings && prevPrevDay && prevPrevDay.rankings) {
+            const r0 = prevPrevDay.rankings.RiskScore || 0;
+            const r1 = prevDay.rankings.RiskScore || 0;
+            const r2 = riskScore;
+            riskAcceleration = AdvancedMath.acceleration(r2, r1, r0, 1);
+        }
+        
+        // 18. Behavioral Momentum
+        const behavioralMomentum = Math.abs(exposureVelocity) + Math.abs(riskAcceleration);
+        
+        return {
+            vector: [
+                // CORE (0-5)
+                sectorHHI,              
+                Math.log(martingaleScore + 1),  
+                avgLeverage,                
+                riskScore,                 
+                complexity,                
+                exposure,  
+                
+                // EXTENDED (6-17)
+                portfolioEntropy,           
+                drawdownSeverity,         
+                winRateDeviation,          
+                positionSkewness,               
+                stressRatio,                    
+                creditImbalance,          
+                highLevFrequency,               
+                copierMomentum,                 
+                aumTier,                        
+                exposureVelocity,          
+                riskAcceleration,           
+                behavioralMomentum         
+            ],
+            metadata: {
+                martingaleCount,
+                losingCount,
+                totalPositions: positions.length,
+                exposure,
+                regime: 'unknown'
+            }
+        };
+    },
+
+    _extractLeverageProfile: (dateStr, historyBlob) => {
+        const targetDate = new Date(dateStr);
+        const lookbackWindow = new Date(targetDate);
+        lookbackWindow.setDate(lookbackWindow.getDate() - 90);
+
+        let sumLev = 0;
+        let tradeCount = 0;
+        let highLevCount = 0;
+
+        for (const t of historyBlob) {
+            const closeDate = new Date(t.CloseDateTime);
+            if (closeDate <= targetDate && closeDate >= lookbackWindow) {
+                const lev = t.Leverage || 1;
+                sumLev += lev;
+                tradeCount++;
+                if (lev > 2) highLevCount++;
+            }
+        }
+
+        return {
+            avgLeverage: tradeCount > 0 ? sumLev / tradeCount : 1,
+            highLevCount
+        };
+    }
+};
+
+// =============================================================================
+// ANOMALY DETECTION ENGINE
+// =============================================================================
+const AnomalyEngine = {
+    score: (todayFeatures, historicalFeatures, regime) => {
+        // 1. DYNAMIC STANDARDIZATION (Z-Scoring)
+        // We capture zVector here because it represents the "True Statistical Deviation"
+        const { zVector, histZVectors } = AnomalyEngine._standardize(todayFeatures.vector, historicalFeatures);
+
+        // 2. HYBRID MAHALANOBIS (Core Features Only)
+        // Slicing indices 0-5 (Core) for structural stability on 30-day lookback
+        const coreZVector = zVector.slice(0, 6);
+        const coreHistZVectors = histZVectors.map(v => v.slice(0, 6));
+        const mahalanobisScore = AnomalyEngine._mahalanobisDistance(coreZVector, coreHistZVectors);
+
+        // 3. FULL SPECTRUM PERCENTILE (All 18 Features)
+        const percentileScore = AnomalyEngine._percentileScore(todayFeatures.vector, historicalFeatures);
+
+        // 4. REGIME CONDITIONAL
+        const regimeScore = AnomalyEngine._regimeScore(todayFeatures, historicalFeatures, regime);
+
+        // 5. VELOCITY
+        const velocityScore = (Math.abs(zVector[15]) + Math.abs(zVector[16]) + Math.abs(zVector[17])) / 3;
+
+        const ensembleScore = 
+            0.40 * mahalanobisScore +
+            0.30 * percentileScore +
+            0.15 * regimeScore +
+            0.15 * velocityScore;
+
+        return {
+            overall: ensembleScore,
+            components: {
+                mahalanobis: mahalanobisScore,
+                percentile: percentileScore,
+                regime: regimeScore,
+                velocity: velocityScore
+            },
+            // EXPORT Z-SCORES FOR INTERPRETER
+            zScores: zVector 
+        };
+    },
+
+    _standardize: (todayVec, history) => {
+        const dim = todayVec.length;
+        const n = history.length;
+        const means = new Array(dim).fill(0);
+        const stdDevs = new Array(dim).fill(0);
+
+        const vectors = history.map(h => h.vector);
+        for (const v of vectors) for (let i = 0; i < dim; i++) means[i] += v[i];
+        for (let i = 0; i < dim; i++) means[i] /= n;
+
+        for (const v of vectors) for (let i = 0; i < dim; i++) stdDevs[i] += Math.pow(v[i] - means[i], 2);
+        for (let i = 0; i < dim; i++) {
+            stdDevs[i] = Math.sqrt(stdDevs[i] / n);
+            if (stdDevs[i] === 0) stdDevs[i] = 1; 
+        }
+
+        const zVector = todayVec.map((v, i) => (v - means[i]) / stdDevs[i]);
+        const histZVectors = vectors.map(v => v.map((val, i) => (val - means[i]) / stdDevs[i]));
+
+        return { zVector, histZVectors };
+    },
+
+    _mahalanobisDistance: (todayZVec, histZVectors) => {
+        const means = new Array(todayZVec.length).fill(0);
+        const cov = AnomalyEngine._covariance(histZVectors, means);
+        const invCov = AnomalyEngine._invert(cov);
+        
+        if (!invCov) return 0;
+        
+        let sum = 0;
+        for (let i = 0; i < todayZVec.length; i++) {
+            for (let j = 0; j < todayZVec.length; j++) {
+                sum += todayZVec[i] * invCov[i][j] * todayZVec[j];
+            }
+        }
+        return Math.sqrt(Math.max(0, sum));
+    },
+
+    _percentileScore: (todayVec, historicalFeatures) => {
+        let totalDeviation = 0;
+        const featureCount = todayVec.length;
+        for (let i = 0; i < featureCount; i++) {
+            const historicalValues = historicalFeatures.map(f => f.vector[i]);
+            const percentile = AdvancedMath.percentileRank(todayVec[i], historicalValues);
+            totalDeviation += Math.abs(percentile - 0.5) * 2;
+        }
+        return totalDeviation / featureCount;
+    },
+
+    _regimeScore: (todayFeatures, historicalFeatures, regime) => {
+        const sameRegimeFeatures = historicalFeatures.filter(f => f.metadata.regime === regime);
+        const comparisonSet = sameRegimeFeatures.length < 5 ? historicalFeatures : sameRegimeFeatures;
+        return AnomalyEngine._percentileScore(todayFeatures.vector, comparisonSet);
+    },
+
+    _covariance: (vectors, means) => {
+        const dim = vectors[0].length;
+        const n = vectors.length;
+        const matrix = Array(dim).fill(0).map(() => Array(dim).fill(0));
+        for (const v of vectors) {
+            for (let i = 0; i < dim; i++) {
+                for (let j = 0; j < dim; j++) {
+                    matrix[i][j] += (v[i] - means[i]) * (v[j] - means[j]);
+                }
+            }
+        }
+        // Ridge Regularization
+        return matrix.map((row, i) => row.map((val, j) => (val / (n - 1)) + (i === j ? 1e-3 : 0)));
+    },
+
+    _invert: (M) => {
+        try {
+            const n = M.length;
+            const A = M.map(row => [...row]);
+            const I = M.map((_, i) => M.map((__, j) => (i === j ? 1 : 0)));
+            for (let i = 0; i < n; i++) {
+                let pivot = A[i][i];
+                if (Math.abs(pivot) < 1e-9) pivot = 1e-9;
+                for (let j = 0; j < n; j++) { A[i][j] /= pivot; I[i][j] /= pivot; }
+                for (let k = 0; k < n; k++) {
+                    if (k !== i) {
+                        const f = A[k][i];
+                        for (let j = 0; j < n; j++) { A[k][j] -= f * A[i][j]; I[k][j] -= f * I[i][j]; }
+                    }
+                }
+            }
+            return I;
+        } catch (e) { return null; }
+    }
+};
+
+// =============================================================================
+// PREDICTIVE ENGINE & INTERPRETER
+// =============================================================================
+const PredictiveEngine = {
+    forecastRisk: (todayFeatures, historicalFeatures, anomalyScore) => {
+        const momentum = todayFeatures.vector[17]; 
+        const riskAccel = todayFeatures.vector[16];
+        
+        let baseProbability = 1 / (1 + Math.exp(-(anomalyScore - 3))); 
+        
+        if (momentum > 1.0) baseProbability = Math.min(0.99, baseProbability * 1.2);
+        if (riskAccel > 1.0) baseProbability = Math.min(0.99, baseProbability * 1.15);
+        
+        return {
+            probability7d: baseProbability,
+            confidence: historicalFeatures.length >= 25 ? 'high' : 'medium',
+            trajectory: momentum > 0 ? 'escalating' : 'stable'
+        };
+    }
+};
+
+const SemanticInterpreter = {
+    interpret: (scores, features, prediction, regime, featureNames) => {
+        // ROBUST LOGIC: Use the Z-Scores passed from AnomalyEngine.
+        const zScores = scores.zScores || [];
+        
+        let maxIdx = 0;
+        let maxZ = 0;
+        
+        // Find feature with highest ABSOLUTE Z-score (Statistically most significant)
+        zScores.forEach((z, i) => {
+            if (Math.abs(z) > maxZ) {
+                maxZ = Math.abs(z);
+                maxIdx = i;
+            }
+        });
+        
+        const primaryDriver = featureNames[maxIdx];
+        const driverZ = zScores[maxIdx];
+        const rawValue = features.vector[maxIdx];
+        
+        let description = '';
+        let severity = 'low';
+        
+        if (scores.overall > 5.0) { severity = 'critical'; description = `🚨 CRITICAL: Severe anomaly in ${regime} regime. `; }
+        else if (scores.overall > 4.0) { severity = 'high'; description = `⚠️ HIGH RISK: Significant deviation. `; }
+        else if (scores.overall > 3.0) { severity = 'medium'; description = `⚡ MODERATE: Unusual pattern. `; }
+        else { description = `ℹ️ NOTICE: Minor shift. `; }
+        
+        const direction = driverZ > 0 ? "increased" : "decreased";
+        
+        description += `Driven by ${primaryDriver}. `;
+        description += `Value ${direction} to ${rawValue.toFixed(2)} (${driverZ > 0 ? '+' : ''}${maxZ.toFixed(1)}σ). `;
+        
+        if (prediction.probability7d > 0.7) description += `High escalation risk (${(prediction.probability7d*100).toFixed(0)}%).`;
+        
+        return {
+            description,
+            severity,
+            primaryDriver,
+            driverValue: rawValue.toFixed(2),
+            driverSigma: (driverZ > 0 ? '+' : '-') + maxZ.toFixed(1) + 'σ',
+            regime
+        };
+    }
+};
+
+// =============================================================================
+// MAIN COMPUTATION
+// =============================================================================
 class BehavioralAnomaly extends Computation {
 
     static getConfig() {
@@ -20,373 +460,153 @@ class BehavioralAnomaly extends Computation {
             isHistorical: true,
             
             requires: {
-                // We need 60 days of history to build a statistical baseline
+                // COST CONTROL: 30-day limit
                 'portfolio_snapshots': {
-                    lookback: 60,
+                    lookback: 30, 
                     mandatory: true,
                     fields: ['user_id', 'portfolio_data', 'date']
                 },
                 'pi_rankings': {
-                    lookback: 60,
-                    mandatory: true,
+                    lookback: 30,
+                    mandatory: true, 
                     fields: ['pi_id', 'rankings_data', 'date']
                 },
                 'trade_history_snapshots': {
-                    lookback: 60,
-                    mandatory: false, // Not all users have trade history every day
+                    lookback: 0,
+                    mandatory: false,
                     fields: ['user_id', 'history_data', 'date']
-                }
+                },
+                'ticker_mappings': { mandatory: false, fields: ['instrument_id', 'ticker'] },
+                'sector_mappings': { mandatory: false, fields: ['symbol', 'sector'] },
+                'pi_master_list': { mandatory: false, fields: ['cid', 'username'] }
             },
 
             storage: {
                 bigquery: true,
-                firestore: {
-                    enabled: true,
-                    path: 'alerts/{date}/BehavioralAnomaly/{entityId}',
-                    merge: true
+                firestore: { 
+                    enabled: true, 
+                    path: 'alerts/{date}/BehavioralAnomaly/{entityId}', 
+                    merge: true 
                 }
             },
-
+            
             userType: 'POPULAR_INVESTOR',
             alert: {
-                id: 'behavioralAnomaly',
-                frontendName: 'Behavioral Anomaly',
-                description: 'Alert when a Popular Investor deviates significantly from their baseline behavior',
-                messageTemplate: 'Behavioral Alert for {username}: {primaryDriver} Deviation ({driverSignificance}) detected.',
-                severity: 'high',
-                configKey: 'behavioralAnomaly',
-                isDynamic: true,
-                thresholds: [
-                    {
-                        key: 'anomalyScoreThreshold',
-                        type: 'number',
-                        label: 'Sensitivity Threshold',
-                        default: 3.5,
-                        min: 2.0, max: 10.0, step: 0.1
-                    }
-                ],
-                resultFields: {
-                    driver: 'primaryDriver',
-                    score: 'driverSignificance'
-                }
+                id: 'behavioral_anomaly_v4',
+                frontendName: 'Behavioral Risk Intelligence',
+                severity: 'high', 
+                isDynamic: true
             }
         };
     }
 
     async process(context) {
-        const { data, entityId, date, rules } = context;
+        const { data, entityId, date, rules, references } = context;
 
-        // =====================================================================
-        // 1. DATA PREPARATION
-        // =====================================================================
-        
-        // Helper: Safe Date String
-        const toDateStr = (d) => {
-            if (!d) return "";
-            if (d.value) return d.value;
-            return d instanceof Date ? d.toISOString().slice(0, 10) : String(d);
-        };
+        // 1. Map Building
+        const tickerMap = new Map();
+        if (references.ticker_mappings) {
+            Object.values(references.ticker_mappings).forEach(r => tickerMap.set(String(r.instrument_id), r.ticker));
+        }
+        const sectorMap = new Map();
+        if (references.sector_mappings) {
+            Object.values(references.sector_mappings).forEach(r => sectorMap.set(r.symbol, r.sector));
+        }
+        const maps = { tickerMap, sectorMap };
 
-        // Helper: Access Data Map or Array safely (V2 Pattern)
-        const getEntityRows = (dataset) => {
+        // 2. Data Alignment
+        const toDateStr = (d) => d && d.value ? d.value : (d instanceof Date ? d.toISOString().slice(0, 10) : String(d));
+        const getRows = (dataset) => {
             if (!dataset) return [];
-            if (dataset[entityId]) {
-                return Array.isArray(dataset[entityId]) ? dataset[entityId] : [dataset[entityId]];
-            }
-            if (Array.isArray(dataset)) {
-                return dataset.filter(r => String(r.pi_id || r.user_id || r.cid) === String(entityId));
-            }
+            if (dataset[entityId]) return Array.isArray(dataset[entityId]) ? dataset[entityId] : [dataset[entityId]];
+            if (Array.isArray(dataset)) return dataset.filter(r => String(r.user_id || r.pi_id || r.cid) === String(entityId));
             return [];
         };
 
-        // Fetch Data
-        const portfolios = getEntityRows(data['portfolio_snapshots']);
-        const rankings = getEntityRows(data['pi_rankings']);
-        const history = getEntityRows(data['trade_history_snapshots']);
-
-        // Index by Date for fast alignment
-        // { "2026-01-28": row }
-        const portMap = new Map(portfolios.map(r => [toDateStr(r.date), r]));
-        const rankMap = new Map(rankings.map(r => [toDateStr(r.date), r]));
-        const histMap = new Map(history.map(r => [toDateStr(r.date), r]));
-
-        // Identify "Today" (Execution Date)
-        const todayPort = portMap.get(date);
-        const todayRank = rankMap.get(date);
-        const todayHist = histMap.get(date);
-
-        if (!todayPort || !todayRank) {
-            // Cannot run anomaly detection without today's data
-            return; 
-        }
-
-        // =====================================================================
-        // 2. FEATURE ENGINEERING
-        // =====================================================================
-
-        /**
-         * 1. Concentration (HHI)
-         * Sum of squared weights. High HHI = Low Diversification.
-         */
-        const calculateHHI = (portRow) => {
-            const pData = rules.portfolio.extractPortfolioData(portRow);
-            const positions = rules.portfolio.extractPositions(pData);
-            
-            if (!positions || positions.length === 0) return 0;
-
-            let sumSquares = 0;
-            let totalInvested = 0;
-
-            positions.forEach(p => {
-                // Invested is typically a %. If it's absolute $, we normalize later.
-                // Assuming standard eToro data where Invested is a % (0-100 or 0-1)
-                const val = rules.portfolio.getInvested(p) || 0; 
-                totalInvested += val;
-                sumSquares += (val * val);
-            });
-
-            // Normalize if total > 0 (Standard HHI ranges 0 to 10,000)
-            // If weights sum to 100, HHI is sum(w^2).
-            return sumSquares; 
-        };
-
-        /**
-         * 2. Martingale Score
-         * Checks if leverage increases after a loss (Loss Chasing).
-         */
-        const calculateMartingale = (histRow) => {
-            const trades = rules.trades.extractTrades(histRow);
-            if (!trades || trades.length < 2) return 0;
-
-            // Sort by close date (Oldest -> Newest)
-            trades.sort((a, b) => {
-                const dA = rules.trades.getCloseDate(a);
-                const dB = rules.trades.getCloseDate(b);
-                return (dA || 0) - (dB || 0);
-            });
-
-            // Look at recent behavior (last 30 trades in this snapshot)
-            const recent = trades.slice(-30);
-            let lossEvents = 0;
-            let martingaleResponses = 0;
-
-            for (let i = 0; i < recent.length - 1; i++) {
-                const current = recent[i];
-                const next = recent[i+1];
-
-                const profit = rules.trades.getNetProfit(current);
-                if (profit < 0) {
-                    lossEvents++;
-                    const curLev = rules.trades.getLeverage(current) || 1;
-                    const nextLev = rules.trades.getLeverage(next) || 1;
-                    
-                    if (nextLev > curLev) {
-                        martingaleResponses++;
-                    }
-                }
-            }
-
-            return lossEvents > 0 ? (martingaleResponses / lossEvents) : 0;
-        };
-
-        /**
-         * 3. Capacity Strain
-         * Copiers / AUM. If copiers grow but AUM doesn't, efficiency drops.
-         */
-        const calculateStrain = (rankRow) => {
-            const rData = rules.rankings.extractRankingsData(rankRow);
-            if (!rData) return 0;
-
-            const copiers = rules.rankings.getCopiers(rData) || 0;
-            // AUM Tier is usually an int (1-6). We need approximate value or raw AUM if available.
-            // Using raw AUMValue if available in JSON, else estimate from tier.
-            const aum = rData.AUMValue || (rules.rankings.getAUMTier(rData) * 10000) || 1; 
-            
-            return (copiers / (aum / 1000)); // Normalized ratio
-        };
-
-        /**
-         * 4. Risk Score
-         */
-        const getRisk = (rankRow) => {
-            const rData = rules.rankings.extractRankingsData(rankRow);
-            return rules.rankings.getRiskScore(rData) || 1;
-        };
-
-        // --- Build Vectors ---
-        const getDailyVector = (pRow, rRow, hRow) => {
-            if (!pRow || !rRow) return null;
-            return [
-                calculateHHI(pRow),
-                hRow ? calculateMartingale(hRow) : 0,
-                calculateStrain(rRow),
-                getRisk(rRow)
-            ];
-        };
-
-        // =====================================================================
-        // 3. BASELINE CONSTRUCTION
-        // =====================================================================
-        
-        const trainingVectors = [];
-        
-        // Loop through last 60 days (excluding today)
-        const sortedDates = Array.from(portMap.keys()).sort();
+        const portfolios = getRows(data.portfolio_snapshots);
+        const rankings = getRows(data.pi_rankings);
+        const historyRows = getRows(data.trade_history_snapshots);
+        const currentHistoryBlob = historyRows.length > 0 ? (historyRows[0].history_data?.PublicHistoryPositions || []) : [];
+        const identityRows = getRows(data.pi_master_list);
+        const username = identityRows.length > 0 ? identityRows[0].username : "Unknown";
+
+        const dailyData = new Map();
+        portfolios.forEach(p => {
+            const d = toDateStr(p.date);
+            if (!dailyData.has(d)) dailyData.set(d, {});
+            dailyData.get(d).portfolio = p.portfolio_data;
+            dailyData.get(d).date = d;
+        });
+        rankings.forEach(r => {
+            const d = toDateStr(r.date);
+            if (dailyData.has(d)) dailyData.get(d).rankings = r.rankings_data;
+        });
+
+        if (!dailyData.has(date)) return;
+
+        // 3. Historical Extraction (30 days)
+        const historicalFeatures = [];
+        const lookbackDate = new Date(date);
+        lookbackDate.setDate(lookbackDate.getDate() - 30); 
+
+        let prevDay = null;
+        let prevPrevDay = null;
         
-        for (const d of sortedDates) {
-            if (d === date) continue; // Don't include today in the baseline training
-
-            const pRow = portMap.get(d);
-            const rRow = rankMap.get(d); // Matches date?
-            const hRow = histMap.get(d); // History available for this date?
-
-            // We need intersection of Portfolio and Ranking to form a valid vector point
-            if (pRow && rRow) {
-                const vec = getDailyVector(pRow, rRow, hRow);
-                if (vec) trainingVectors.push(vec);
+        for (let d = new Date(lookbackDate); d < new Date(date); d.setDate(d.getDate() + 1)) {
+            const dStr = d.toISOString().slice(0, 10);
+            const dayData = dailyData.get(dStr);
+            
+            if (dayData && dayData.portfolio) {
+                const features = FeatureExtractor.extract(dayData, prevDay, prevPrevDay, currentHistoryBlob, dayData.rankings, rules, maps);
+                historicalFeatures.push(features);
+                prevPrevDay = prevDay;
+                prevDay = dayData;
             }
         }
 
-        // Need minimum data to be statistically significant
-        if (trainingVectors.length < 15) {
-            this.setResult(entityId, { 
-                triggered: false, 
-                status: 'INSUFFICIENT_BASELINE', 
-                dataPoints: trainingVectors.length 
-            });
-            return;
+        if (historicalFeatures.length < 15) {
+             this.setResult(entityId, { triggered: false, status: 'INSUFFICIENT_HISTORY' });
+             return;
         }
 
-        // =====================================================================
-        // 4. LINEAR ALGEBRA ENGINE
-        // =====================================================================
-        
-        const MathLib = {
-            // Mean of each column
-            mean: (vectors) => {
-                const dim = vectors[0].length;
-                const means = new Array(dim).fill(0);
-                for (const v of vectors) {
-                    for (let i = 0; i < dim; i++) means[i] += v[i];
-                }
-                return means.map(x => x / vectors.length);
-            },
+        // 4. Today's Features
+        const todayData = dailyData.get(date);
+        const todayFeatures = FeatureExtractor.extract(todayData, prevDay, prevPrevDay, currentHistoryBlob, todayData.rankings, rules, maps);
 
-            // Covariance Matrix
-            covariance: (vectors, means) => {
-                const dim = vectors[0].length;
-                const n = vectors.length;
-                const matrix = Array(dim).fill(0).map(() => Array(dim).fill(0));
-
-                for (const v of vectors) {
-                    for (let i = 0; i < dim; i++) {
-                        for (let j = 0; j < dim; j++) {
-                            matrix[i][j] += (v[i] - means[i]) * (v[j] - means[j]);
-                        }
-                    }
-                }
-                return matrix.map(row => row.map(val => val / (n - 1))); // Sample Covariance
-            },
-
-            // Matrix Inversion (Gauss-Jordan)
-            invert: (M) => {
-                // Simplified for 4x4. Returns null if singular.
-                try {
-                    const n = M.length;
-                    const A = M.map(row => [...row]); // Clone
-                    const I = M.map((row, i) => M.map((_, j) => (i === j ? 1 : 0))); // Identity
-
-                    for (let i = 0; i < n; i++) {
-                        let pivot = A[i][i];
-                        if (Math.abs(pivot) < 1e-8) return null; // Singular
-
-                        for (let j = 0; j < n; j++) {
-                            A[i][j] /= pivot;
-                            I[i][j] /= pivot;
-                        }
-
-                        for (let k = 0; k < n; k++) {
-                            if (k !== i) {
-                                const factor = A[k][i];
-                                for (let j = 0; j < n; j++) {
-                                    A[k][j] -= factor * A[i][j];
-                                    I[k][j] -= factor * I[i][j];
-                                }
-                            }
-                        }
-                    }
-                    return I;
-                } catch (e) { return null; }
-            },
-
-            // Mahalanobis Distance
-            distance: (v, means, invCov) => {
-                const diff = v.map((val, i) => val - means[i]);
-                const dim = diff.length;
-                let sum = 0;
-                
-                for (let i = 0; i < dim; i++) {
-                    let temp = 0;
-                    for (let j = 0; j < dim; j++) {
-                        temp += diff[j] * invCov[j][i];
-                    }
-                    sum += temp * diff[i];
-                }
-                return Math.sqrt(Math.max(0, sum));
-            }
-        };
-
-        // =====================================================================
-        // 5. ANOMALY DETECTION
-        // =====================================================================
-
-        const means = MathLib.mean(trainingVectors);
-        const covMatrix = MathLib.covariance(trainingVectors, means);
-        const invCov = MathLib.invert(covMatrix);
-
-        if (!invCov) {
-            this.setResult(entityId, { triggered: false, status: 'SINGULAR_MATRIX_ERROR' });
-            return;
-        }
-
-        const todayVector = getDailyVector(todayPort, todayRank, todayHist);
-        const distance = MathLib.distance(todayVector, means, invCov);
-        
-        // Threshold from Alert Config (or default 3.5 sigma)
-        const threshold = 3.5; 
-        const isAnomaly = distance > threshold;
-
-        const featureNames = ['Concentration (HHI)', 'Martingale Behavior', 'Capacity Strain', 'Risk Score'];
+        // 5. Detection & Scoring
+        const recentVectors = historicalFeatures.slice(-5).map(f => f.vector);
+        const regime = AdvancedMath.detectRegime(recentVectors);
+        historicalFeatures.forEach(f => f.metadata.regime = regime); 
         
-        let primaryDriver = 'Unknown';
-        let maxZ = 0;
+        const scores = AnomalyEngine.score(todayFeatures, historicalFeatures, regime);
+        const prediction = PredictiveEngine.forecastRisk(todayFeatures, historicalFeatures, scores.overall);
 
-        // Determine *Why* it's an anomaly (Z-Score contribution)
-        if (isAnomaly) {
-            todayVector.forEach((val, i) => {
-                const stdDev = Math.sqrt(covMatrix[i][i]);
-                const z = stdDev > 0 ? Math.abs((val - means[i]) / stdDev) : 0;
-                if (z > maxZ) {
-                    maxZ = z;
-                    primaryDriver = featureNames[i];
-                }
-            });
-        }
+        const featureNames = [
+            'Sector HHI', 'Martingale', 'Avg Leverage', 'Risk Score', 'Complexity', 'Exposure', 
+            'Entropy', 'Drawdown', 'Win Rate Dev', 'Skewness', 'Stress Ratio', 'Credit Imbal', 
+            'High Lev Freq', 'Copier Mom', 'AUM Tier', 'Exp Velocity', 'Risk Accel', 'Behav Mom'
+        ];
 
-        const result = {
-            triggered: isAnomaly,
-            anomalyScore: Number(distance.toFixed(2)),
-            primaryDriver: isAnomaly ? primaryDriver : null,
-            driverSignificance: isAnomaly ? `${maxZ.toFixed(1)}σ` : null,
-            baselineDays: trainingVectors.length,
-            username: rules.rankings.extractRankingsData(todayRank)?.UserName || "Unknown",
-            _debug: {
-                currentVector: todayVector,
-                baselineMeans: means
-            }
-        };
-
-        this.setResult(entityId, result);
+        const interpretation = SemanticInterpreter.interpret(scores, todayFeatures, prediction, regime, featureNames);
+        
+        const THRESHOLD = 3.5;
+        const triggered = scores.overall > THRESHOLD || prediction.probability7d > 0.75;
+
+        this.setResult(entityId, {
+            triggered,
+            score: Number(scores.overall.toFixed(2)),
+            severity: interpretation.severity,
+            regime,
+            description: interpretation.description,
+            username,
+            driver: interpretation.primaryDriver,
+            driverValue: interpretation.driverValue,
+            prediction: {
+                probability: (prediction.probability7d * 100).toFixed(1) + '%',
+                trajectory: prediction.trajectory
+            },
+            features: todayFeatures.vector.map(v => Number(v.toFixed(3)))
+        });
     }
 }