aiden-shared-calculations-unified 1.0.45 → 1.0.47

package/calculations/meta/social-predictive-regime-state.js

@@ -0,0 +1,181 @@
+/**
+ * @fileoverview NEW CALCULATION (PASS 4 - META)
+ * This is the "Quant Regime Engine" or "Master Switch."
+ *
+ * It determines if the entire class of "social-to-price" signals
+ * is currently effective ("ON") or not ("OFF").
+ *
+ * It is a STATEFUL, ROLLING calculation.
+ * 1. It consumes the 'daily_topic_signals' from Pass 3 for the current day.
+ * 2. It reads its *own* rolling history of past "breadth" metrics
+ * from a single document in Firestore ('social_prediction_regime_state/history').
+ * 3. It calculates the "predictive breadth" for the current day:
+ * (What % of all topics analyzed had a Predictive Potential > threshold?)
+ * 4. It appends this new breadth metric to its history.
+ * 5. It calculates a short-term and long-term EMA of this breadth metric.
+ * 6. It uses the EMA crossover to determine the `regime_state`.
+ * 7. It returns its updated state AND the daily regime signal.
+ */
+
+const { FieldValue } = require('@google-cloud/firestore');
+
+// --- CONFIGURATION ---
+// The single doc where this calc stores its rolling history
+const STATE_DOC_PATH = 'social_prediction_regime_state/history'; 
+const ROLLING_HISTORY_DAYS = 90; // History for EMAs
+const EMA_SHORT_PERIOD = 3;  // 3-day EMA
+const EMA_LONG_PERIOD = 14; // 14-day EMA
+// How far apart the EMAs must be to declare a firm regime
+const REGIME_THRESHOLD = 0.02; // e.g., 2% difference
+// The Predictive Potential (from Pass 3) threshold to be counted
+const PP_THRESHOLD = 0.5; 
+
+// --- STATS HELPER ---
+/**
+ * Calculates a simple Exponential Moving Average (EMA) from an array of values.
+ * @param {number[]} data - Array of numbers (oldest to newest).
+ * @param {number} period - The EMA period.
+ * @returns {number|null} The final EMA value, or null if not enough data.
+ */
+function _calculateEMA(data, period) {
+    if (data.length < period) {
+        return null;
+    }
+    const k = 2 / (period + 1); // Smoothing factor
+    let ema = data[0]; // Start with the first value
+    for (let i = 1; i < data.length; i++) {
+        ema = (data[i] * k) + (ema * (1 - k));
+    }
+    return ema;
+}
+
+
+class SocialPredictiveRegimeState {
+
+    static getDependencies() {
+        // Depends on the *daily output* from Pass 3
+        return ['social-topic-predictive-potential'];
+    }
+
+    constructor() {}
+
+    /**
+     * @param {string} dateStr The date to run the analysis for (e.g., "2025-11-06").
+     * @param {object} dependencies The shared dependencies (db, logger).
+     * @param {object} config The computation system configuration.
+     * @param {object} fetchedDependencies In-memory results from Pass 3.
+     * @returns {Promise<object|null>} The analysis result or null.
+     */
+    async process(dateStr, dependencies, config, fetchedDependencies) {
+        const { db, logger } = dependencies;
+        
+        // 1. Get Pass 3 Dependency
+        const pass3_output = fetchedDependencies['social-topic-predictive-potential'];
+        if (!pass3_output || !pass3_output.daily_topic_signals) {
+            logger.log('WARN', `[SocialRegime] Missing or empty dependency 'social-topic-predictive-potential' for ${dateStr}. Skipping.`);
+            return null;
+        }
+        const daily_signals = pass3_output.daily_topic_signals;
+
+        // 2. Calculate Today's "Predictive Breadth"
+        let totalTopicsAnalyzed = 0;
+        let totalPredictiveTopics = 0;
+
+        for (const ticker in daily_signals) {
+            const allTopics = [
+                ...(daily_signals[ticker].topPredictiveBullishTopics || []),
+                ...(daily_signals[ticker].topPredictiveBearishTopics || [])
+            ];
+            
+            for (const topic of allTopics) {
+                totalTopicsAnalyzed++;
+                // Check if this topic is "predictive"
+                if (topic.predictivePotential > PP_THRESHOLD) {
+                    // Check if it has at least one stable window
+                    const hasStableWindow = Object.values(topic.correlations).some(
+                        corr => corr.samples >= MIN_SAMPLE_COUNT && Math.abs(corr.value) > CORR_THRESHOLD
+                    );
+                    if (hasStableWindow) {
+                        totalPredictiveTopics++;
+                    }
+                }
+            }
+        }
+        
+        const todayBreadth = (totalTopicsAnalyzed > 0)
+            ? (totalPredictiveTopics / totalTopicsAnalyzed)
+            : 0; // 0% breadth if no topics found
+
+        // 3. Load State, Update, and Save (all in one transaction)
+        
+        let dailyOutput = {}; // The daily signal for the API
+        let stateToSave = {}; // The new state to save back to Firestore
+
+        try {
+            await db.runTransaction(async (transaction) => {
+                const docRef = db.doc(STATE_DOC_PATH);
+                const doc = await transaction.get(docRef);
+                
+                // 4a. Load and update history
+                const history = doc.exists ? (doc.data().history || []) : [];
+                history.push({ date: dateStr, breadth: todayBreadth });
+                
+                // 4b. Prune history
+                const prunedHistory = history.slice(-ROLLING_HISTORY_DAYS);
+                
+                // 4c. Calculate EMAs
+                const breadthValues = prunedHistory.map(h => h.breadth);
+                const emaShort = _calculateEMA(breadthValues, EMA_SHORT_PERIOD);
+                const emaLong = _calculateEMA(breadthValues, EMA_LONG_PERIOD);
+
+                // 4d. Determine Regime State
+                let regimeState = "TRANSITIONING";
+                if (emaShort !== null && emaLong !== null) {
+                    const diff = emaShort - emaLong;
+                    if (diff > REGIME_THRESHOLD) {
+                        regimeState = "ON";
+                    } else if (diff < -REGIME_THRESHOLD) {
+                        regimeState = "OFF";
+                    }
+                }
+                
+                // 4e. Prepare data to save and return
+                stateToSave = { 
+                    history: prunedHistory,
+                    lastUpdated: FieldValue.serverTimestamp()
+                };
+                
+                dailyOutput = {
+                    social_predictive_regime_strength: emaShort, // The "fast" signal
+                    regime_state: regimeState,
+                    components: {
+                        today_breadth_pct: todayBreadth,
+                        ema_short: emaShort,
+                        ema_long: emaLong,
+                        threshold: REGIME_THRESHOLD
+                    }
+                };
+                
+                // 4f. Save state back to Firestore
+                transaction.set(docRef, stateToSave);
+            });
+            
+        } catch (error) {
+            logger.log('ERROR', `[SocialRegime] Transaction failed for ${dateStr}`, { err: error.message });
+            return null; // Don't return partial data
+        }
+
+        // 5. Return both state (for sharding) and daily signal (for API)
+        return {
+            // This key name must be unique
+            sharded_regime_state: { [STATE_DOC_PATH]: stateToSave }, 
+            // This is the daily result for unified_insights
+            daily_regime_signal: dailyOutput
+        };
+    }
+
+    async getResult() { return null; } // Logic is in process()
+    reset() {}
+}
+
+module.exports = SocialPredictiveRegimeState;

package/calculations/meta/social-topic-driver-index.js

@@ -0,0 +1,153 @@
+/**
+ * @fileoverview NEW CALCULATION (PASS 2 - META)
+ * This is the advanced signal calculation. It consumes the
+ * 'social-topic-sentiment-matrix' from Pass 1, loads the
+ * daily price change for each asset, and then determines
+ * the primary bullish and bearish topics driving the conversation.
+ */
+
+// We need calculation utils to load price and mapping data
+const { loadAllPriceData, getDailyPriceChange } = require('../../utils/price_data_provider');
+const { loadInstrumentMappings } = require('../../utils/sector_mapping_provider');
+
+class SocialTopicDriverIndex {
+
+    /**
+     * Statically declare dependencies.
+     * This will run in Pass 2, after the matrix is built.
+     */
+    static getDependencies() {
+        return ['social-topic-sentiment-matrix'];
+    }
+
+    constructor() {
+        this.priceMap = null;
+        this.tickerToIdMap = null; // For { "AAPL" -> 123 }
+        this.dependenciesLoaded = false;
+    }
+
+    /**
+     * Helper to load price/mapping data on the first run.
+     */
+    async _loadDependencies(calculationUtils) {
+        if (this.dependenciesLoaded) return;
+        
+        const [priceData, mappings] = await Promise.all([
+            calculationUtils.loadAllPriceData(),
+            calculationUtils.loadInstrumentMappings()
+        ]);
+        
+        this.priceMap = priceData;
+        
+        // Create the Ticker -> InstrumentID map
+        this.tickerToIdMap = {};
+        if (mappings && mappings.instrumentToTicker) {
+            for (const [id, ticker] of Object.entries(mappings.instrumentToTicker)) {
+                this.tickerToIdMap[ticker] = id;
+            }
+        }
+        
+        this.dependenciesLoaded = true;
+    }
+
+    /**
+     * Helper to get the previous day's date string.
+     */
+    _getYesterday(dateStr) {
+        const date = new Date(dateStr + 'T00:00:00Z');
+        date.setUTCDate(date.getUTCDate() - 1);
+        return date.toISOString().slice(0, 10);
+    }
+
+    /**
+     * @param {string} dateStr The date to run the analysis for (e.g., "2025-11-05").
+     * @param {object} dependencies The shared dependencies (db, logger, calculationUtils).
+     * @param {object} config The computation system configuration.
+     * @param {object} fetchedDependencies In-memory results from Pass 1.
+     * e.g., { 'social-topic-sentiment-matrix': { "AAPL": { ... } } }
+     * @returns {Promise<object|null>} The analysis result or null.
+     */
+    async process(dateStr, dependencies, config, fetchedDependencies) {
+        const { logger, calculationUtils } = dependencies;
+
+        // 1. Load dependencies
+        await this._loadDependencies(calculationUtils);
+        const matrix = fetchedDependencies['social-topic-sentiment-matrix'];
+
+        if (!matrix || Object.keys(matrix).length === 0) {
+            logger.log('WARN', `[SocialTopicDriverIndex] Missing or empty dependency 'social-topic-sentiment-matrix' for ${dateStr}. Skipping.`);
+            return null;
+        }
+
+        if (!this.priceMap || !this.tickerToIdMap) {
+            logger.log('ERROR', `[SocialTopicDriverIndex] Price map or Ticker map failed to load. Aborting.`);
+            return null;
+        }
+
+        const yesterdayStr = this._getYesterday(dateStr); // e.g., "2025-11-04"
+        const finalResults = {};
+
+        // 2. Correlate for each ticker
+        for (const ticker in matrix) {
+            const topicData = matrix[ticker];
+            const instrumentId = this.tickerToIdMap[ticker];
+
+            if (!instrumentId) {
+                logger.log('TRACE', `[SocialTopicDriverIndex] Skipping ${ticker}, no instrumentId found.`);
+                continue;
+            }
+
+            // 3. Get Price Change
+            // This correctly uses the resilient _findPreviousAvailablePrice helper
+            const priceChangePct = getDailyPriceChange(instrumentId, yesterdayStr, dateStr, this.priceMap);
+
+            if (priceChangePct === null) {
+                logger.log('TRACE', `[SocialTopicDriverIndex] Skipping ${ticker}, no price data found for ${dateStr}.`);
+                continue; // Skip if we can't get price data
+            }
+
+            // 4. Analyze topics for this ticker
+            const topicDrivers = [];
+            for (const topic in topicData) {
+                const data = topicData[topic];
+                const totalSentientPosts = data.bullishPosts + data.bearishPosts;
+                
+                let sentimentScore = 0;
+                if (totalSentientPosts > 0) {
+                    sentimentScore = (data.bullishPosts - data.bearishPosts) / totalSentientPosts;
+                }
+
+                topicDrivers.push({
+                    topic: topic,
+                    sentimentScore: sentimentScore, // -1 (Bearish) to +1 (Bullish)
+                    convictionScore: data.convictionScore,
+                    totalPosts: data.totalPosts,
+                    bullishPosts: data.bullishPosts,
+                    bearishPosts: data.bearishPosts
+                });
+            }
+
+            // 5. Find top drivers by conviction
+            // Sort by most conviction (likes/comments)
+            topicDrivers.sort((a, b) => b.convictionScore - a.convictionScore);
+            
+            // Find the most-discussed bullish and bearish topics
+            const topBullishDriver = topicDrivers.find(d => d.sentimentScore > 0.1) || null;
+            const topBearishDriver = topicDrivers.find(d => d.sentimentScore < -0.1) || null;
+
+            finalResults[ticker] = {
+                priceChangePct: priceChangePct,
+                topBullishDriver: topBullishDriver,
+                topBearishDriver: topBearishDriver,
+                allDrivers: topicDrivers // Store all for potential future analysis
+            };
+        }
+
+        return finalResults;
+    }
+
+    async getResult() { return null; } // This is a meta-calc, logic is in process()
+    reset() {}
+}
+
+module.exports = SocialTopicDriverIndex;

package/calculations/meta/social-topic-predictive-potential.js

@@ -0,0 +1,455 @@
+/**
+ * @fileoverview NEW CALCULATION (PASS 3 - META)
+ * This is the "Quant" signal discovery engine.
+ *
+ * This is a stateful, rolling calculation that implements several
+ * critical optimizations based on production-level review:
+ * 1. It uses `p-limit` to run concurrent transactions safely.
+ * 2. It returns state directly from transactions, avoiding costly re-reads.
+ * 3. It uses a forward-looking price helper (`_findPriceForward`)
+ * to be resilient to market holidays and missing data.
+ * 4. All logic is factored into testable helper functions.
+ *
+ * --- V3 MODIFICATION (Quant Upgrade) ---
+ * 5. Implements RECENCY WEIGHTING on the correlation calculation.
+ * Signals from recent days are given exponentially more weight
+ * than signals from 90 days ago, per standard quant practice.
+ * This is achieved by upgrading _calculatePearson to
+ * _calculateWeightedPearson.
+ */
+
+const { FieldValue } = require('@google-cloud/firestore');
+// p-limit is a non-standard-lib, but is included in the bulltrackers-module
+// and available via the root `index.js` dependency injection.
+// We will assume it's passed in via dependencies.calculationUtils.pLimit
+const pLimit = require('p-limit'); 
+
+// Import all required utils
+const { loadAllPriceData, getDailyPriceChange } = require('../../utils/price_data_provider');
+const { loadInstrumentMappings } = require('../../utils/sector_mapping_provider');
+
+// --- CONFIGURATION ---
+const SHARD_COLLECTION_NAME = 'social_topic_rolling_stats';
+const ROLLING_HISTORY_DAYS = 90;
+const MIN_SAMPLE_COUNT = 12; // Min samples needed to trust a correlation
+const CORR_THRESHOLD = 0.25; // Min abs correlation to be considered "stable"
+const FORWARD_WINDOWS = [1, 3, 7, 21]; // [1d, 3d, 7d, 21d]
+const MAX_CONCURRENT_TRANSACTIONS = 50; // Max parallel Firestore transactions
+const MAX_LOOKAHEAD_DAYS = 7; // How far to look for a non-holiday price
+const WEIGHTING_DECAY_K = 3.0; // Decay factor for recency weighting (e.g., k=3.0)
+
+// --- STATS HELPER ---
+/**
+ * --- MODIFIED: Upgraded to Weighted Pearson Correlation ---
+ * Calculates the Pearson correlation coefficient for two arrays,
+ * weighted by a third array.
+ * Gracefully handles nulls by only using paired data.
+ */
+function _calculateWeightedPearson(vecX, vecY, vecWeights) {
+    let sumW = 0;
+    let sumWX = 0;
+    let sumWY = 0;
+    let validPairs = []; // To store non-null pairs
+
+    for (let i = 0; i < vecX.length; i++) {
+        const x = vecX[i];
+        const y = vecY[i];
+        const w = vecWeights[i]; // Get the weight
+
+        // Only use pairs where both values are valid numbers and weight is positive
+        if (x !== null && y !== null && isFinite(x) && isFinite(y) && w > 0) {
+            sumW += w;
+            sumWX += w * x;
+            sumWY += w * y;
+            validPairs.push({x, y, w});
+        }
+    }
+
+    // Need at least 2 data points and positive total weight
+    if (sumW === 0 || validPairs.length < 2) {
+        return { value: 0, samples: validPairs.length };
+    }
+
+    // Calculate weighted means
+    const meanX = sumWX / sumW;
+    const meanY = sumWY / sumW;
+
+    let sumCov = 0;
+    let sumVarX = 0;
+    let sumVarY = 0;
+
+    // Second pass to calculate weighted covariance and variances
+    for (const pair of validPairs) {
+        const { x, y, w } = pair;
+        sumCov += w * (x - meanX) * (y - meanY);
+        sumVarX += w * (x - meanX) * (x - meanX);
+        sumVarY += w * (y - meanY) * (y - meanY);
+    }
+
+    const denominator = Math.sqrt(sumVarX * sumVarY);
+
+    if (denominator === 0) {
+        return { value: 0, samples: validPairs.length };
+    }
+
+    // The weighted correlation is cov(x,y) / (std(x) * std(y))
+    // The sumW terms cancel out from the numerator and denominator.
+    const corr = sumCov / denominator;
+    
+    // Clamp correlation to valid range [-1, 1] to handle floating point errors
+    return { value: Math.max(-1, Math.min(1, corr)), samples: validPairs.length };
+}
+
+// --- DATE HELPERS ---
+function _getDateStr(baseDate, daysOffset) {
+    const date = new Date(baseDate); // baseDate is Date object or string
+    date.setUTCDate(date.getUTCDate() + daysOffset);
+    return date.toISOString().slice(0, 10);
+}
+
+// --- PRICE HELPERS ---
+/**
+ * Resiliently finds a price for a given date from the price map.
+ * (Looks *backward* for last available price)
+ */
+function _findPrice(instrumentId, dateStr, priceMap) {
+    if (!priceMap || !priceMap[instrumentId]) return null;
+    const priceHistory = priceMap[instrumentId];
+    
+    let checkDate = new Date(dateStr + 'T00:00:00Z');
+    
+    for (let i = 0; i < MAX_LOOKAHEAD_DAYS; i++) {
+        const checkDateStr = checkDate.toISOString().slice(0, 10);
+        const price = priceHistory[checkDateStr];
+        
+        if (price !== undefined && price !== null && price > 0) return price;
+        checkDate.setUTCDate(checkDate.getUTCDate() - 1);
+    }
+    return null;
+}
+
+/**
+ * NEW: Resiliently finds the next available price *on or after* a given date.
+ * (Looks *forward* to handle holidays/weekends)
+ */
+function _findPriceForward(instrumentId, dateStr, priceMap) {
+    if (!priceMap || !priceMap[instrumentId]) return null;
+    const priceHistory = priceMap[instrumentId];
+    
+    let checkDate = new Date(dateStr + 'T00:00:00Z');
+    
+    for (let i = 0; i <= MAX_LOOKAHEAD_DAYS; i++) {
+        const checkDateStr = checkDate.toISOString().slice(0, 10);
+        const price = priceHistory[checkDateStr];
+        
+        if (price !== undefined && price !== null && price > 0) return price;
+        checkDate.setUTCDate(checkDate.getUTCDate() + 1);
+    }
+    return null;
+}
+
+
+class SocialTopicPredictivePotentialIndex {
+
+    static getDependencies() {
+        return ['social-topic-driver-index'];
+    }
+
+    constructor() {
+        this.priceMap = null;
+        this.tickerToIdMap = null;
+        this.dependenciesLoaded = false;
+        // Concurrency limiter for Firestore transactions
+        this.pLimit = pLimit(MAX_CONCURRENT_TRANSACTIONS);
+    }
+
+    async _loadDependencies(calculationUtils) {
+        if (this.dependenciesLoaded) return;
+        
+        const [priceData, mappings] = await Promise.all([
+            calculationUtils.loadAllPriceData(),
+            calculationUtils.loadInstrumentMappings()
+        ]);
+        
+        this.priceMap = priceData;
+        this.tickerToIdMap = {};
+        if (mappings && mappings.instrumentToTicker) {
+            for (const [id, ticker] of Object.entries(mappings.instrumentToTicker)) {
+                this.tickerToIdMap[ticker] = id;
+            }
+        }
+        this.dependenciesLoaded = true;
+    }
+
+    /**
+     * @param {string} dateStr The date to run the analysis for (e.g., "2025-11-05").
+     * @param {object} dependencies The shared dependencies (db, logger, calculationUtils).
+     * @param {object} config The computation system configuration.
+     * @param {object} fetchedDependencies In-memory results from Pass 2.
+     * @returns {Promise<object|null>} The analysis result or null.
+     */
+    async process(dateStr, dependencies, config, fetchedDependencies) {
+        const { db, logger, calculationUtils } = dependencies;
+        
+        // 1. Load all dependencies
+        // pLimit is not in calculationUtils by default, so we'll use our own
+        // If it were, we'd use: this.pLimit = calculationUtils.pLimit(MAX_CONCURRENT_TRANSACTIONS);
+        await this._loadDependencies(calculationUtils);
+        const todaySignals = fetchedDependencies['social-topic-driver-index'];
+
+        if (!todaySignals || Object.keys(todaySignals).length === 0) {
+            logger.log('WARN', `[SocialTopicPredictive] Missing or empty dependency 'social-topic-driver-index' for ${dateStr}. Skipping.`);
+            return null;
+        }
+
+        if (!this.priceMap || !this.tickerToIdMap) {
+            logger.log('ERROR', `[SocialTopicPredictive] Price map or Ticker map failed to load. Aborting.`);
+            return null;
+        }
+
+        // --- Prepare final output objects ---
+        const shardedData = {};
+        const dailyOutput = {};
+        
+        const allTickers = Object.keys(todaySignals);
+
+        // 2. Run all ticker updates in parallel, limited by pLimit
+        const transactionPromises = allTickers.map(ticker => 
+            this.pLimit(() => this._processTickerTransaction(
+                ticker, 
+                dateStr, 
+                todaySignals[ticker] || {}, 
+                dependencies
+            ))
+        );
+        
+        const txResults = await Promise.all(transactionPromises);
+
+        // 3. Collect results from transactions (FIX A)
+        for (const res of txResults) {
+            if (!res) continue; // Transaction may have failed and returned null
+            shardedData[res.ticker] = res.state;
+            dailyOutput[res.ticker] = res.dailyOutputForTicker;
+        }
+
+        // 4. Return both the state (for sharded write) and daily signal (for API)
+        return {
+            sharded_social_stats: { [SHARD_COLLECTION_NAME]: shardedData },
+            daily_topic_signals: dailyOutput
+        };
+    }
+
+    /**
+     * This is the core logic, run for a single ticker *inside* a transaction.
+     * It reads, modifies, and writes state for one ticker, then returns
+     * the new state and daily signal.
+     */
+    async _processTickerTransaction(ticker, dateStr, todaySignal, dependencies) {
+        const { db, logger } = dependencies;
+        
+        try {
+            return await db.runTransaction(async (transaction) => {
+                const instrumentId = this.tickerToIdMap[ticker];
+                if (!instrumentId) return null;
+
+                const todayPrice = _findPrice(instrumentId, dateStr, this.priceMap);
+                if (todayPrice === null) return null; // Cannot update returns
+
+                // --- 4a. Load State ---
+                const docRef = db.collection(SHARD_COLLECTION_NAME).doc(ticker);
+                const doc = await transaction.get(docRef);
+                const state = doc.exists ? doc.data() : {}; 
+
+                // --- 4b. Update Forward Returns (Factored Helper) ---
+                this._updateForwardReturns(state, instrumentId, dateStr, todayPrice, this.priceMap);
+
+                // --- 4c. Add New Signals (Factored Helper) ---
+                this._addNewSignals(state, todaySignal.allDrivers || [], dateStr);
+
+                // --- 4d. Recalculate Correlations (Factored Helper) ---
+                const dailyOutputForTicker = this._recalculateAllTopics(state, logger);
+
+                // --- 4e. Save State (FIX 3.2: Use merge) ---
+                transaction.set(docRef, state, { merge: true });
+
+                // --- 4f. Return (FIX A) ---
+                return { ticker, state, dailyOutputForTicker };
+            });
+        } catch (error) {
+            logger.log('ERROR', `[SocialTopicPredictive] Transaction failed for ticker ${ticker}`, { err: error.message, stack: error.stack });
+            return null; // Return null on transaction failure
+        }
+    }
+
+    /**
+     * Helper to update past forward-return windows.
+     * Modifies the `state` object in-place.
+     */
+    _updateForwardReturns(state, instrumentId, dateStr, todayPrice, priceMap) {
+        for (const topic in state) {
+            if (!state[topic].rolling_90d_history) continue;
+            
+            for (const historyEntry of state[topic].rolling_90d_history) {
+                const signalPrice = _findPrice(instrumentId, historyEntry.date, priceMap);
+                if (signalPrice === null) continue;
+
+                // FIX D: Check all windows
+                for (const window of FORWARD_WINDOWS) {
+                    // Skip if already filled
+                    if (historyEntry[`fwd_${window}d`] !== null) continue; 
+
+                    const targetDateStr = _getDateStr(new Date(historyEntry.date + 'T00:00:00Z'), window);
+                    
+                    // Not yet time to fill this window
+                    if (dateStr < targetDateStr) continue; 
+                    
+                    // FIX C: Use _findPriceForward
+                    const targetPrice = _findPriceForward(instrumentId, targetDateStr, priceMap);
+                    
+                    if (targetPrice !== null) {
+                        historyEntry[`fwd_${window}d`] = (targetPrice / signalPrice) - 1;
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Helper to add the new day's signals to the state.
+     * Modifies the `state` object in-place.
+     */
+    _addNewSignals(state, newTickerSignals, dateStr) {
+        for (const newSignal of newTickerSignals) {
+            // FIX B: Normalize topic key
+            const topic = (newSignal.topic || 'untagged').toLowerCase();
+            
+            if (!state[topic]) {
+                state[topic] = { 
+                    rolling_90d_history: [],
+                    correlations: {},
+                    predictivePotential: 0,
+                    avgDailyConviction: 0
+                };
+            }
+
+            state[topic].rolling_90d_history.push({
+                date: dateStr,
+                sentimentScore: newSignal.sentimentScore,
+                conviction: newSignal.convictionScore,
+                fwd_1d: null, fwd_3d: null, fwd_7d: null, fwd_21d: null,
+            });
+
+            // Prune history
+            state[topic].rolling_90d_history = state[topic].rolling_90d_history.slice(-ROLLING_HISTORY_DAYS);
+        }
+    }
+
+    /**
+     * Helper to recalculate all stats for a ticker's topics.
+     * Modifies the `state` object in-place.
+     * @returns {object} The dailyOutputForTicker
+     */
+    _recalculateAllTopics(state, logger) {
+        const predictiveTopics = [];
+
+        for (const topic in state) {
+            const history = state[topic].rolling_90d_history;
+            if (!history || history.length === 0) continue;
+
+            // FIX G: Warn if doc size is at risk
+            if (history.length > 500) { // arbitrary high number
+                logger.log('WARN', `[SocialTopicPredictive] Topic "${topic}" has ${history.length} history entries. May approach doc size limit.`);
+            }
+
+            // FIX E: Ensure sort order
+            history.sort((a, b) => new Date(a.date) - new Date(b.date));
+            
+            // --- START V3 MODIFICATION: RECENCY WEIGHTING ---
+            const N = history.length;
+            if (N === 0) continue;
+
+            // Create recency weights (exponential decay)
+            // Newest item (i=N-1) gets weight 1.0 (age=0)
+            // Oldest item (i=0) gets weight exp(-K) (age=N-1)
+            const vecWeights = history.map((h, i) => {
+                const age_in_days = N - 1 - i; // 0 for newest, N-1 for oldest
+                // Normalize age by history length so K is consistent
+                return Math.exp(-WEIGHTING_DECAY_K * age_in_days / N);
+            });
+            // --- END V3 MODIFICATION ---
+
+            const vecSentiment = history.map(h => h.sentimentScore);
+            const vecConviction = history.map(h => h.conviction); // Used for PP score
+
+            let totalPP = 0;
+            let windowsCounted = 0;
+            let totalStableWindows = 0;
+
+            for (const window of FORWARD_WINDOWS) {
+                const vecForwardReturn = history.map(h => h[`fwd_${window}d`]);
+                
+                // --- V3 MODIFICATION ---
+                // Use the new weighted Pearson calculation
+                const corr = _calculateWeightedPearson(vecSentiment, vecForwardReturn, vecWeights);
+                // --- END V3 MODIFICATION ---
+                
+                state[topic].correlations[`fwd_${window}d`] = corr; // Save { value, samples }
+
+                if (corr.samples >= MIN_SAMPLE_COUNT) {
+                    totalPP += Math.abs(corr.value);
+                    windowsCounted++;
+                    if (Math.abs(corr.value) > CORR_THRESHOLD) {
+                        totalStableWindows++;
+                    }
+                }
+            }
+            
+            let ppScore = 0;
+            if (windowsCounted > 0) {
+                ppScore = totalPP / windowsCounted; 
+                
+                const avgConviction = state[topic].rolling_90d_history.reduce((acc, h) => acc + h.conviction, 0) / history.length;
+                state[topic].avgDailyConviction = avgConviction;
+                
+                // FIX F: Clamp before log
+                ppScore *= Math.log(1 + Math.max(0, avgConviction));
+                
+                ppScore *= (totalStableWindows / windowsCounted);
+            }
+            
+            state[topic].predictivePotential = ppScore;
+
+            predictiveTopics.push({
+                topic: topic,
+                predictivePotential: ppScore,
+                sentimentScore: vecSentiment[vecSentiment.length - 1], // Latest sentiment
+                correlations: state[topic].correlations
+            });
+        }
+
+        // --- Generate Daily Output ---
+        predictiveTopics.sort((a, b) => b.predictivePotential - a.predictivePotential);
+        const confidence = predictiveTopics.length > 0 ? predictiveTopics[0].predictivePotential : 0;
+        const normalizedConfidence = Math.min(1, confidence / 10); // Simple normalization
+
+        return {
+            topPredictiveBullishTopics: predictiveTopics
+                .filter(t => t.sentimentScore > 0.1)
+                .slice(0, 5),
+            topPredictiveBearishTopics: predictiveTopics
+                .filter(t => t.sentimentScore < -0.1)
+                .slice(0, 5),
+            predictiveConfidenceScore: normalizedConfidence
+        };
+    }
+
+    async getResult() { return null; } // Logic is in process()
+    reset() {
+        this.priceMap = null;
+        this.tickerToIdMap = null;
+        this.dependenciesLoaded = false;
+        this.pLimit = pLimit(MAX_CONCURRENT_TRANSACTIONS);
+    }
+}
+
+module.exports = SocialTopicPredictivePotentialIndex;

package/calculations/pnl/pnl_distribution_per_stock.js

@@ -1,91 +1,66 @@
 /**
- * @fileoverview Meta-calculation (Pass 2) to calculate a proxy for the
- * crowd's "Sharpe Ratio" (risk-adjusted return) on a per-asset basis.
- *
- * --- META REFACTOR (v2) ---
- * This calculation is now stateless. It declares its dependencies and
- * expects them to be passed to its `process` method.
+ * @fileoverview Aggregates P&L data points for statistical analysis.
+ * This calculation is a dependency for the 'Crowd Sharpe Ratio Proxy'.
+ * It gathers the sum, sum of squares, and count of P&L for each stock.
  */
+const { loadInstrumentMappings } = require('../../utils/sector_mapping_provider');
 
-class CrowdSharpeRatioProxy {
-    
-    /**
-     * (NEW) Statically declare dependencies.
-     */
-    static getDependencies() {
-        return ['pnl-distribution-per-stock'];
+class PnlDistributionPerStock {
+    constructor() {
+        this.distributionData = {}; // { [instrumentId]: { pnl_sum: 0, pnl_sum_sq: 0, position_count: 0 } }
+        this.mappings = null;
     }
 
-    constructor() {}
-
-    /**
-     * REFACTORED PROCESS METHOD
-     * @param {string} dateStr The date to run the analysis for (e.g., "2025-10-31").
-     * @param {object} dependencies The shared dependencies (db, logger).
-     * @param {object} config The computation system configuration.
-     * @param {object} fetchedDependencies In-memory results from previous passes.
-     * e.g., { 'pnl-distribution-per-stock': ... }
-     * @returns {Promise<object|null>} The analysis result or null.
-     */
-    async process(dateStr, dependencies, config, fetchedDependencies) {
-        const { logger } = dependencies;
-
-        // 1. Get dependency from in-memory cache
-        const data = fetchedDependencies['pnl-distribution-per-stock'];
-
-        // 2. Handle missing dependency
-        if (!data) {
-            logger.log('WARN', `[CrowdSharpeRatioProxy] Missing dependency 'pnl-distribution-per-stock' for ${dateStr}. Skipping.`);
-            return null;
+    process(portfolioData, yesterdayPortfolio, userId, context) {
+        const positions = portfolioData.AggregatedPositions || portfolioData.PublicPositions;
+        if (!positions) {
+            return;
         }
 
-        const pnlDistribution = data.pnl_distribution_by_asset;
-
-        if (!pnlDistribution) {
-             logger.log('WARN', `[CrowdSharpeRatioProxy] Dependency data for ${dateStr} is empty. Skipping.`);
-             return null;
-        }
-
-        const results = {};
-
-        // 3. Calculate Sharpe Proxy for each asset
-        for (const ticker in pnlDistribution) {
-            const stats = pnlDistribution[ticker];
-            const N = stats.position_count;
-            
-            if (N < 2) continue; // Need at least 2 data points for variance
-
-            const mean = stats.pnl_sum / N; // E(x)
-            const mean_sq = stats.pnl_sum_sq / N; // E(x^2)
-            
-            const variance = mean_sq - (mean * mean);
-
-            if (variance <= 0) {
-                results[ticker] = {
-                    average_pnl: mean,
-                    std_dev_pnl: 0,
-                    sharpe_ratio_proxy: 0,
-                    position_count: N
-                };
-                continue;
+        for (const position of positions) {
+            const instrumentId = position.InstrumentID;
+            // Use NetProfit (which is the P&L value)
+            const netProfit = position.NetProfit; 
+
+            // Ensure netProfit is a valid number
+            if (instrumentId && typeof netProfit === 'number' && isFinite(netProfit)) {
+                if (!this.distributionData[instrumentId]) {
+                    this.distributionData[instrumentId] = {
+                        pnl_sum: 0,
+                        pnl_sum_sq: 0, // Sum of squares
+                        position_count: 0
+                    };
+                }
+
+                this.distributionData[instrumentId].pnl_sum += netProfit;
+                this.distributionData[instrumentId].pnl_sum_sq += (netProfit * netProfit); // Add the square
+                this.distributionData[instrumentId].position_count++;
             }
+        }
+    }
 
-            const std_dev = Math.sqrt(variance); // "Risk"
-            const sharpe_proxy = mean / std_dev; // Return / Risk
+    async getResult() {
+        if (!this.mappings) {
+            this.mappings = await loadInstrumentMappings();
+        }
 
-            results[ticker] = {
-                average_pnl: mean,
-                std_dev_pnl: std_dev,
-                sharpe_ratio_proxy: sharpe_proxy,
-                position_count: N
-            };
+        const result = {};
+        for (const instrumentId in this.distributionData) {
+            const ticker = this.mappings.instrumentToTicker[instrumentId] || instrumentId.toString();
+            result[ticker] = this.distributionData[instrumentId];
         }
 
-        return results;
+        if (Object.keys(result).length === 0) return {};
+        
+        return { 
+            pnl_distribution_by_asset: result 
+        };
     }
 
-    async getResult() { return null; }
-    reset() {}
+    reset() {
+        this.distributionData = {};
+        this.mappings = null;
+    }
 }
 
-module.exports = CrowdSharpeRatioProxy;
+module.exports = PnlDistributionPerStock;

package/calculations/socialPosts/social-topic-sentiment-matrix.js

@@ -0,0 +1,105 @@
+/**
+ * @fileoverview NEW CALCULATION (PASS 1)
+ * Aggregates all social post data for the day into a detailed matrix.
+ * For each ticker, it maps each discussed topic to its aggregated
+ * sentiment (Bullish/Bearish/Neutral) and a "Conviction Score"
+ * derived from likes and comments.
+ *
+ * This is a "socialPosts" category calculation, so it runs once per day.
+ */
+
+class SocialTopicSentimentMatrix {
+    constructor() {
+        // The main data structure.
+        // Format: { [ticker]: { [topic]: { ...stats } } }
+        this.matrix = {};
+        
+        // Flag to ensure this runs only once per day
+        this.processed = false;
+    }
+
+    /**
+     * @param {null} todayPortfolio - Not used.
+     * @param {null} yesterdayPortfolio - Not used.
+     * @param {null} userId - Not used.
+     * @param {object} context - Shared context.
+     * @param {object} todayInsights - Not used.
+     * @param {object} yesterdayInsights - Not used.
+     * @param {object} todaySocialPostInsights - Map of { [postId]: postData } for today.
+     * @param {object} yesterdaySocialPostInsights - Not used.
+     */
+    async process(todayPortfolio, yesterdayPortfolio, userId, context, todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights) {
+        
+        // Run only once per day
+        if (this.processed || !todaySocialPostInsights) {
+            return;
+        }
+        this.processed = true;
+
+        const posts = Object.values(todaySocialPostInsights);
+
+        for (const post of posts) {
+            if (!post || !post.tickers || !post.sentiment) continue;
+
+            const tickers = post.tickers;
+            const sentiment = post.sentiment.overallSentiment || 'Neutral';
+            
+            // Use a default topic if Gemini found none
+            let topics = post.sentiment.topics || [];
+            if (topics.length === 0) {
+                topics.push('untagged');
+            }
+
+            // Calculate conviction (e.g., comments are 2x as valuable as likes)
+            const convictionScore = (post.likeCount || 0) + ((post.commentCount || 0) * 2);
+
+            for (const ticker of tickers) {
+                if (!ticker) continue;
+                
+                for (const topic of topics) {
+                    const topicLower = topic.toLowerCase();
+
+                    // Initialize ticker map
+                    if (!this.matrix[ticker]) {
+                        this.matrix[ticker] = {};
+                    }
+                    // Initialize topic map
+                    if (!this.matrix[ticker][topicLower]) {
+                        this.matrix[ticker][topicLower] = {
+                            bullishPosts: 0,
+                            bearishPosts: 0,
+                            neutralPosts: 0,
+                            totalPosts: 0,
+                            convictionScore: 0
+                        };
+                    }
+
+                    // Aggregate data into the bucket
+                    const bucket = this.matrix[ticker][topicLower];
+                    bucket.totalPosts++;
+                    bucket.convictionScore += convictionScore;
+
+                    if (sentiment === 'Bullish') {
+                        bucket.bullishPosts++;
+                    } else if (sentiment === 'Bearish') {
+                        bucket.bearishPosts++;
+                    } else {
+                        bucket.neutralPosts++;
+                    }
+                }
+            }
+        }
+    }
+
+    async getResult() {
+        // This entire matrix will be the result
+        return this.matrix;
+    }
+
+    reset() {
+        this.matrix = {};
+        this.processed = false;
+    }
+}
+
+module.exports = SocialTopicSentimentMatrix;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "aiden-shared-calculations-unified",
-  "version": "1.0.45",
+  "version": "1.0.47",
   "description": "Shared calculation modules for the BullTrackers Computation System.",
   "main": "index.js",
   "files": [