aiden-shared-calculations-unified 1.0.46 → 1.0.48

package/calculations/meta/social-topic-predictive-potential.js

@@ -0,0 +1,455 @@
+/**
+ * @fileoverview NEW CALCULATION (PASS 3 - META)
+ * This is the "Quant" signal discovery engine.
+ *
+ * This is a stateful, rolling calculation that implements several
+ * critical optimizations based on production-level review:
+ * 1. It uses `p-limit` to run concurrent transactions safely.
+ * 2. It returns state directly from transactions, avoiding costly re-reads.
+ * 3. It uses a forward-looking price helper (`_findPriceForward`)
+ * to be resilient to market holidays and missing data.
+ * 4. All logic is factored into testable helper functions.
+ *
+ * --- V3 MODIFICATION (Quant Upgrade) ---
+ * 5. Implements RECENCY WEIGHTING on the correlation calculation.
+ * Signals from recent days are given exponentially more weight
+ * than signals from 90 days ago, per standard quant practice.
+ * This is achieved by upgrading _calculatePearson to
+ * _calculateWeightedPearson.
+ */
+
+const { FieldValue } = require('@google-cloud/firestore');
+// p-limit is a non-standard-lib, but is included in the bulltrackers-module
+// and available via the root `index.js` dependency injection.
+// We will assume it's passed in via dependencies.calculationUtils.pLimit
+const pLimit = require('p-limit'); 
+
+// Import all required utils
+const { loadAllPriceData, getDailyPriceChange } = require('../../utils/price_data_provider');
+const { loadInstrumentMappings } = require('../../utils/sector_mapping_provider');
+
+// --- CONFIGURATION ---
+const SHARD_COLLECTION_NAME = 'social_topic_rolling_stats';
+const ROLLING_HISTORY_DAYS = 90;
+const MIN_SAMPLE_COUNT = 12; // Min samples needed to trust a correlation
+const CORR_THRESHOLD = 0.25; // Min abs correlation to be considered "stable"
+const FORWARD_WINDOWS = [1, 3, 7, 21]; // [1d, 3d, 7d, 21d]
+const MAX_CONCURRENT_TRANSACTIONS = 50; // Max parallel Firestore transactions
+const MAX_LOOKAHEAD_DAYS = 7; // How far to look for a non-holiday price
+const WEIGHTING_DECAY_K = 3.0; // Decay factor for recency weighting (e.g., k=3.0)
+
+// --- STATS HELPER ---
+/**
+ * --- MODIFIED: Upgraded to Weighted Pearson Correlation ---
+ * Calculates the Pearson correlation coefficient for two arrays,
+ * weighted by a third array.
+ * Gracefully handles nulls by only using paired data.
+ */
+function _calculateWeightedPearson(vecX, vecY, vecWeights) {
+    let sumW = 0;
+    let sumWX = 0;
+    let sumWY = 0;
+    let validPairs = []; // To store non-null pairs
+
+    for (let i = 0; i < vecX.length; i++) {
+        const x = vecX[i];
+        const y = vecY[i];
+        const w = vecWeights[i]; // Get the weight
+
+        // Only use pairs where both values are valid numbers and weight is positive
+        if (x !== null && y !== null && isFinite(x) && isFinite(y) && w > 0) {
+            sumW += w;
+            sumWX += w * x;
+            sumWY += w * y;
+            validPairs.push({x, y, w});
+        }
+    }
+
+    // Need at least 2 data points and positive total weight
+    if (sumW === 0 || validPairs.length < 2) {
+        return { value: 0, samples: validPairs.length };
+    }
+
+    // Calculate weighted means
+    const meanX = sumWX / sumW;
+    const meanY = sumWY / sumW;
+
+    let sumCov = 0;
+    let sumVarX = 0;
+    let sumVarY = 0;
+
+    // Second pass to calculate weighted covariance and variances
+    for (const pair of validPairs) {
+        const { x, y, w } = pair;
+        sumCov += w * (x - meanX) * (y - meanY);
+        sumVarX += w * (x - meanX) * (x - meanX);
+        sumVarY += w * (y - meanY) * (y - meanY);
+    }
+
+    const denominator = Math.sqrt(sumVarX * sumVarY);
+
+    if (denominator === 0) {
+        return { value: 0, samples: validPairs.length };
+    }
+
+    // The weighted correlation is cov(x,y) / (std(x) * std(y))
+    // The sumW terms cancel out from the numerator and denominator.
+    const corr = sumCov / denominator;
+    
+    // Clamp correlation to valid range [-1, 1] to handle floating point errors
+    return { value: Math.max(-1, Math.min(1, corr)), samples: validPairs.length };
+}
+
+// --- DATE HELPERS ---
+function _getDateStr(baseDate, daysOffset) {
+    const date = new Date(baseDate); // baseDate is Date object or string
+    date.setUTCDate(date.getUTCDate() + daysOffset);
+    return date.toISOString().slice(0, 10);
+}
+
+// --- PRICE HELPERS ---
+/**
+ * Resiliently finds a price for a given date from the price map.
+ * (Looks *backward* for last available price)
+ */
+function _findPrice(instrumentId, dateStr, priceMap) {
+    if (!priceMap || !priceMap[instrumentId]) return null;
+    const priceHistory = priceMap[instrumentId];
+    
+    let checkDate = new Date(dateStr + 'T00:00:00Z');
+    
+    for (let i = 0; i < MAX_LOOKAHEAD_DAYS; i++) {
+        const checkDateStr = checkDate.toISOString().slice(0, 10);
+        const price = priceHistory[checkDateStr];
+        
+        if (price !== undefined && price !== null && price > 0) return price;
+        checkDate.setUTCDate(checkDate.getUTCDate() - 1);
+    }
+    return null;
+}
+
+/**
+ * NEW: Resiliently finds the next available price *on or after* a given date.
+ * (Looks *forward* to handle holidays/weekends)
+ */
+function _findPriceForward(instrumentId, dateStr, priceMap) {
+    if (!priceMap || !priceMap[instrumentId]) return null;
+    const priceHistory = priceMap[instrumentId];
+    
+    let checkDate = new Date(dateStr + 'T00:00:00Z');
+    
+    for (let i = 0; i <= MAX_LOOKAHEAD_DAYS; i++) {
+        const checkDateStr = checkDate.toISOString().slice(0, 10);
+        const price = priceHistory[checkDateStr];
+        
+        if (price !== undefined && price !== null && price > 0) return price;
+        checkDate.setUTCDate(checkDate.getUTCDate() + 1);
+    }
+    return null;
+}
+
+
+class SocialTopicPredictivePotentialIndex {
+
+    static getDependencies() {
+        return ['social-topic-driver-index'];
+    }
+
+    constructor() {
+        this.priceMap = null;
+        this.tickerToIdMap = null;
+        this.dependenciesLoaded = false;
+        // Concurrency limiter for Firestore transactions
+        this.pLimit = pLimit(MAX_CONCURRENT_TRANSACTIONS);
+    }
+
+    async _loadDependencies(calculationUtils) {
+        if (this.dependenciesLoaded) return;
+        
+        const [priceData, mappings] = await Promise.all([
+            calculationUtils.loadAllPriceData(),
+            calculationUtils.loadInstrumentMappings()
+        ]);
+        
+        this.priceMap = priceData;
+        this.tickerToIdMap = {};
+        if (mappings && mappings.instrumentToTicker) {
+            for (const [id, ticker] of Object.entries(mappings.instrumentToTicker)) {
+                this.tickerToIdMap[ticker] = id;
+            }
+        }
+        this.dependenciesLoaded = true;
+    }
+
+    /**
+     * @param {string} dateStr The date to run the analysis for (e.g., "2025-11-05").
+     * @param {object} dependencies The shared dependencies (db, logger, calculationUtils).
+     * @param {object} config The computation system configuration.
+     * @param {object} fetchedDependencies In-memory results from Pass 2.
+     * @returns {Promise<object|null>} The analysis result or null.
+     */
+    async process(dateStr, dependencies, config, fetchedDependencies) {
+        const { db, logger, calculationUtils } = dependencies;
+        
+        // 1. Load all dependencies
+        // pLimit is not in calculationUtils by default, so we'll use our own
+        // If it were, we'd use: this.pLimit = calculationUtils.pLimit(MAX_CONCURRENT_TRANSACTIONS);
+        await this._loadDependencies(calculationUtils);
+        const todaySignals = fetchedDependencies['social-topic-driver-index'];
+
+        if (!todaySignals || Object.keys(todaySignals).length === 0) {
+            logger.log('WARN', `[SocialTopicPredictive] Missing or empty dependency 'social-topic-driver-index' for ${dateStr}. Skipping.`);
+            return null;
+        }
+
+        if (!this.priceMap || !this.tickerToIdMap) {
+            logger.log('ERROR', `[SocialTopicPredictive] Price map or Ticker map failed to load. Aborting.`);
+            return null;
+        }
+
+        // --- Prepare final output objects ---
+        const shardedData = {};
+        const dailyOutput = {};
+        
+        const allTickers = Object.keys(todaySignals);
+
+        // 2. Run all ticker updates in parallel, limited by pLimit
+        const transactionPromises = allTickers.map(ticker => 
+            this.pLimit(() => this._processTickerTransaction(
+                ticker, 
+                dateStr, 
+                todaySignals[ticker] || {}, 
+                dependencies
+            ))
+        );
+        
+        const txResults = await Promise.all(transactionPromises);
+
+        // 3. Collect results from transactions (FIX A)
+        for (const res of txResults) {
+            if (!res) continue; // Transaction may have failed and returned null
+            shardedData[res.ticker] = res.state;
+            dailyOutput[res.ticker] = res.dailyOutputForTicker;
+        }
+
+        // 4. Return both the state (for sharded write) and daily signal (for API)
+        return {
+            sharded_social_stats: { [SHARD_COLLECTION_NAME]: shardedData },
+            daily_topic_signals: dailyOutput
+        };
+    }
+
+    /**
+     * This is the core logic, run for a single ticker *inside* a transaction.
+     * It reads, modifies, and writes state for one ticker, then returns
+     * the new state and daily signal.
+     */
+    async _processTickerTransaction(ticker, dateStr, todaySignal, dependencies) {
+        const { db, logger } = dependencies;
+        
+        try {
+            return await db.runTransaction(async (transaction) => {
+                const instrumentId = this.tickerToIdMap[ticker];
+                if (!instrumentId) return null;
+
+                const todayPrice = _findPrice(instrumentId, dateStr, this.priceMap);
+                if (todayPrice === null) return null; // Cannot update returns
+
+                // --- 4a. Load State ---
+                const docRef = db.collection(SHARD_COLLECTION_NAME).doc(ticker);
+                const doc = await transaction.get(docRef);
+                const state = doc.exists ? doc.data() : {}; 
+
+                // --- 4b. Update Forward Returns (Factored Helper) ---
+                this._updateForwardReturns(state, instrumentId, dateStr, todayPrice, this.priceMap);
+
+                // --- 4c. Add New Signals (Factored Helper) ---
+                this._addNewSignals(state, todaySignal.allDrivers || [], dateStr);
+
+                // --- 4d. Recalculate Correlations (Factored Helper) ---
+                const dailyOutputForTicker = this._recalculateAllTopics(state, logger);
+
+                // --- 4e. Save State (FIX 3.2: Use merge) ---
+                transaction.set(docRef, state, { merge: true });
+
+                // --- 4f. Return (FIX A) ---
+                return { ticker, state, dailyOutputForTicker };
+            });
+        } catch (error) {
+            logger.log('ERROR', `[SocialTopicPredictive] Transaction failed for ticker ${ticker}`, { err: error.message, stack: error.stack });
+            return null; // Return null on transaction failure
+        }
+    }
+
+    /**
+     * Helper to update past forward-return windows.
+     * Modifies the `state` object in-place.
+     */
+    _updateForwardReturns(state, instrumentId, dateStr, todayPrice, priceMap) {
+        for (const topic in state) {
+            if (!state[topic].rolling_90d_history) continue;
+            
+            for (const historyEntry of state[topic].rolling_90d_history) {
+                const signalPrice = _findPrice(instrumentId, historyEntry.date, priceMap);
+                if (signalPrice === null) continue;
+
+                // FIX D: Check all windows
+                for (const window of FORWARD_WINDOWS) {
+                    // Skip if already filled
+                    if (historyEntry[`fwd_${window}d`] !== null) continue; 
+
+                    const targetDateStr = _getDateStr(new Date(historyEntry.date + 'T00:00:00Z'), window);
+                    
+                    // Not yet time to fill this window
+                    if (dateStr < targetDateStr) continue; 
+                    
+                    // FIX C: Use _findPriceForward
+                    const targetPrice = _findPriceForward(instrumentId, targetDateStr, priceMap);
+                    
+                    if (targetPrice !== null) {
+                        historyEntry[`fwd_${window}d`] = (targetPrice / signalPrice) - 1;
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Helper to add the new day's signals to the state.
+     * Modifies the `state` object in-place.
+     */
+    _addNewSignals(state, newTickerSignals, dateStr) {
+        for (const newSignal of newTickerSignals) {
+            // FIX B: Normalize topic key
+            const topic = (newSignal.topic || 'untagged').toLowerCase();
+            
+            if (!state[topic]) {
+                state[topic] = { 
+                    rolling_90d_history: [],
+                    correlations: {},
+                    predictivePotential: 0,
+                    avgDailyConviction: 0
+                };
+            }
+
+            state[topic].rolling_90d_history.push({
+                date: dateStr,
+                sentimentScore: newSignal.sentimentScore,
+                conviction: newSignal.convictionScore,
+                fwd_1d: null, fwd_3d: null, fwd_7d: null, fwd_21d: null,
+            });
+
+            // Prune history
+            state[topic].rolling_90d_history = state[topic].rolling_90d_history.slice(-ROLLING_HISTORY_DAYS);
+        }
+    }
+
+    /**
+     * Helper to recalculate all stats for a ticker's topics.
+     * Modifies the `state` object in-place.
+     * @returns {object} The dailyOutputForTicker
+     */
+    _recalculateAllTopics(state, logger) {
+        const predictiveTopics = [];
+
+        for (const topic in state) {
+            const history = state[topic].rolling_90d_history;
+            if (!history || history.length === 0) continue;
+
+            // FIX G: Warn if doc size is at risk
+            if (history.length > 500) { // arbitrary high number
+                logger.log('WARN', `[SocialTopicPredictive] Topic "${topic}" has ${history.length} history entries. May approach doc size limit.`);
+            }
+
+            // FIX E: Ensure sort order
+            history.sort((a, b) => new Date(a.date) - new Date(b.date));
+            
+            // --- START V3 MODIFICATION: RECENCY WEIGHTING ---
+            const N = history.length;
+            if (N === 0) continue;
+
+            // Create recency weights (exponential decay)
+            // Newest item (i=N-1) gets weight 1.0 (age=0)
+            // Oldest item (i=0) gets weight exp(-K) (age=N-1)
+            const vecWeights = history.map((h, i) => {
+                const age_in_days = N - 1 - i; // 0 for newest, N-1 for oldest
+                // Normalize age by history length so K is consistent
+                return Math.exp(-WEIGHTING_DECAY_K * age_in_days / N);
+            });
+            // --- END V3 MODIFICATION ---
+
+            const vecSentiment = history.map(h => h.sentimentScore);
+            const vecConviction = history.map(h => h.conviction); // Used for PP score
+
+            let totalPP = 0;
+            let windowsCounted = 0;
+            let totalStableWindows = 0;
+
+            for (const window of FORWARD_WINDOWS) {
+                const vecForwardReturn = history.map(h => h[`fwd_${window}d`]);
+                
+                // --- V3 MODIFICATION ---
+                // Use the new weighted Pearson calculation
+                const corr = _calculateWeightedPearson(vecSentiment, vecForwardReturn, vecWeights);
+                // --- END V3 MODIFICATION ---
+                
+                state[topic].correlations[`fwd_${window}d`] = corr; // Save { value, samples }
+
+                if (corr.samples >= MIN_SAMPLE_COUNT) {
+                    totalPP += Math.abs(corr.value);
+                    windowsCounted++;
+                    if (Math.abs(corr.value) > CORR_THRESHOLD) {
+                        totalStableWindows++;
+                    }
+                }
+            }
+            
+            let ppScore = 0;
+            if (windowsCounted > 0) {
+                ppScore = totalPP / windowsCounted; 
+                
+                const avgConviction = state[topic].rolling_90d_history.reduce((acc, h) => acc + h.conviction, 0) / history.length;
+                state[topic].avgDailyConviction = avgConviction;
+                
+                // FIX F: Clamp before log
+                ppScore *= Math.log(1 + Math.max(0, avgConviction));
+                
+                ppScore *= (totalStableWindows / windowsCounted);
+            }
+            
+            state[topic].predictivePotential = ppScore;
+
+            predictiveTopics.push({
+                topic: topic,
+                predictivePotential: ppScore,
+                sentimentScore: vecSentiment[vecSentiment.length - 1], // Latest sentiment
+                correlations: state[topic].correlations
+            });
+        }
+
+        // --- Generate Daily Output ---
+        predictiveTopics.sort((a, b) => b.predictivePotential - a.predictivePotential);
+        const confidence = predictiveTopics.length > 0 ? predictiveTopics[0].predictivePotential : 0;
+        const normalizedConfidence = Math.min(1, confidence / 10); // Simple normalization
+
+        return {
+            topPredictiveBullishTopics: predictiveTopics
+                .filter(t => t.sentimentScore > 0.1)
+                .slice(0, 5),
+            topPredictiveBearishTopics: predictiveTopics
+                .filter(t => t.sentimentScore < -0.1)
+                .slice(0, 5),
+            predictiveConfidenceScore: normalizedConfidence
+        };
+    }
+
+    async getResult() { return null; } // Logic is in process()
+    reset() {
+        this.priceMap = null;
+        this.tickerToIdMap = null;
+        this.dependenciesLoaded = false;
+        this.pLimit = pLimit(MAX_CONCURRENT_TRANSACTIONS);
+    }
+}
+
+module.exports = SocialTopicPredictivePotentialIndex;

package/calculations/meta/user_expectancy_score.js

@@ -0,0 +1,54 @@
+/**
+ * @fileoverview Pass 2: Calculates the "Expectancy Score" for each user
+ * based on their historical trading performance.
+ * Expectancy = (WinRatio * AvgProfitPct) + (LossRatio * AvgLossPct)
+ */
+class UserExpectancyScore {
+
+    static getDependencies() {
+        // Depends on our new Pass 1 calculation
+        return ['historical-performance-aggregator'];
+    }
+
+    constructor() {}
+
+    async process(dateStr, dependencies, config, fetchedDependencies) {
+        const { logger } = dependencies;
+        const performanceData = fetchedDependencies['historical-performance-aggregator'];
+
+        if (!performanceData) {
+            logger.log('WARN', `[UserExpectancyScore] Missing dependency 'historical-performance-aggregator' for ${dateStr}. Skipping.`);
+            return null;
+        }
+
+        const expectancyScores = {};
+
+        for (const userId in performanceData) {
+            const stats = performanceData[userId];
+
+            // Ensure we have all the data needed
+            if (typeof stats.winRatio !== 'number' || typeof stats.avgProfitPct !== 'number' || typeof stats.avgLossPct !== 'number') {
+                continue;
+            }
+
+            const winRatio = stats.winRatio / 100.0;
+            const lossRatio = 1.0 - winRatio;
+
+            // Expectancy formula
+            const expectancy = (winRatio * stats.avgProfitPct) + (lossRatio * stats.avgLossPct);
+
+            expectancyScores[userId] = {
+                expectancy_score: expectancy,
+                winRatio: stats.winRatio, // Pass through for debugging/filtering
+                totalTrades: stats.totalTrades
+            };
+        }
+
+        return expectancyScores;
+    }
+
+    async getResult() { return null; }
+    reset() {}
+}
+
+module.exports = UserExpectancyScore;

package/calculations/socialPosts/social-topic-sentiment-matrix.js

@@ -0,0 +1,105 @@
+/**
+ * @fileoverview NEW CALCULATION (PASS 1)
+ * Aggregates all social post data for the day into a detailed matrix.
+ * For each ticker, it maps each discussed topic to its aggregated
+ * sentiment (Bullish/Bearish/Neutral) and a "Conviction Score"
+ * derived from likes and comments.
+ *
+ * This is a "socialPosts" category calculation, so it runs once per day.
+ */
+
+class SocialTopicSentimentMatrix {
+    constructor() {
+        // The main data structure.
+        // Format: { [ticker]: { [topic]: { ...stats } } }
+        this.matrix = {};
+        
+        // Flag to ensure this runs only once per day
+        this.processed = false;
+    }
+
+    /**
+     * @param {null} todayPortfolio - Not used.
+     * @param {null} yesterdayPortfolio - Not used.
+     * @param {null} userId - Not used.
+     * @param {object} context - Shared context.
+     * @param {object} todayInsights - Not used.
+     * @param {object} yesterdayInsights - Not used.
+     * @param {object} todaySocialPostInsights - Map of { [postId]: postData } for today.
+     * @param {object} yesterdaySocialPostInsights - Not used.
+     */
+    async process(todayPortfolio, yesterdayPortfolio, userId, context, todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights) {
+        
+        // Run only once per day
+        if (this.processed || !todaySocialPostInsights) {
+            return;
+        }
+        this.processed = true;
+
+        const posts = Object.values(todaySocialPostInsights);
+
+        for (const post of posts) {
+            if (!post || !post.tickers || !post.sentiment) continue;
+
+            const tickers = post.tickers;
+            const sentiment = post.sentiment.overallSentiment || 'Neutral';
+            
+            // Use a default topic if Gemini found none
+            let topics = post.sentiment.topics || [];
+            if (topics.length === 0) {
+                topics.push('untagged');
+            }
+
+            // Calculate conviction (e.g., comments are 2x as valuable as likes)
+            const convictionScore = (post.likeCount || 0) + ((post.commentCount || 0) * 2);
+
+            for (const ticker of tickers) {
+                if (!ticker) continue;
+                
+                for (const topic of topics) {
+                    const topicLower = topic.toLowerCase();
+
+                    // Initialize ticker map
+                    if (!this.matrix[ticker]) {
+                        this.matrix[ticker] = {};
+                    }
+                    // Initialize topic map
+                    if (!this.matrix[ticker][topicLower]) {
+                        this.matrix[ticker][topicLower] = {
+                            bullishPosts: 0,
+                            bearishPosts: 0,
+                            neutralPosts: 0,
+                            totalPosts: 0,
+                            convictionScore: 0
+                        };
+                    }
+
+                    // Aggregate data into the bucket
+                    const bucket = this.matrix[ticker][topicLower];
+                    bucket.totalPosts++;
+                    bucket.convictionScore += convictionScore;
+
+                    if (sentiment === 'Bullish') {
+                        bucket.bullishPosts++;
+                    } else if (sentiment === 'Bearish') {
+                        bucket.bearishPosts++;
+                    } else {
+                        bucket.neutralPosts++;
+                    }
+                }
+            }
+        }
+    }
+
+    async getResult() {
+        // This entire matrix will be the result
+        return this.matrix;
+    }
+
+    reset() {
+        this.matrix = {};
+        this.processed = false;
+    }
+}
+
+module.exports = SocialTopicSentimentMatrix;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "aiden-shared-calculations-unified",
-  "version": "1.0.46",
+  "version": "1.0.48",
   "description": "Shared calculation modules for the BullTrackers Computation System.",
   "main": "index.js",
   "files": [