aiden-shared-calculations-unified 1.0.95 → 1.0.97

package/calculations/legacy/social-topic-driver-index.js

@@ -1,147 +0,0 @@
-/**
- * @fileoverview Calculation (Pass 5) for social topic driver index.
- *
- * This metric answers: "Which social media topics are driving
- * the most asset price movement or capital flow?"
- *
- * It *depends* on 'social-topic-predictive-potential' (Pass 5).
- */
-class SocialTopicDriverIndex {
-    constructor() {
-        // No per-user processing
-    }
-
-    /**
-     * Defines the output schema for this calculation.
-     * @returns {object} JSON Schema object
-     */
-    static getSchema() {
-        const topicSchema = {
-            "type": "object",
-            "properties": {
-                "topic": { "type": "string" },
-                "driver_score": { "type": "number" },
-                // These fields are from an older version but kept for schema
-                // compatibility. They will be null in the corrected logic.
-                "correlation_flow_30d": { "type": ["number", "null"] },
-                "correlation_price_30d": { "type": ["number", "null"] }
-            },
-            "required": ["topic", "driver_score"]
-        };
-
-        return {
-            "type": "object",
-            "description": "Ranks social media topics by their 'driver_score' (predictive potential for flow and price).",
-            "properties": {
-                "top_driving_topics": {
-                    "type": "array",
-                    "description": "Top 5 topics driving market activity.",
-                    "items": topicSchema
-                },
-                "all_topics": {
-                    "type": "array",
-                    "description": "Full list of all topics and their driver scores.",
-                    "items": topicSchema
-                }
-            },
-            "required": ["top_driving_topics", "all_topics"]
-        };
-    }
-
-    /**
-     * Statically declare dependencies.
-     * (This was already correct)
-     */
-    static getDependencies() {
-        return [
-            'social-topic-predictive-potential' // Pass 5
-        ];
-    }
-
-    process() {
-        // No-op
-    }
-
-    /**
-     * --- LOGIC FIXED ---
-     * This function is rewritten to correctly consume the output of
-     * 'social-topic-predictive-potential'. It aggregates the
-     * 'predictivePotential' score for each topic across *all* tickers
-     * to create a global driver score.
-     */
-    getResult(fetchedDependencies) {
-        const potentialData = fetchedDependencies['social-topic-predictive-potential'];
-        
-        const defaults = {
-            top_driving_topics: [],
-            all_topics: []
-        };
-
-        // The dependency returns a nested object. We need 'daily_topic_signals'.
-        const dailyTopicSignals = potentialData?.daily_topic_signals;
-
-        if (!dailyTopicSignals || Object.keys(dailyTopicSignals).length === 0) {
-            return defaults;
-        }
-
-        // Use a Map to aggregate scores for each topic
-        const topicAggregator = new Map();
-
-        // Iterate over each TICKER (e.g., 'AAPL', 'TSLA') in the signals
-        for (const tickerData of Object.values(dailyTopicSignals)) {
-            
-            // Combine bullish and bearish topics for that ticker
-            const allTickerTopics = [
-                ...(tickerData.topPredictiveBullishTopics || []),
-                ...(tickerData.topPredictiveBearishTopics || [])
-            ];
-
-            // Iterate over the topics *for that ticker*
-            for (const topicData of allTickerTopics) {
-                const topicName = topicData.topic;
-                
-                // Use the 'predictivePotential' score calculated by the dependency
-                const score = topicData.predictivePotential || 0;
-
-                if (!topicAggregator.has(topicName)) {
-                    topicAggregator.set(topicName, { totalScore: 0, count: 0 });
-                }
-                
-                const agg = topicAggregator.get(topicName);
-                agg.totalScore += score;
-                agg.count += 1;
-            }
-        }
-        
-        const allTopics = [];
-        // Now, create the final ranked list
-        for (const [topic, data] of topicAggregator.entries()) {
-            if (data.count === 0) continue;
-            
-            // Calculate the average driver score across all tickers
-            const avgScore = data.totalScore / data.count;
-            
-            allTopics.push({
-                topic: topic,
-                driver_score: avgScore,
-                // Set old/incompatible fields to null to match schema
-                correlation_flow_30d: null, 
-                correlation_price_30d: null 
-            });
-        }
-        
-        // Sort by the new, correct driver_score
-        allTopics.sort((a, b) => b.driver_score - a.driver_score);
-
-        return {
-            top_driving_topics: allTopics.slice(0, 5),
-            all_topics: allTopics
-        };
-    }
-
-    reset() {
-        // No state
-    }
-}
-
-module.exports = SocialTopicDriverIndex;

package/calculations/legacy/social-topic-predictive-potential.js

@@ -1,461 +0,0 @@
-/**
- * @fileoverview NEW CALCULATION (PASS 3 - META)
- * This is the "Quant" signal discovery engine.
- *
- * This is a stateful, rolling calculation that implements several
- * critical optimizations based on production-level review:
- * 1. It uses `p-limit` to run concurrent transactions safely.
- * 2. It returns state directly from transactions, avoiding costly re-reads.
- * 3. It uses a forward-looking price helper (`_findPriceForward`)
- * to be resilient to market holidays and missing data.
- * 4. All logic is factored into testable helper functions.
- *
- * --- V3 MODIFICATION (Quant Upgrade) ---
- * 5. Implements RECENCY WEIGHTING on the correlation calculation.
- * Signals from recent days are given exponentially more weight
- * than signals from 90 days ago, per standard quant practice.
- * This is achieved by upgrading _calculatePearson to
- * _calculateWeightedPearson.
- */
-
-const { FieldValue } = require('@google-cloud/firestore');
-// p-limit is a non-standard-lib, but is included in the bulltrackers-module
-// and available via the root `index.js` dependency injection.
-// We will assume it's passed in via dependencies.calculationUtils.pLimit
-const pLimit = require('p-limit'); 
-
-// Import all required utils
-const { loadAllPriceData, getDailyPriceChange } = require('../../utils/price_data_provider');
-const { loadInstrumentMappings } = require('../../utils/sector_mapping_provider');
-
-// --- CONFIGURATION ---
-const SHARD_COLLECTION_NAME = 'social_topic_rolling_stats';
-const ROLLING_HISTORY_DAYS = 90;
-const MIN_SAMPLE_COUNT = 12; // Min samples needed to trust a correlation
-const CORR_THRESHOLD = 0.25; // Min abs correlation to be considered "stable"
-const FORWARD_WINDOWS = [1, 3, 7, 21]; // [1d, 3d, 7d, 21d]
-const MAX_CONCURRENT_TRANSACTIONS = 50; // Max parallel Firestore transactions
-const MAX_LOOKAHEAD_DAYS = 7; // How far to look for a non-holiday price
-const WEIGHTING_DECAY_K = 3.0; // Decay factor for recency weighting (e.g., k=3.0)
-
-// --- STATS HELPER ---
-/**
- * --- MODIFIED: Upgraded to Weighted Pearson Correlation ---
- * Calculates the Pearson correlation coefficient for two arrays,
- * weighted by a third array.
- * Gracefully handles nulls by only using paired data.
- */
-function _calculateWeightedPearson(vecX, vecY, vecWeights) {
-    let sumW = 0;
-    let sumWX = 0;
-    let sumWY = 0;
-    let validPairs = []; // To store non-null pairs
-
-    for (let i = 0; i < vecX.length; i++) {
-        const x = vecX[i];
-        const y = vecY[i];
-        const w = vecWeights[i]; // Get the weight
-
-        // Only use pairs where both values are valid numbers and weight is positive
-        if (x !== null && y !== null && isFinite(x) && isFinite(y) && w > 0) {
-            sumW += w;
-            sumWX += w * x;
-            sumWY += w * y;
-            validPairs.push({x, y, w});
-        }
-    }
-
-    // Need at least 2 data points and positive total weight
-    if (sumW === 0 || validPairs.length < 2) {
-        return { value: 0, samples: validPairs.length };
-    }
-
-    // Calculate weighted means
-    const meanX = sumWX / sumW;
-    const meanY = sumWY / sumW;
-
-    let sumCov = 0;
-    let sumVarX = 0;
-    let sumVarY = 0;
-
-    // Second pass to calculate weighted covariance and variances
-    for (const pair of validPairs) {
-        const { x, y, w } = pair;
-        sumCov += w * (x - meanX) * (y - meanY);
-        sumVarX += w * (x - meanX) * (x - meanX);
-        sumVarY += w * (y - meanY) * (y - meanY);
-    }
-
-    const denominator = Math.sqrt(sumVarX * sumVarY);
-
-    if (denominator === 0) {
-        return { value: 0, samples: validPairs.length };
-    }
-
-    // The weighted correlation is cov(x,y) / (std(x) * std(y))
-    // The sumW terms cancel out from the numerator and denominator.
-    const corr = sumCov / denominator;
-    
-    // Clamp correlation to valid range [-1, 1] to handle floating point errors
-    return { value: Math.max(-1, Math.min(1, corr)), samples: validPairs.length };
-}
-
-// --- DATE HELPERS ---
-function _getDateStr(baseDate, daysOffset) {
-    const date = new Date(baseDate); // baseDate is Date object or string
-    date.setUTCDate(date.getUTCDate() + daysOffset);
-    return date.toISOString().slice(0, 10);
-}
-
-// --- PRICE HELPERS ---
-/**
- * Resiliently finds a price for a given date from the price map.
- * (Looks *backward* for last available price)
- */
-function _findPrice(instrumentId, dateStr, priceMap) {
-    if (!priceMap || !priceMap[instrumentId]) return null;
-    const priceHistory = priceMap[instrumentId];
-    
-    let checkDate = new Date(dateStr + 'T00:00:00Z');
-    
-    for (let i = 0; i < MAX_LOOKAHEAD_DAYS; i++) {
-        const checkDateStr = checkDate.toISOString().slice(0, 10);
-        const price = priceHistory[checkDateStr];
-        
-        if (price !== undefined && price !== null && price > 0) return price;
-        checkDate.setUTCDate(checkDate.getUTCDate() - 1);
-    }
-    return null;
-}
-
-/**
- * NEW: Resiliently finds the next available price *on or after* a given date.
- * (Looks *forward* to handle holidays/weekends)
- */
-function _findPriceForward(instrumentId, dateStr, priceMap) {
-    if (!priceMap || !priceMap[instrumentId]) return null;
-    const priceHistory = priceMap[instrumentId];
-    
-    let checkDate = new Date(dateStr + 'T00:00:00Z');
-    
-    for (let i = 0; i <= MAX_LOOKAHEAD_DAYS; i++) {
-        const checkDateStr = checkDate.toISOString().slice(0, 10);
-        const price = priceHistory[checkDateStr];
-        
-        if (price !== undefined && price !== null && price > 0) return price;
-        checkDate.setUTCDate(checkDate.getUTCDate() + 1);
-    }
-    return null;
-}
-
-
-class SocialTopicPredictivePotentialIndex {
-
-    static getDependencies() {
-        // --- FIX 1: Changed from 'social-topic-driver-index' to break the cycle ---
-        return ['social-topic-sentiment-matrix'];
-    }
-
-    constructor() {
-        this.priceMap = null;
-        this.tickerToIdMap = null;
-        this.dependenciesLoaded = false;
-        // Concurrency limiter for Firestore transactions
-        this.pLimit = pLimit(MAX_CONCURRENT_TRANSACTIONS);
-    }
-
-    async _loadDependencies(calculationUtils) {
-        if (this.dependenciesLoaded) return;
-        
-        const [priceData, mappings] = await Promise.all([
-            calculationUtils.loadAllPriceData(),
-            calculationUtils.loadInstrumentMappings()
-        ]);
-        
-        this.priceMap = priceData;
-        this.tickerToIdMap = {};
-        if (mappings && mappings.instrumentToTicker) {
-            for (const [id, ticker] of Object.entries(mappings.instrumentToTicker)) {
-                this.tickerToIdMap[ticker] = id;
-            }
-        }
-        this.dependenciesLoaded = true;
-    }
-
-    /**
-     * @param {string} dateStr The date to run the analysis for (e.g., "2025-11-05").
-     * @param {object} dependencies The shared dependencies (db, logger, calculationUtils).
-     * @param {object} config The computation system configuration.
-     * @param {object} fetchedDependencies In-memory results from Pass 2.
-     * @returns {Promise<object|null>} The analysis result or null.
-     */
-    async process(dateStr, dependencies, config, fetchedDependencies) {
-        const { db, logger, calculationUtils } = dependencies;
-        
-        // 1. Load all dependencies
-        // pLimit is not in calculationUtils by default, so we'll use our own
-        // If it were, we'd use: this.pLimit = calculationUtils.pLimit(MAX_CONCURRENT_TRANSACTIONS);
-        await this._loadDependencies(calculationUtils);
-        
-        // --- FIX 2: Read from the correct dependency ---
-        const todaySignals = fetchedDependencies['social-topic-sentiment-matrix'];
-
-        if (!todaySignals || Object.keys(todaySignals).length === 0) {
-            // --- FIX 2.1: Updated log message ---
-            logger.log('WARN', `[SocialTopicPredictive] Missing or empty dependency 'social-topic-sentiment-matrix' for ${dateStr}. Skipping.`);
-            return null;
-        }
-
-        if (!this.priceMap || !this.tickerToIdMap) {
-            logger.log('ERROR', `[SocialTopicPredictive] Price map or Ticker map failed to load. Aborting.`);
-            return null;
-        }
-
-        // --- Prepare final output objects ---
-        const shardedData = {};
-        const dailyOutput = {};
-        
-        const allTickers = Object.keys(todaySignals);
-
-        // 2. Run all ticker updates in parallel, limited by pLimit
-        const transactionPromises = allTickers.map(ticker => 
-            this.pLimit(() => this._processTickerTransaction(
-                ticker, 
-                dateStr, 
-                todaySignals[ticker] || {}, 
-                dependencies
-            ))
-        );
-        
-        const txResults = await Promise.all(transactionPromises);
-
-        // 3. Collect results from transactions (FIX A)
-        for (const res of txResults) {
-            if (!res) continue; // Transaction may have failed and returned null
-            shardedData[res.ticker] = res.state;
-            dailyOutput[res.ticker] = res.dailyOutputForTicker;
-        }
-
-        // 4. Return both the state (for sharded write) and daily signal (for API)
-        return {
-            sharded_social_stats: { [SHARD_COLLECTION_NAME]: shardedData },
-            daily_topic_signals: dailyOutput
-        };
-    }
-
-    /**
-     * This is the core logic, run for a single ticker *inside* a transaction.
-     * It reads, modifies, and writes state for one ticker, then returns
-     * the new state and daily signal.
-     */
-    async _processTickerTransaction(ticker, dateStr, todaySignal, dependencies) {
-        const { db, logger } = dependencies;
-        
-        try {
-            return await db.runTransaction(async (transaction) => {
-                const instrumentId = this.tickerToIdMap[ticker];
-                if (!instrumentId) return null;
-
-                const todayPrice = _findPrice(instrumentId, dateStr, this.priceMap);
-                if (todayPrice === null) return null; // Cannot update returns
-
-                // --- 4a. Load State ---
-                const docRef = db.collection(SHARD_COLLECTION_NAME).doc(ticker);
-                const doc = await transaction.get(docRef);
-                const state = doc.exists ? doc.data() : {}; 
-
-                // --- 4b. Update Forward Returns (Factored Helper) ---
-                this._updateForwardReturns(state, instrumentId, dateStr, todayPrice, this.priceMap);
-
-                // --- 4c. Add New Signals (Factored Helper) ---
-                // We assume 'todaySignal' (from social-topic-sentiment-matrix)
-                // has an 'allDrivers' property.
-                this._addNewSignals(state, todaySignal.allDrivers || [], dateStr);
-
-                // --- 4d. Recalculate Correlations (Factored Helper) ---
-                const dailyOutputForTicker = this._recalculateAllTopics(state, logger);
-
-                // --- 4e. Save State (FIX 3.2: Use merge) ---
-                transaction.set(docRef, state, { merge: true });
-
-                // --- 4f. Return (FIX A) ---
-                return { ticker, state, dailyOutputForTicker };
-            });
-        } catch (error) {
-            logger.log('ERROR', `[SocialTopicPredictive] Transaction failed for ticker ${ticker}`, { err: error.message, stack: error.stack });
-            return null; // Return null on transaction failure
-        }
-    }
-
-    /**
-     * Helper to update past forward-return windows.
-     * Modifies the `state` object in-place.
-     */
-    _updateForwardReturns(state, instrumentId, dateStr, todayPrice, priceMap) {
-        for (const topic in state) {
-            if (!state[topic].rolling_90d_history) continue;
-            
-            for (const historyEntry of state[topic].rolling_90d_history) {
-                const signalPrice = _findPrice(instrumentId, historyEntry.date, priceMap);
-                if (signalPrice === null) continue;
-
-                // FIX D: Check all windows
-                for (const window of FORWARD_WINDOWS) {
-                    // Skip if already filled
-                    if (historyEntry[`fwd_${window}d`] !== null) continue; 
-
-                    const targetDateStr = _getDateStr(new Date(historyEntry.date + 'T00:00:00Z'), window);
-                    
-                    // Not yet time to fill this window
-                    if (dateStr < targetDateStr) continue; 
-                    
-                    // FIX C: Use _findPriceForward
-                    const targetPrice = _findPriceForward(instrumentId, targetDateStr, priceMap);
-                    
-                    if (targetPrice !== null) {
-                        historyEntry[`fwd_${window}d`] = (targetPrice / signalPrice) - 1;
-                    }
-                }
-            }
-        }
-    }
-
-    /**
-     * Helper to add the new day's signals to the state.
-     * Modifies the `state` object in-place.
-     */
-    _addNewSignals(state, newTickerSignals, dateStr) {
-        for (const newSignal of newTickerSignals) {
-            // FIX B: Normalize topic key
-            const topic = (newSignal.topic || 'untagged').toLowerCase();
-            
-            if (!state[topic]) {
-                state[topic] = { 
-                    rolling_90d_history: [],
-                    correlations: {},
-                    predictivePotential: 0,
-                    avgDailyConviction: 0
-                };
-            }
-
-            state[topic].rolling_90d_history.push({
-                date: dateStr,
-                sentimentScore: newSignal.sentimentScore,
-                conviction: newSignal.convictionScore,
-                fwd_1d: null, fwd_3d: null, fwd_7d: null, fwd_21d: null,
-            });
-
-            // Prune history
-            state[topic].rolling_90d_history = state[topic].rolling_90d_history.slice(-ROLLING_HISTORY_DAYS);
-        }
-    }
-
-    /**
-     * Helper to recalculate all stats for a ticker's topics.
-     * Modifies the `state` object in-place.
-     * @returns {object} The dailyOutputForTicker
-     */
-    _recalculateAllTopics(state, logger) {
-        const predictiveTopics = [];
-
-        for (const topic in state) {
-            const history = state[topic].rolling_90d_history;
-            if (!history || history.length === 0) continue;
-
-            // FIX G: Warn if doc size is at risk
-            if (history.length > 500) { // arbitrary high number
-                logger.log('WARN', `[SocialTopicPredictive] Topic "${topic}" has ${history.length} history entries. May approach doc size limit.`);
-            }
-
-            // FIX E: Ensure sort order
-            history.sort((a, b) => new Date(a.date) - new Date(b.date));
-            
-            // --- START V3 MODIFICATION: RECENCY WEIGHTING ---
-            const N = history.length;
-            if (N === 0) continue;
-
-            // Create recency weights (exponential decay)
-            // Newest item (i=N-1) gets weight 1.0 (age=0)
-            // Oldest item (i=0) gets weight exp(-K) (age=N-1)
-            const vecWeights = history.map((h, i) => {
-                const age_in_days = N - 1 - i; // 0 for newest, N-1 for oldest
-                // Normalize age by history length so K is consistent
-                return Math.exp(-WEIGHTING_DECAY_K * age_in_days / N);
-            });
-            // --- END V3 MODIFICATION ---
-
-            const vecSentiment = history.map(h => h.sentimentScore);
-            const vecConviction = history.map(h => h.conviction); // Used for PP score
-
-            let totalPP = 0;
-            let windowsCounted = 0;
-            let totalStableWindows = 0;
-
-            for (const window of FORWARD_WINDOWS) {
-                const vecForwardReturn = history.map(h => h[`fwd_${window}d`]);
-                
-                // --- V3 MODIFICATION ---
-                // Use the new weighted Pearson calculation
-                const corr = _calculateWeightedPearson(vecSentiment, vecForwardReturn, vecWeights);
-                // --- END V3 MODIFICATION ---
-                
-                state[topic].correlations[`fwd_${window}d`] = corr; // Save { value, samples }
-
-                if (corr.samples >= MIN_SAMPLE_COUNT) {
-                    totalPP += Math.abs(corr.value);
-                    windowsCounted++;
-                    if (Math.abs(corr.value) > CORR_THRESHOLD) {
-                        totalStableWindows++;
-                    }
-                }
-            }
-            
-            let ppScore = 0;
-            if (windowsCounted > 0) {
-                ppScore = totalPP / windowsCounted; 
-                
-                const avgConviction = state[topic].rolling_90d_history.reduce((acc, h) => acc + h.conviction, 0) / history.length;
-                state[topic].avgDailyConviction = avgConviction;
-                
-                // FIX F: Clamp before log
-                ppScore *= Math.log(1 + Math.max(0, avgConviction));
-                
-                ppScore *= (totalStableWindows / windowsCounted);
-            }
-            
-            state[topic].predictivePotential = ppScore;
-
-            predictiveTopics.push({
-                topic: topic,
-                predictivePotential: ppScore,
-                sentimentScore: vecSentiment[vecSentiment.length - 1], // Latest sentiment
-                correlations: state[topic].correlations
-            });
-        }
-
-        // --- Generate Daily Output ---
-        predictiveTopics.sort((a, b) => b.predictivePotential - a.predictivePotential);
-        const confidence = predictiveTopics.length > 0 ? predictiveTopics[0].predictivePotential : 0;
-        const normalizedConfidence = Math.min(1, confidence / 10); // Simple normalization
-
-        return {
-            topPredictiveBullishTopics: predictiveTopics
-                .filter(t => t.sentimentScore > 0.1)
-                .slice(0, 5),
-            topPredictiveBearishTopics: predictiveTopics
-                .filter(t => t.sentimentScore < -0.1)
-                .slice(0, 5),
-            predictiveConfidenceScore: normalizedConfidence
-        };
-    }
-
-    async getResult() { return null; } // Logic is in process()
-    reset() {
-        this.priceMap = null;
-        this.tickerToIdMap = null;
-        this.dependenciesLoaded = false;
-        this.pLimit = pLimit(MAX_CONCURRENT_TRANSACTIONS);
-    }
-}
-
-module.exports = SocialTopicPredictivePotentialIndex;