aiden-shared-calculations-unified 1.0.64 → 1.0.66

package/calculations/socialPosts/social-top-mentioned-words.js

@@ -1,102 +1,123 @@
 /**
- * @fileoverview Finds the top mentioned words in all posts for the day,
- * filtering out common stop words, and calculates the trend from yesterday.
- * This calculation ONLY uses the social post insights context.
+ * @fileoverview Social Calculation (Pass 1)
+ *
+ * This metric answers: "What are the top 50 most mentioned
+ * words in social media posts today, and what is their
+ * trend since yesterday?"
+ *
+ * This is a stateful calculation.
  */
-
 class SocialTopMentionedWords {
     constructor() {
-        this.todayWordCounts = {};
-        this.yesterdayWordCounts = {};
-        this.processed = false;
-
-        // A small, representative list of stop words.
-        // In a real implementation, this list would be much larger.
-        this.stopWords = new Set([
-            'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
-            'have', 'has', 'had', 'do', 'does', 'did', 'but', 'if', 'or', 'and',
-            'of', 'at', 'by', 'for', 'with', 'about', 'to', 'from', 'in', 'on',
-            'it', 'i', 'me', 'my', 'you', 'your', 'he', 'she', 'we', 'they', 'them',
-            'this', 'that', 'these', 'those', 'what', 'which', 'who', 'whom',
-            'not', 'will', 'just', 'so', 'up', 'out', 'no', 'yes', 'all', 'any'
-        ]);
+        // { [word]: { count: 0, history: [] } }
+        this.wordCounts = new Map();
+        // A set of common words to ignore
+        this.stopWords = new Set(['the', 'a', 'is', 'in', 'to', 'for', 'of', 'and', 'it', 'on', 'with', 'that', 'this', 'buy', 'sell']);
     }
 
     /**
-     * Helper to process posts and populate a word count map.
-     * @param {object} socialPostInsights - Map of { [postId]: postData }
-     * @param {object} targetCountMap - The object to store word counts.
+     * Defines the output schema for this calculation.
+     * @returns {object} JSON Schema object
      */
-    _countWords(socialPostInsights, targetCountMap) {
-        if (!socialPostInsights) return;
-        const posts = Object.values(socialPostInsights);
-
-        for (const post of posts) {
-            const text = post.fullText?.toLowerCase() || '';
-            if (!text) continue;
-
-            // Get all words 3+ chars long
-            const words = text.match(/\b[a-z]{3,}\b/g) || [];
+    static getSchema() {
+        const wordSchema = {
+            "type": "object",
+            "properties": {
+                "word": { "type": "string" },
+                "count": { "type": "number" },
+                "trend_30d_pct": { "type": "number" },
+                "history_30d": { "type": "array", "items": { "type": "number" } }
+            },
+            "required": ["word", "count", "trend_30d_pct", "history_30d"]
+        };
 
-            for (const word of words) {
-                if (!this.stopWords.has(word)) {
-                    targetCountMap[word] = (targetCountMap[word] || 0) + 1;
-                }
-            }
-        }
+        return {
+            "type": "array",
+            "description": "Top 50 most mentioned words (non-stop words) with their 30-day trend.",
+            "items": wordSchema
+        };
     }
 
     /**
-     * @param {object} todaySocialPostInsights - Map of { [postId]: postData } for today.
-     * @param {object} yesterdaySocialPostInsights - Map of { [postId]: postData } for yesterday.
+     * @param {string} dateStr - Today's date.
+     * @param {object} dependencies - db, logger.
+     * @param {object} config - Computation config.
+     * @param {object} fetchedDependencies - (UNUSED) In-memory results.
      */
-    async process(todayPortfolio, yesterdayPortfolio, userId, context, todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights) {
+    async process(dateStr, dependencies, config, fetchedDependencies) {
+        const { db, logger, calculationUtils } = dependencies;
+        
+        // 1. Load this metric's history from yesterday
+        // Note: History is stored by word
+        const yHistoryData = await calculationUtils.loadYesterdayData('social-top-mentioned-words-history'); // Fictional helper
         
-        if (this.processed) {
-            return;
+        if (yHistoryData) {
+            for (const [word, data] of Object.entries(yHistoryData)) {
+                this.wordCounts.set(word, {
+                    count: 0,
+                    history: data.history_30d || []
+                });
+            }
         }
-        this.processed = true;
-
-        // Process today's posts
-        this._countWords(todaySocialPostInsights, this.todayWordCounts);
         
-        // Process yesterday's posts for trend calculation
-        this._countWords(yesterdaySocialPostInsights, this.yesterdayWordCounts);
-    }
+        // 2. Process today's raw social data
+        const todaySocialPosts = await calculationUtils.loadSocialData(dateStr); 
 
-    async getResult() {
-        if (Object.keys(this.todayWordCounts).length === 0) {
-            return {};
+        for (const post of todaySocialPosts) {
+            const text = (post.content || '').toLowerCase().replace(/[^a-z\s]/g, '');
+            const words = text.split(/\s+/);
+            
+            for (const word of words) {
+                if (word.length > 2 && !this.stopWords.has(word)) {
+                    if (!this.wordCounts.has(word)) {
+                        this.wordCounts.set(word, { count: 0, history: [] });
+                    }
+                    this.wordCounts.get(word).count++;
+                }
+            }
         }
+    }
 
-        const totalTodayWords = Object.values(this.todayWordCounts).reduce((a, b) => a + b, 0);
-
-        const topWords = Object.entries(this.todayWordCounts)
-            .sort(([, countA], [, countB]) => countB - countA)
-            .slice(0, 50); // Return the Top 50
+    async getResult() {
+        const allWords = [];
+        
+        for (const [word, data] of this.wordCounts.entries()) {
+            const history = data.history;
+            const todayCount = data.count;
+            
+            // Don't include words that weren't mentioned today
+            if (todayCount === 0) continue; 
 
-        const result = topWords.map(([term, count]) => {
-            const yesterdayCount = this.yesterdayWordCounts[term] || 0;
-            const trend = count - yesterdayCount;
-            const percent = (count / totalTodayWords) * 100;
+            const newHistory = [todayCount, ...history].slice(0, 30);
             
-            return {
-                term,
-                count,
-                percent: parseFloat(percent.toFixed(2)),
-                trend 
-            };
-        });
+            let trend_30d_pct = 0;
+            if (history.length > 0) {
+                const avg_30d = history.reduce((a, b) => a + b, 0) / history.length;
+                if (avg_30d > 0) {
+                    trend_30d_pct = ((todayCount - avg_30d) / avg_30d) * 100;
+                }
+            }
 
-        // Returns an array like:
-        // [ { term: "btc", count: 150, percent: 5.2, trend: 25 }, ... ]
-        return { top_words: result };
+            allWords.push({
+                word: word,
+                count: todayCount,
+                trend_30d_pct: trend_30d_pct,
+                history_30d: newHistory
+            });
+        }
+        
+        // Sort by today's count and take top 50
+        allWords.sort((a, b) => b.count - a.count);
+        
+        // The *full* result (for history) should be saved to a different
+        // doc ('social-top-mentioned-words-history') by the computation system.
+        
+        // Return only the top 50 for the API.
+        return allWords.slice(0, 50);
     }
 
     reset() {
-        this.todayWordCounts = {};
-        this.yesterdayWordCounts = {};
-        this.processed = false;
+        this.wordCounts.clear();
     }
 }
 

package/calculations/socialPosts/social-topic-interest-evolution.js

@@ -1,53 +1,116 @@
 /**
- * @fileoverview Aggregates counts of all topics extracted by the AI
- * from social posts for the day. Used to build a "Topic Interest Area" chart.
- * This calculation ONLY uses the social post insights context.
+ * @fileoverview Social Calculation (Pass 1)
+ *
+ * This metric answers: "What is the daily count of
+ * posts for each topic extracted from social media?"
+ *
+ * This is a stateful calculation.
  */
-
 class SocialTopicInterestEvolution {
     constructor() {
-        this.topicCounts = {};
-        this.processed = false;
+        // { [topic]: { count: 0, history: [] } }
+        this.topicData = new Map();
     }
 
     /**
-     * @param {null} todayPortfolio - Not used.
-     * ... (other params not used) ...
-     * @param {object} todaySocialPostInsights - Map of { [postId]: postData } for today.
-     * @param {object} yesterdaySocialPostInsights - Not used.
+     * Defines the output schema for this calculation.
+     * @returns {object} JSON Schema object
      */
-    async process(todayPortfolio, yesterdayPortfolio, userId, context, todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights) {
-        
-        if (this.processed || !todaySocialPostInsights) {
-            return;
-        }
-        this.processed = true;
+    static getSchema() {
+        const topicSchema = {
+            "type": "object",
+            "description": "Daily and trended post count for a specific topic.",
+            "properties": {
+                "count": {
+                    "type": "number",
+                    "description": "Today's raw post count for this topic."
+                },
+                "trend_30d_pct": {
+                    "type": "number",
+                    "description": "Percentage change from the 30-day average post count."
+                },
+                "history_30d": {
+                    "type": "array",
+                    "description": "30-day history of post counts (today first).",
+                    "items": { "type": "number" }
+                }
+            },
+            "required": ["count", "trend_30d_pct", "history_30d"]
+        };
 
-        const posts = Object.values(todaySocialPostInsights);
+        return {
+            "type": "object",
+            "description": "Tracks the daily post count and 30-day trend for each extracted topic.",
+            "patternProperties": {
+                "^.*$": topicSchema // Topic name
+            },
+            "additionalProperties": topicSchema
+        };
+    }
 
-        for (const post of posts) {
-            // Topics are in post.sentiment.topics
-            const topics = post.sentiment?.topics;
-            if (!topics || !Array.isArray(topics)) continue;
+    /**
+     * @param {string} dateStr - Today's date.
+     * @param {object} dependencies - db, logger.
+     * @param {object} config - Computation config.
+     * @param {object} fetchedDependencies - (UNUSED) In-memory results.
+     */
+    async process(dateStr, dependencies, config, fetchedDependencies) {
+        const { db, logger, calculationUtils } = dependencies;
+        
+        // 1. Load this metric's history from yesterday
+        const yHistoryData = await calculationUtils.loadYesterdayData('social-topic-interest-evolution');
 
+        if (yHistoryData) {
+            for (const [topic, data] of Object.entries(yHistoryData)) {
+                this.topicData.set(topic, {
+                    count: 0,
+                    history: data.history_30d || []
+                });
+            }
+        }
+        
+        // 2. Process today's raw social data
+        const todaySocialPosts = await calculationUtils.loadSocialData(dateStr); 
+
+        for (const post of todaySocialPosts) {
+            // Assume topics are extracted by the social loader
+            const topics = post.topics || []; 
             for (const topic of topics) {
-                // Normalize topic for consistent aggregation
-                const normalizedTopic = String(topic).toLowerCase().trim();
-                if (normalizedTopic) {
-                    this.topicCounts[normalizedTopic] = (this.topicCounts[normalizedTopic] || 0) + 1;
+                if (!this.topicData.has(topic)) {
+                    this.topicData.set(topic, { count: 0, history: [] });
                 }
+                this.topicData.get(topic).count++;
             }
         }
     }
 
     async getResult() {
-        // Returns an object like: { "earnings": 45, "cpi": 12, "fed": 22 }
-        return this.topicCounts;
+        const result = {};
+        for (const [topic, data] of this.topicData.entries()) {
+            const history = data.history;
+            const todayCount = data.count;
+
+            const newHistory = [todayCount, ...history].slice(0, 30);
+            
+            let trend_30d_pct = 0;
+            if (history.length > 0) {
+                const avg_30d = history.reduce((a, b) => a + b, 0) / history.length;
+                if (avg_30d > 0) {
+                    trend_30d_pct = ((todayCount - avg_30d) / avg_30d) * 100;
+                }
+            }
+
+            result[topic] = {
+                count: todayCount,
+                trend_30d_pct: trend_30d_pct,
+                history_30d: newHistory
+            };
+        }
+        return result;
     }
 
     reset() {
-        this.topicCounts = {};
-        this.processed = false;
+        this.topicData.clear();
     }
 }
 

package/calculations/socialPosts/social-topic-sentiment-matrix.js

@@ -1,104 +1,96 @@
 /**
- * @fileoverview NEW CALCULATION (PASS 1)
- * Aggregates all social post data for the day into a detailed matrix.
- * For each ticker, it maps each discussed topic to its aggregated
- * sentiment (Bullish/Bearish/Neutral) and a "Conviction Score"
- * derived from likes and comments.
+ * @fileoverview Social Calculation (Pass 1)
  *
- * This is a "socialPosts" category calculation, so it runs once per day.
+ * This metric answers: "What is the sentiment breakdown
+ * (bullish, bearish) for each topic?"
+ *
+ * Creates a matrix of Topic <-> Sentiment.
  */
-
 class SocialTopicSentimentMatrix {
     constructor() {
-        // The main data structure.
-        // Format: { [ticker]: { [topic]: { ...stats } } }
-        this.matrix = {};
-        
-        // Flag to ensure this runs only once per day
-        this.processed = false;
+        // { [topic]: { bullish: 0, bearish: 0, neutral: 0 } }
+        this.matrix = new Map();
     }
 
     /**
-     * @param {null} todayPortfolio - Not used.
-     * @param {null} yesterdayPortfolio - Not used.
-     * @param {null} userId - Not used.
-     * @param {object} context - Shared context.
-     * @param {object} todayInsights - Not used.
-     * @param {object} yesterdayInsights - Not used.
-     * @param {object} todaySocialPostInsights - Map of { [postId]: postData } for today.
-     * @param {object} yesterdaySocialPostInsights - Not used.
+     * Defines the output schema for this calculation.
+     * @returns {object} JSON Schema object
      */
-    async process(todayPortfolio, yesterdayPortfolio, userId, context, todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights) {
-        
-        // Run only once per day
-        if (this.processed || !todaySocialPostInsights) {
-            return;
-        }
-        this.processed = true;
-
-        const posts = Object.values(todaySocialPostInsights);
+    static getSchema() {
+        const sentimentSchema = {
+            "type": "object",
+            "properties": {
+                "bullish": { "type": "number" },
+                "bearish": { "type": "number" },
+                "neutral": { "type": "number" },
+                "net_sentiment": { "type": "number" }
+            },
+            "required": ["bullish", "bearish", "neutral", "net_sentiment"]
+        };
 
-        for (const post of posts) {
-            if (!post || !post.tickers || !post.sentiment) continue;
+        return {
+            "type": "object",
+            "description": "Calculates the sentiment breakdown (bullish, bearish, neutral) for each topic.",
+            "patternProperties": {
+                "^.*$": sentimentSchema // Topic name
+            },
+            "additionalProperties": sentimentSchema
+        };
+    }
 
-            const tickers = post.tickers;
-            const sentiment = post.sentiment.overallSentiment || 'Neutral';
-            
-            // Use a default topic if Gemini found none
-            let topics = post.sentiment.topics || [];
-            if (topics.length === 0) {
-                topics.push('untagged');
-            }
+    _initTopic(topic) {
+        if (!this.matrix.has(topic)) {
+            this.matrix.set(topic, {
+                bullish: 0,
+                bearish: 0,
+                neutral: 0
+            });
+        }
+    }
 
-            // Calculate conviction (e.g., comments are 2x as valuable as likes)
-            const convictionScore = (post.likeCount || 0) + ((post.commentCount || 0) * 2);
+    /**
+     * @param {string} dateStr - Today's date.
+     * @param {object} dependencies - db, logger.
+     * @param {object} config - Computation config.
+     * @param {object} fetchedDependencies - (UNUSED) In-memory results.
+     */
+    async process(dateStr, dependencies, config, fetchedDependencies) {
+        const { calculationUtils } = dependencies;
+        const todaySocialPosts = await calculationUtils.loadSocialData(dateStr); 
 
-            for (const ticker of tickers) {
-                if (!ticker) continue;
+        for (const post of todaySocialPosts) {
+            const topics = post.topics || [];
+            const sentiment = post.sentiment || 'neutral'; // 'bullish', 'bearish', 'neutral'
+            
+            for (const topic of topics) {
+                this._initTopic(topic);
+                const data = this.matrix.get(topic);
                 
-                for (const topic of topics) {
-                    const topicLower = topic.toLowerCase();
-
-                    // Initialize ticker map
-                    if (!this.matrix[ticker]) {
-                        this.matrix[ticker] = {};
-                    }
-                    // Initialize topic map
-                    if (!this.matrix[ticker][topicLower]) {
-                        this.matrix[ticker][topicLower] = {
-                            bullishPosts: 0,
-                            bearishPosts: 0,
-                            neutralPosts: 0,
-                            totalPosts: 0,
-                            convictionScore: 0
-                        };
-                    }
-
-                    // Aggregate data into the bucket
-                    const bucket = this.matrix[ticker][topicLower];
-                    bucket.totalPosts++;
-                    bucket.convictionScore += convictionScore;
-
-                    if (sentiment === 'Bullish') {
-                        bucket.bullishPosts++;
-                    } else if (sentiment === 'Bearish') {
-                        bucket.bearishPosts++;
-                    } else {
-                        bucket.neutralPosts++;
-                    }
+                if (sentiment === 'bullish') {
+                    data.bullish++;
+                } else if (sentiment === 'bearish') {
+                    data.bearish++;
+                } else {
+                    data.neutral++;
                 }
             }
         }
     }
 
     async getResult() {
-        // This entire matrix will be the result
-        return this.matrix;
+        const result = {};
+        for (const [topic, data] of this.matrix.entries()) {
+            result[topic] = {
+                ...data,
+                // Net Sentiment = (Bullish - Bearish)
+                net_sentiment: data.bullish - data.bearish
+            };
+        }
+        return result;
     }
 
     reset() {
-        this.matrix = {};
-        this.processed = false;
+        this.matrix.clear();
     }
 }