bulltrackers-module 1.0.105 → 1.0.106

package/functions/social-task-handler/helpers/handler_helpers.js

@@ -1,289 +1,289 @@
-/**
- * @fileoverview Main pipe: pipe.maintenance.handleSocialTask
- * This function is triggered by Pub/Sub for a single ticker.
- * It fetches posts, deduplicates, analyzes, and stores them.
- */
-
-const { FieldValue, FieldPath } = require('@google-cloud/firestore');
-
-/**
- * --- UPDATED HELPER FUNCTION ---
- * Calls Gemini to classify sentiment and extract topics.
- * @param {object} dependencies - Contains logger and geminiModel.
- * @param {string} snippet - The text snippet to analyze.
- * @returns {Promise<object>} An object: { overallSentiment: '...', topics: [...] }.
- */
-async function getSentimentFromGemini(dependencies, snippet) {
-    const { logger, geminiModel } = dependencies;
-
-    if (!geminiModel) {
-        logger.log('WARN', '[getSentimentFromGemini] Gemini model not found in dependencies.');
-        return { overallSentiment: "Neutral", topics: [] }; // Return default object
-    }
-
-    // --- NEW PROMPT ---
-    const prompt = `You are a financial analyst. Analyze the following social media post.
-Your task is to provide:
-1.  The overall sentiment (Bullish, Bearish, or Neutral) toward the main asset. Return only one word.
-2.  A list of key topics or events mentioned (e.g., "FOMC", "CPI", "Earnings", "Inflation", "War", "Acquisition"). If no specific event is mentioned, return an empty array.
-
-Return ONLY a valid JSON object in this exact format:
-{
-  "overallSentiment": "...",
-  "topics": ["...", "..."]
-}
-
-Post: "${snippet}"`
-    // --- END NEW PROMPT ---
-
-    try {
-        const request = {
-            contents: [
-                {
-                    role: "user",
-                    parts: [ { text: prompt } ]
-                }
-            ],
-            generationConfig: {
-                temperature: 0.1,
-                topP: 0.1,
-                maxOutputTokens: 256 // Increased tokens for JSON
-            }
-        };
-
-        const result = await geminiModel.generateContent(request);
-        const response = result.response;
-        
-        const text = response?.candidates?.[0]?.content?.parts?.[0]?.text?.trim() || '';
-
-        // --- NEW: Parse JSON response ---
-        try {
-            // Find the JSON block
-            const jsonMatch = text.match(/\{[\s\S]*\}/);
-            if (jsonMatch && jsonMatch[0]) {
-                const parsedJson = JSON.parse(jsonMatch[0]);
-                // Validate structure
-                if (parsedJson && parsedJson.overallSentiment && Array.isArray(parsedJson.topics)) {
-                    logger.log('INFO', `[getSentimentFromGemini] Classified sentiment: ${parsedJson.overallSentiment}, Topics: ${parsedJson.topics.join(', ')}`);
-                    return parsedJson; // Return the full object
-                }
-            }
-            logger.log('WARN', `[getSentimentFromGemini] Unexpected JSON structure from AI: "${text}". Defaulting.`);
-            return { overallSentiment: "Neutral", topics: [] };
-        } catch (parseError) {
-            logger.log('WARN', `[getSentimentFromGemini] Failed to parse JSON response from AI: "${text}". Defaulting.`, { error: parseError.message });
-            return { overallSentiment: "Neutral", topics: [] };
-        }
-        // --- END NEW JSON PARSE ---
-
-    } catch (error) {
-        logger.log('ERROR', '[getSentimentFromGemini] Error calling Gemini API.', { 
-            errorMessage: error.message,
-            errorStack: error.stack
-        });
-        // Default to 'Neutral' on API error to avoid halting the pipeline
-        return { overallSentiment: "Neutral", topics: [] }; // Return default object
-    }
-}
-
-
-/**
- * Main pipe (Task Handler): pipe.maintenance.handleSocialTask
- * @param {object} message - The Pub/Sub message.
- * @param {object} context - The message context.
- * @param {object} config - Configuration object.
- * @param {object} dependencies - Contains db, logger, headerManager, proxyManager, geminiModel.
- * @returns {Promise<void>}
- */
-exports.handleSocialTask = async (message, context, config, dependencies) => {
-    const { db, logger, headerManager, proxyManager } = dependencies;
-
-    let task;
-    try {
-        task = JSON.parse(Buffer.from(message.data, 'base64').toString('utf-8'));
-    } catch (e) {
-        logger.log('ERROR', '[SocialTask] Failed to parse Pub/Sub message data.', { error: e.message, data: message.data });
-        return; // Acknowledge the message
-    }
-
-    const { tickerId, since } = task;
-    const sinceDate = new Date(since);
-    const taskId = `social-${tickerId}-${context.eventId || Date.now()}`;
-    logger.log('INFO', `[SocialTask/${taskId}] Processing ticker ${tickerId} for posts since ${since}.`);
-
-    // --- Config validation ---
-    if (!config.socialApiBaseUrl || !config.socialInsightsCollectionName || !config.processedPostsCollectionName) {
-        logger.log('ERROR', `[SocialTask/${taskId}] Missing required configuration.`);
-        throw new Error('Missing required configuration for Social Task.');
-    }
-
-    const processedPostsRef = db.collection(config.processedPostsCollectionName);
-    const today = new Date().toISOString().slice(0, 10);
-    const insightsCollectionRef = db.collection(config.socialInsightsCollectionName).doc(today).collection('posts');
-
-    let offset = 0;
-    const take = 10; // hardcode to 10
-    let keepFetching = true;
-    let postsProcessed = 0;
-    let postsSaved = 0;
-    const processedInThisRun = new Set(); // Local dedupe
-
-    try {
-        while (keepFetching) {
-            const url = `${config.socialApiBaseUrl}${tickerId}?take=${take}&offset=${offset}&reactionsPageSize=20`;
-            logger.log('TRACE', `[SocialTask/${taskId}] Fetching: ${url}`);
-            
-            const selectedHeader = await headerManager.selectHeader();
-            let wasSuccess = false;
-            let response;
-            
-            try {
-                response = await proxyManager.fetch(url, { headers: selectedHeader.header });
-                if (!response.ok) {
-                    throw new Error(`API error ${response.status}`);
-                }
-                wasSuccess = true;
-            } catch (fetchError) {
-                logger.log('WARN', `[SocialTask/${taskId}] Fetch failed for offset ${offset}.`, { err: fetchError.message });
-                keepFetching = false; // Stop on fetch error
-                if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, false);
-                continue;
-            } finally {
-                if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, wasSuccess);
-            }
-
-            const page = await response.json();
-            const discussions = page?.discussions;
-
-            if (!Array.isArray(discussions) || discussions.length === 0) {
-                logger.log('INFO', `[SocialTask/${taskId}] No more posts found at offset ${offset}. Stopping.`);
-                keepFetching = false;
-                continue;
-            }
-
-            const postIds = discussions.map(d => d.post.id).filter(Boolean);
-            if (postIds.length === 0) {
-                offset += take;
-                continue;
-            }
-
-            // --- Deduplication Check ---
-            const existingDocs = await processedPostsRef.where(FieldPath.documentId(), 'in', postIds).get();
-            const existingIds = new Set(existingDocs.docs.map(d => d.id));
-
-            const batch = db.batch();
-            let newPostsInBatch = 0;
-
-            for (const discussion of discussions) {
-                const post = discussion?.post;
-                if (!post || !post.id || !post.message?.text) continue;
-
-                // Stop pagination if we've reached posts older than our window
-                const postCreatedDate = new Date(post.created);
-                if (postCreatedDate < sinceDate) {
-                    keepFetching = false;
-                    continue; 
-                }
-
-                // Skip if already processed
-                if (existingIds.has(post.id) || processedInThisRun.has(post.id)) {
-                    continue;
-                }
-                
-                // Filter language (accept 'en' and 'en-gb')
-                const lang = post.message.languageCode || 'unknown';
-                if (lang !== 'en' && lang !== 'en-gb') {
-                    continue; // Skip non-English posts
-                }
-
-                // --- Process the new post ---
-                postsProcessed++;
-                processedInThisRun.add(post.id);
-                const text = post.message.text;
-
-                // --- START: Enhanced Data Extraction ---
-                const likeCount = discussion.emotionsData?.like?.paging?.totalCount || 0;
-                const commentCount = discussion.summary?.totalCommentsAndReplies || 0;
-                let pollData = null;
-
-                // Check for poll data and extract it
-                if (post.type === 'Poll' && post.metadata?.poll?.options) {
-                    pollData = {
-                        id: post.metadata.poll.id,
-                        options: post.metadata.poll.options.map(opt => ({
-                            id: opt.id,
-                            text: opt.text,
-                            votesCount: opt.votesCount || 0
-                        }))
-                    };
-                }
-                // --- END: Enhanced Data Extraction ---
-
-
-                // 1. Truncate for AI
-                const MAX_CHARS = 500; // ~125 tokens, very cheap
-                let snippet = text;
-                if (text.length > (MAX_CHARS * 2)) {
-                  // Create a "summary" snippet
-                  snippet = text.substring(0, MAX_CHARS) + " ... " + text.substring(text.length - MAX_CHARS);
-                } else if (text.length > MAX_CHARS) {
-                  // Just truncate
-                  snippet = text.substring(0, MAX_CHARS);
-                }
-
-                // 2. AI Sentiment Analysis
-                // Pass full dependencies object
-                // --- MODIFIED: Store the entire result object ---
-                const sentimentResult = await getSentimentFromGemini(dependencies, snippet);
-
-                // 3. Prepare data for storage
-                const postData = {
-                    sentiment: sentimentResult, // <-- Store the full {overallSentiment, topics} object
-                    textSnippet: snippet,
-                    fullText: text, // Store the full original text
-                    language: lang,
-                    tickers: post.tags.map(t => t.market?.symbolName).filter(Boolean),
-                    postOwnerId: post.owner?.id,
-                    likeCount: likeCount, // Store like count
-                    commentCount: commentCount, // Store comment count
-                    pollData: pollData, // Store poll data (will be null if not a poll)
-                    createdAt: post.created,
-                    fetchedAt: FieldValue.serverTimestamp()
-                };
-                // --- END MODIFICATION ---
-                
-                // 4. Add to batch for `daily_social_insights`
-                const insightDocRef = insightsCollectionRef.doc(post.id);
-                batch.set(insightDocRef, postData);
-
-                // 5. Add to batch for `processed_social_posts` (dedupe collection)
-                const dedupeDocRef = processedPostsRef.doc(post.id);
-                batch.set(dedupeDocRef, { processedAt: FieldValue.serverTimestamp() });
-                
-                newPostsInBatch++;
-            }
-
-            if (newPostsInBatch > 0) {
-                await batch.commit();
-                postsSaved += newPostsInBatch;
-                logger.log('INFO', `[SocialTask/${taskId}] Saved ${newPostsInBatch} new posts from offset ${offset}.`);
-            }
-
-            // Continue to next page
-            offset += take;
-        }
-
-        logger.log('SUCCESS', `[SocialTask/${taskId}] Run complete. Processed ${postsProcessed} new posts, saved ${postsSaved}.`);
-
-    } catch (error) {
-        logger.log('ERROR', `[SocialTask/${taskId}] Fatal error during task execution.`, { errorMessage: error.message, errorStack: error.stack });
-        throw error;
-    } finally {
-        // Always flush header performance
-        try {
-            await headerManager.flushPerformanceUpdates();
-        } catch (flushError) {
-            logger.log('ERROR', `[SocialTask/${taskId}] Failed to flush header performance.`, { errorMessage: flushError.message });
-        }
-    }
+/**
+ * @fileoverview Main pipe: pipe.maintenance.handleSocialTask
+ * This function is triggered by Pub/Sub for a single ticker.
+ * It fetches posts, deduplicates, analyzes, and stores them.
+ */
+
+const { FieldValue, FieldPath } = require('@google-cloud/firestore');
+
+/**
+ * --- UPDATED HELPER FUNCTION ---
+ * Calls Gemini to classify sentiment and extract topics.
+ * @param {object} dependencies - Contains logger and geminiModel.
+ * @param {string} snippet - The text snippet to analyze.
+ * @returns {Promise<object>} An object: { overallSentiment: '...', topics: [...] }.
+ */
+async function getSentimentFromGemini(dependencies, snippet) {
+    const { logger, geminiModel } = dependencies;
+
+    if (!geminiModel) {
+        logger.log('WARN', '[getSentimentFromGemini] Gemini model not found in dependencies.');
+        return { overallSentiment: "Neutral", topics: [] }; // Return default object
+    }
+
+    // --- NEW PROMPT ---
+    const prompt = `You are a financial analyst. Analyze the following social media post.
+Your task is to provide:
+1.  The overall sentiment (Bullish, Bearish, or Neutral) toward the main asset. Return only one word.
+2.  A list of key topics or events mentioned (e.g., "FOMC", "CPI", "Earnings", "Inflation", "War", "Acquisition"). If no specific event is mentioned, return an empty array.
+
+Return ONLY a valid JSON object in this exact format:
+{
+  "overallSentiment": "...",
+  "topics": ["...", "..."]
+}
+
+Post: "${snippet}"`
+    // --- END NEW PROMPT ---
+
+    try {
+        const request = {
+            contents: [
+                {
+                    role: "user",
+                    parts: [ { text: prompt } ]
+                }
+            ],
+            generationConfig: {
+                temperature: 0.1,
+                topP: 0.1,
+                maxOutputTokens: 256 // Increased tokens for JSON
+            }
+        };
+
+        const result = await geminiModel.generateContent(request);
+        const response = result.response;
+        
+        const text = response?.candidates?.[0]?.content?.parts?.[0]?.text?.trim() || '';
+
+        // --- NEW: Parse JSON response ---
+        try {
+            // Find the JSON block
+            const jsonMatch = text.match(/\{[\s\S]*\}/);
+            if (jsonMatch && jsonMatch[0]) {
+                const parsedJson = JSON.parse(jsonMatch[0]);
+                // Validate structure
+                if (parsedJson && parsedJson.overallSentiment && Array.isArray(parsedJson.topics)) {
+                    logger.log('INFO', `[getSentimentFromGemini] Classified sentiment: ${parsedJson.overallSentiment}, Topics: ${parsedJson.topics.join(', ')}`);
+                    return parsedJson; // Return the full object
+                }
+            }
+            logger.log('WARN', `[getSentimentFromGemini] Unexpected JSON structure from AI: "${text}". Defaulting.`);
+            return { overallSentiment: "Neutral", topics: [] };
+        } catch (parseError) {
+            logger.log('WARN', `[getSentimentFromGemini] Failed to parse JSON response from AI: "${text}". Defaulting.`, { error: parseError.message });
+            return { overallSentiment: "Neutral", topics: [] };
+        }
+        // --- END NEW JSON PARSE ---
+
+    } catch (error) {
+        logger.log('ERROR', '[getSentimentFromGemini] Error calling Gemini API.', { 
+            errorMessage: error.message,
+            errorStack: error.stack
+        });
+        // Default to 'Neutral' on API error to avoid halting the pipeline
+        return { overallSentiment: "Neutral", topics: [] }; // Return default object
+    }
+}
+
+
+/**
+ * Main pipe (Task Handler): pipe.maintenance.handleSocialTask
+ * @param {object} message - The Pub/Sub message.
+ * @param {object} context - The message context.
+ * @param {object} config - Configuration object.
+ * @param {object} dependencies - Contains db, logger, headerManager, proxyManager, geminiModel.
+ * @returns {Promise<void>}
+ */
+exports.handleSocialTask = async (message, context, config, dependencies) => {
+    const { db, logger, headerManager, proxyManager } = dependencies;
+
+    let task;
+    try {
+        task = JSON.parse(Buffer.from(message.data, 'base64').toString('utf-8'));
+    } catch (e) {
+        logger.log('ERROR', '[SocialTask] Failed to parse Pub/Sub message data.', { error: e.message, data: message.data });
+        return; // Acknowledge the message
+    }
+
+    const { tickerId, since } = task;
+    const sinceDate = new Date(since);
+    const taskId = `social-${tickerId}-${context.eventId || Date.now()}`;
+    logger.log('INFO', `[SocialTask/${taskId}] Processing ticker ${tickerId} for posts since ${since}.`);
+
+    // --- Config validation ---
+    if (!config.socialApiBaseUrl || !config.socialInsightsCollectionName || !config.processedPostsCollectionName) {
+        logger.log('ERROR', `[SocialTask/${taskId}] Missing required configuration.`);
+        throw new Error('Missing required configuration for Social Task.');
+    }
+
+    const processedPostsRef = db.collection(config.processedPostsCollectionName);
+    const today = new Date().toISOString().slice(0, 10);
+    const insightsCollectionRef = db.collection(config.socialInsightsCollectionName).doc(today).collection('posts');
+
+    let offset = 0;
+    const take = 10; // hardcode to 10
+    let keepFetching = true;
+    let postsProcessed = 0;
+    let postsSaved = 0;
+    const processedInThisRun = new Set(); // Local dedupe
+
+    try {
+        while (keepFetching) {
+            const url = `${config.socialApiBaseUrl}${tickerId}?take=${take}&offset=${offset}&reactionsPageSize=20`;
+            logger.log('TRACE', `[SocialTask/${taskId}] Fetching: ${url}`);
+            
+            const selectedHeader = await headerManager.selectHeader();
+            let wasSuccess = false;
+            let response;
+            
+            try {
+                response = await proxyManager.fetch(url, { headers: selectedHeader.header });
+                if (!response.ok) {
+                    throw new Error(`API error ${response.status}`);
+                }
+                wasSuccess = true;
+            } catch (fetchError) {
+                logger.log('WARN', `[SocialTask/${taskId}] Fetch failed for offset ${offset}.`, { err: fetchError.message });
+                keepFetching = false; // Stop on fetch error
+                if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, false);
+                continue;
+            } finally {
+                if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, wasSuccess);
+            }
+
+            const page = await response.json();
+            const discussions = page?.discussions;
+
+            if (!Array.isArray(discussions) || discussions.length === 0) {
+                logger.log('INFO', `[SocialTask/${taskId}] No more posts found at offset ${offset}. Stopping.`);
+                keepFetching = false;
+                continue;
+            }
+
+            const postIds = discussions.map(d => d.post.id).filter(Boolean);
+            if (postIds.length === 0) {
+                offset += take;
+                continue;
+            }
+
+            // --- Deduplication Check ---
+            const existingDocs = await processedPostsRef.where(FieldPath.documentId(), 'in', postIds).get();
+            const existingIds = new Set(existingDocs.docs.map(d => d.id));
+
+            const batch = db.batch();
+            let newPostsInBatch = 0;
+
+            for (const discussion of discussions) {
+                const post = discussion?.post;
+                if (!post || !post.id || !post.message?.text) continue;
+
+                // Stop pagination if we've reached posts older than our window
+                const postCreatedDate = new Date(post.created);
+                if (postCreatedDate < sinceDate) {
+                    keepFetching = false;
+                    continue; 
+                }
+
+                // Skip if already processed
+                if (existingIds.has(post.id) || processedInThisRun.has(post.id)) {
+                    continue;
+                }
+                
+                // Filter language (accept 'en' and 'en-gb')
+                const lang = post.message.languageCode || 'unknown';
+                if (lang !== 'en' && lang !== 'en-gb') {
+                    continue; // Skip non-English posts
+                }
+
+                // --- Process the new post ---
+                postsProcessed++;
+                processedInThisRun.add(post.id);
+                const text = post.message.text;
+
+                // --- START: Enhanced Data Extraction ---
+                const likeCount = discussion.emotionsData?.like?.paging?.totalCount || 0;
+                const commentCount = discussion.summary?.totalCommentsAndReplies || 0;
+                let pollData = null;
+
+                // Check for poll data and extract it
+                if (post.type === 'Poll' && post.metadata?.poll?.options) {
+                    pollData = {
+                        id: post.metadata.poll.id,
+                        options: post.metadata.poll.options.map(opt => ({
+                            id: opt.id,
+                            text: opt.text,
+                            votesCount: opt.votesCount || 0
+                        }))
+                    };
+                }
+                // --- END: Enhanced Data Extraction ---
+
+
+                // 1. Truncate for AI
+                const MAX_CHARS = 500; // ~125 tokens, very cheap
+                let snippet = text;
+                if (text.length > (MAX_CHARS * 2)) {
+                  // Create a "summary" snippet
+                  snippet = text.substring(0, MAX_CHARS) + " ... " + text.substring(text.length - MAX_CHARS);
+                } else if (text.length > MAX_CHARS) {
+                  // Just truncate
+                  snippet = text.substring(0, MAX_CHARS);
+                }
+
+                // 2. AI Sentiment Analysis
+                // Pass full dependencies object
+                // --- MODIFIED: Store the entire result object ---
+                const sentimentResult = await getSentimentFromGemini(dependencies, snippet);
+
+                // 3. Prepare data for storage
+                const postData = {
+                    sentiment: sentimentResult, // <-- Store the full {overallSentiment, topics} object
+                    textSnippet: snippet,
+                    fullText: text, // Store the full original text
+                    language: lang,
+                    tickers: post.tags.map(t => t.market?.symbolName).filter(Boolean),
+                    postOwnerId: post.owner?.id,
+                    likeCount: likeCount, // Store like count
+                    commentCount: commentCount, // Store comment count
+                    pollData: pollData, // Store poll data (will be null if not a poll)
+                    createdAt: post.created,
+                    fetchedAt: FieldValue.serverTimestamp()
+                };
+                // --- END MODIFICATION ---
+                
+                // 4. Add to batch for `daily_social_insights`
+                const insightDocRef = insightsCollectionRef.doc(post.id);
+                batch.set(insightDocRef, postData);
+
+                // 5. Add to batch for `processed_social_posts` (dedupe collection)
+                const dedupeDocRef = processedPostsRef.doc(post.id);
+                batch.set(dedupeDocRef, { processedAt: FieldValue.serverTimestamp() });
+                
+                newPostsInBatch++;
+            }
+
+            if (newPostsInBatch > 0) {
+                await batch.commit();
+                postsSaved += newPostsInBatch;
+                logger.log('INFO', `[SocialTask/${taskId}] Saved ${newPostsInBatch} new posts from offset ${offset}.`);
+            }
+
+            // Continue to next page
+            offset += take;
+        }
+
+        logger.log('SUCCESS', `[SocialTask/${taskId}] Run complete. Processed ${postsProcessed} new posts, saved ${postsSaved}.`);
+
+    } catch (error) {
+        logger.log('ERROR', `[SocialTask/${taskId}] Fatal error during task execution.`, { errorMessage: error.message, errorStack: error.stack });
+        throw error;
+    } finally {
+        // Always flush header performance
+        try {
+            await headerManager.flushPerformanceUpdates();
+        } catch (flushError) {
+            logger.log('ERROR', `[SocialTask/${taskId}] Failed to flush header performance.`, { errorMessage: flushError.message });
+        }
+    }
 };