bulltrackers-module 1.0.449 → 1.0.451

package/functions/social-task-handler/helpers/handler_helpers.js

@@ -97,6 +97,9 @@ async function getAdvancedAnalysisFromGemini(dependencies, snippet) {
         Post: "${snippet}"
     `;
 
+    let result = null;
+    let part = null;
+    
     try {
         const request = {
             contents: [{ role: "user", parts: [{ text: prompt }] }],
@@ -107,10 +110,10 @@ async function getAdvancedAnalysisFromGemini(dependencies, snippet) {
             }
         };
 
-        const result = await geminiModel.generateContent(request);
+        result = await geminiModel.generateContent(request);
 
         // Gemini 2.5 Flash returns JSON directly in parts[0].json
-        const part = result?.response?.candidates?.[0]?.content?.parts?.[0];
+        part = result?.response?.candidates?.[0]?.content?.parts?.[0];
 
         let parsed;
 
@@ -133,7 +136,25 @@ async function getAdvancedAnalysisFromGemini(dependencies, snippet) {
         };
 
     } catch (e) {
-        logger.log("ERROR", "[Gemini AI] JSON handling failed", e);
+        logger.log("ERROR", "[Gemini AI] JSON handling failed", {
+            error: e.message,
+            stack: e.stack,
+            snippet: snippet.substring(0, 100),
+            part: part ? {
+                hasJson: !!part.json,
+                hasText: !!part.text,
+                textPreview: part.text ? part.text.substring(0, 200) : null
+            } : null,
+            resultStructure: result?.response ? {
+                hasCandidates: !!result.response.candidates,
+                candidatesLength: result.response.candidates?.length,
+                firstCandidate: result.response.candidates?.[0] ? {
+                    hasContent: !!result.response.candidates[0].content,
+                    hasParts: !!result.response.candidates[0].content?.parts,
+                    partsLength: result.response.candidates[0].content?.parts?.length
+                } : null
+            } : null
+        });
         return { overallSentiment: "Neutral", topics: [], isSpam: false, qualityScore: 0.5 };
     }
 }
@@ -235,21 +256,36 @@ exports.handleSocialTask = async (message, context, config, dependencies) => {
                 existingDocs.forEach(d => existingIds.add(d.id));
             }
 
+            // Filter and prepare posts: only new posts in English
+            const newPostsWithDiscussion = discussions
+                .filter(d => {
+                    const post = d.post;
+                    if (!post || !post.id || !post.message?.text || existingIds.has(post.id)) return false;
+                    const lang = (post.message.languageCode || 'unknown').toLowerCase();
+                    return lang === 'en' || lang === 'en-gb';
+                })
+                // Sort by text length (longer posts are generally more useful for analysis)
+                .sort((a, b) => (b.post.message.text?.length || 0) - (a.post.message.text?.length || 0));
+
+            // COST OPTIMIZATION: Cap AI processing to top 5 posts per user to reduce costs
+            // Remaining posts will be stored with default AI values
+            const MAX_POSTS_FOR_AI = 5;
+            const postsForAI = newPostsWithDiscussion.slice(0, MAX_POSTS_FOR_AI);
+            const postsWithoutAI = newPostsWithDiscussion.slice(MAX_POSTS_FOR_AI);
+
+            logger.log('INFO', `[SocialTask/${taskId}] Filtered ${newPostsWithDiscussion.length} new posts. Sending ${postsForAI.length} to AI, ${postsWithoutAI.length} without AI.`);
+
             const batch = db.batch();
             let pageBatchCount = 0;
 
-            for (const discussion of discussions) {
+            // Process posts that will get AI analysis (top 5 by text length)
+            for (const discussion of postsForAI) {
                 if (totalSaved + pageBatchCount >= MAX_POSTS_TO_STORE) {
                     keepFetching = false;
                     break;
                 }
 
                 const post = discussion.post;
-                if (!post || !post.id || !post.message?.text || existingIds.has(post.id)) continue;
-
-                const lang = (post.message.languageCode || 'unknown').toLowerCase();
-                if (lang !== 'en' && lang !== 'en-gb') continue;
-
                 const snippet = post.message.text.substring(0, 500);
                 const aiResult = await getAdvancedAnalysisFromGemini(dependencies, snippet);
 
@@ -260,7 +296,7 @@ exports.handleSocialTask = async (message, context, config, dependencies) => {
                     username: post.owner?.username,
                     createdAt: post.created,
                     fetchedAt: FieldValue.serverTimestamp(),
-                    snippet: snippet, // Added for debugging
+                    snippet: snippet,
                     stats: {
                         likes: discussion.emotionsData?.like?.paging?.totalCount || 0,
                         comments: discussion.summary?.totalCommentsAndReplies || 0
@@ -274,6 +310,40 @@ exports.handleSocialTask = async (message, context, config, dependencies) => {
                 pageBatchCount++;
             }
 
+            // Process remaining posts WITHOUT AI analysis (to avoid costs, but still store them)
+            for (const discussion of postsWithoutAI) {
+                if (totalSaved + pageBatchCount >= MAX_POSTS_TO_STORE) {
+                    keepFetching = false;
+                    break;
+                }
+
+                const post = discussion.post;
+                const docData = {
+                    postId: post.id,
+                    text: post.message.text,
+                    ownerId: post.owner?.id,
+                    username: post.owner?.username,
+                    createdAt: post.created,
+                    fetchedAt: FieldValue.serverTimestamp(),
+                    snippet: post.message.text.substring(0, 500),
+                    stats: {
+                        likes: discussion.emotionsData?.like?.paging?.totalCount || 0,
+                        comments: discussion.summary?.totalCommentsAndReplies || 0
+                    },
+                    aiAnalysis: {
+                        overallSentiment: "Neutral",
+                        topics: [],
+                        isSpam: false,
+                        qualityScore: 0.5
+                    }, // Default values when AI is skipped to reduce costs
+                    tags: post.tags?.map(t => t.market?.symbolName).filter(Boolean) || []
+                };
+
+                batch.set(db.collection(targetCollectionPath).doc(post.id), docData);
+                batch.set(processedRef.doc(post.id), { processedAt: FieldValue.serverTimestamp() });
+                pageBatchCount++;
+            }
+
             if (pageBatchCount > 0) {
                 await batch.commit();
                 totalSaved += pageBatchCount;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.449",
+  "version": "1.0.451",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [