bulltrackers-module 1.0.448 → 1.0.450

package/functions/computation-system/utils/data_loader.js

@@ -199,8 +199,8 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
         signedIn: {}            // Map<UserId, Map<PostId, Data>> - For Signed-In Users
     };
 
-    const PI_COL_NAME = config.piSocialCollectionName || 'pi_social_posts';
-    const SIGNED_IN_COL_NAME = config.signedInUserSocialCollection || 'signed_in_users';
+    const PI_COL_NAME = config.piSocialCollectionName || config.piSocialCollection || 'pi_social_posts';
+    const SIGNED_IN_COL_NAME = config.signedInUserSocialCollection || 'signed_in_users_social';
 
     // 2. Define Time Range (UTC Day)
     const startDate = new Date(dateString + 'T00:00:00Z');
@@ -232,7 +232,7 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
                     }
                 } 
                 else if (path.includes(SIGNED_IN_COL_NAME)) {
-                    // Path format: .../signed_in_users/{userId}/posts/{postId}
+                    // Path format: .../signed_in_users_social/{userId}/posts/{postId}
                     const parts = path.split('/');
                     const colIndex = parts.indexOf(SIGNED_IN_COL_NAME);
                     if (colIndex !== -1 && parts[colIndex + 1]) {

package/functions/root-data-indexer/index.js

@@ -70,7 +70,7 @@ exports.runRootDataIndexer = async (config, dependencies) => {
     
     // Collection Names (Fail-safe defaults)
     const PI_SOCIAL_COLL_NAME      = collections.piSocial || 'pi_social_posts';
-    const SIGNED_IN_SOCIAL_COLL_NAME = collections.signedInUserSocialCollection || 'signed_in_users';
+    const SIGNED_IN_SOCIAL_COLL_NAME = collections.signedInUserSocialCollection || 'signed_in_users_social';
 
     const scanMode = targetDate ? 'SINGLE_DATE' : 'FULL_SCAN';
     logger.log('INFO', `[RootDataIndexer] Starting Root Data Availability Scan... Mode: ${scanMode}`, { targetDate });
@@ -252,8 +252,14 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             if (!universalSocialSnap.empty) {
                 universalSocialSnap.docs.forEach(doc => {
                     const path = doc.ref.path;
-                    if (path.startsWith(PI_SOCIAL_COLL_NAME)) foundPISocial = true;
-                    if (path.startsWith(SIGNED_IN_SOCIAL_COLL_NAME)) foundSignedInSocial = true;
+                    // Use includes() to match collection name anywhere in path (more robust)
+                    // Path format: {collectionName}/{userId}/posts/{postId}
+                    if (path.includes(`/${PI_SOCIAL_COLL_NAME}/`) || path.startsWith(`${PI_SOCIAL_COLL_NAME}/`)) {
+                        foundPISocial = true;
+                    }
+                    if (path.includes(`/${SIGNED_IN_SOCIAL_COLL_NAME}/`) || path.startsWith(`${SIGNED_IN_SOCIAL_COLL_NAME}/`)) {
+                        foundSignedInSocial = true;
+                    }
                 });
             }
 

package/functions/social-task-handler/helpers/handler_helpers.js

@@ -133,7 +133,25 @@ async function getAdvancedAnalysisFromGemini(dependencies, snippet) {
         };
 
     } catch (e) {
-        logger.log("ERROR", "[Gemini AI] JSON handling failed", e);
+        logger.log("ERROR", "[Gemini AI] JSON handling failed", {
+            error: e.message,
+            stack: e.stack,
+            snippet: snippet.substring(0, 100),
+            part: part ? {
+                hasJson: !!part.json,
+                hasText: !!part.text,
+                textPreview: part.text ? part.text.substring(0, 200) : null
+            } : null,
+            resultStructure: result?.response ? {
+                hasCandidates: !!result.response.candidates,
+                candidatesLength: result.response.candidates?.length,
+                firstCandidate: result.response.candidates?.[0] ? {
+                    hasContent: !!result.response.candidates[0].content,
+                    hasParts: !!result.response.candidates[0].content?.parts,
+                    partsLength: result.response.candidates[0].content?.parts?.length
+                } : null
+            } : null
+        });
         return { overallSentiment: "Neutral", topics: [], isSpam: false, qualityScore: 0.5 };
     }
 }
@@ -235,21 +253,36 @@ exports.handleSocialTask = async (message, context, config, dependencies) => {
                 existingDocs.forEach(d => existingIds.add(d.id));
             }
 
+            // Filter and prepare posts: only new posts in English
+            const newPostsWithDiscussion = discussions
+                .filter(d => {
+                    const post = d.post;
+                    if (!post || !post.id || !post.message?.text || existingIds.has(post.id)) return false;
+                    const lang = (post.message.languageCode || 'unknown').toLowerCase();
+                    return lang === 'en' || lang === 'en-gb';
+                })
+                // Sort by text length (longer posts are generally more useful for analysis)
+                .sort((a, b) => (b.post.message.text?.length || 0) - (a.post.message.text?.length || 0));
+
+            // COST OPTIMIZATION: Cap AI processing to top 5 posts per user to reduce costs
+            // Remaining posts will be stored with default AI values
+            const MAX_POSTS_FOR_AI = 5;
+            const postsForAI = newPostsWithDiscussion.slice(0, MAX_POSTS_FOR_AI);
+            const postsWithoutAI = newPostsWithDiscussion.slice(MAX_POSTS_FOR_AI);
+
+            logger.log('INFO', `[SocialTask/${taskId}] Filtered ${newPostsWithDiscussion.length} new posts. Sending ${postsForAI.length} to AI, ${postsWithoutAI.length} without AI.`);
+
             const batch = db.batch();
             let pageBatchCount = 0;
 
-            for (const discussion of discussions) {
+            // Process posts that will get AI analysis (top 5 by text length)
+            for (const discussion of postsForAI) {
                 if (totalSaved + pageBatchCount >= MAX_POSTS_TO_STORE) {
                     keepFetching = false;
                     break;
                 }
 
                 const post = discussion.post;
-                if (!post || !post.id || !post.message?.text || existingIds.has(post.id)) continue;
-
-                const lang = (post.message.languageCode || 'unknown').toLowerCase();
-                if (lang !== 'en' && lang !== 'en-gb') continue;
-
                 const snippet = post.message.text.substring(0, 500);
                 const aiResult = await getAdvancedAnalysisFromGemini(dependencies, snippet);
 
@@ -260,7 +293,7 @@ exports.handleSocialTask = async (message, context, config, dependencies) => {
                     username: post.owner?.username,
                     createdAt: post.created,
                     fetchedAt: FieldValue.serverTimestamp(),
-                    snippet: snippet, // Added for debugging
+                    snippet: snippet,
                     stats: {
                         likes: discussion.emotionsData?.like?.paging?.totalCount || 0,
                         comments: discussion.summary?.totalCommentsAndReplies || 0
@@ -274,6 +307,40 @@ exports.handleSocialTask = async (message, context, config, dependencies) => {
                 pageBatchCount++;
             }
 
+            // Process remaining posts WITHOUT AI analysis (to avoid costs, but still store them)
+            for (const discussion of postsWithoutAI) {
+                if (totalSaved + pageBatchCount >= MAX_POSTS_TO_STORE) {
+                    keepFetching = false;
+                    break;
+                }
+
+                const post = discussion.post;
+                const docData = {
+                    postId: post.id,
+                    text: post.message.text,
+                    ownerId: post.owner?.id,
+                    username: post.owner?.username,
+                    createdAt: post.created,
+                    fetchedAt: FieldValue.serverTimestamp(),
+                    snippet: post.message.text.substring(0, 500),
+                    stats: {
+                        likes: discussion.emotionsData?.like?.paging?.totalCount || 0,
+                        comments: discussion.summary?.totalCommentsAndReplies || 0
+                    },
+                    aiAnalysis: {
+                        overallSentiment: "Neutral",
+                        topics: [],
+                        isSpam: false,
+                        qualityScore: 0.5
+                    }, // Default values when AI is skipped to reduce costs
+                    tags: post.tags?.map(t => t.market?.symbolName).filter(Boolean) || []
+                };
+
+                batch.set(db.collection(targetCollectionPath).doc(post.id), docData);
+                batch.set(processedRef.doc(post.id), { processedAt: FieldValue.serverTimestamp() });
+                pageBatchCount++;
+            }
+
             if (pageBatchCount > 0) {
                 await batch.commit();
                 totalSaved += pageBatchCount;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.448",
+  "version": "1.0.450",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [